[naver] add blog and post extractors (closes #447)

docs/supportedsites.rst

@@ -73,6 +73,7 @@ Mangapanda           https://www.mangapanda.com/         Chapters, Manga
 MangaPark            https://mangapark.me/               Chapters, Manga
 Mangareader          https://www.mangareader.net/        Chapters, Manga
 Mangoxo              https://www.mangoxo.com/            Albums, Channels                                   Optional
+Naver                https://blog.naver.com/             Blogs, Posts
 Newgrounds           https://www.newgrounds.com/         individual Images, User Profiles, Videos
 Ngomik               http://ngomik.in/                   Chapters
 nhentai              https://nhentai.net/                Galleries, Search Results
@@ -111,7 +112,7 @@ SmugMug              https://www.smugmug.com/            |smugmug-C|
 The /b/ Archive      https://thebarchive.com/            Threads
 Tsumino              https://www.tsumino.com/            Galleries, Search Results                          Optional
 Tumblr               https://www.tumblr.com/             Likes, Posts, Tag-Searches, User Profiles          Optional (OAuth)
-Twitter              https://twitter.com/                Media Timelines, Timelines, Tweets                 Optional
+Twitter              https://twitter.com/                Media Timelines, Search Results, Timelines, Tweets Optional
 VSCO                 https://vsco.co/                    Collections, individual Images, User Profiles
 Wallhaven            https://wallhaven.cc/               individual Images, Search Results                  |wallhaven-A|
 Warosu               https://warosu.org/                 Threads

gallery_dl/extractor/__init__.py

@@ -66,6 +66,7 @@ modules = [
     "mangastream",
     "mangoxo",
     "myportfolio",
+    "naver",
     "newgrounds",
     "ngomik",
     "nhentai",

gallery_dl/extractor/naver.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2019 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Extractors for https://blog.naver.com/"""
+
+from .common import GalleryExtractor, Extractor, Message
+from .. import text
+
+
+class NaverBase():
+    """Base class for naver extractors"""
+    category = "naver"
+    root = "https://blog.naver.com"
+
+
+class NaverPostExtractor(NaverBase, GalleryExtractor):
+    """Extractor for blog posts on blog.naver.com"""
+    subcategory = "post"
+    filename_fmt = "{num:>03}.{extension}"
+    directory_fmt = ("{category}", "{blog_id} {user}", "{post_num} {title}")
+    archive_fmt = "{blog_id}_{post_num}_{num}"
+    pattern = (r"(?:https?://)?blog\.naver\.com/"
+               r"(?:PostView\.nhn\?blogId=(\w+)&logNo=(\d+)|(\w+)/(\d+)/?$)")
+    test = (
+        ("https://blog.naver.com/rlfqjxm0/221430673006", {
+            "url": "6c694f3aced075ed5e9511f1e796d14cb26619cc",
+            "keyword": "f0cc292cb29da5692217fae75aa6384763ebe72c",
+        }),
+        (("https://blog.naver.com/PostView.nhn"
+          "?blogId=rlfqjxm0&logNo=221430673006"), {
+            "url": "6c694f3aced075ed5e9511f1e796d14cb26619cc",
+            "keyword": "f0cc292cb29da5692217fae75aa6384763ebe72c",
+        }),
+    )
+
+    def __init__(self, match):
+        blog_id = match.group(1)
+        if blog_id:
+            self.blog_id = blog_id
+            self.post_id = match.group(2)
+        else:
+            self.blog_id = match.group(3)
+            self.post_id = match.group(4)
+
+        url = "{}/PostView.nhn?blogId={}&logNo={}".format(
+            self.root, self.blog_id, self.post_id)
+        GalleryExtractor.__init__(self, match, url)
+
+    def metadata(self, page):
+        extr = text.extract_from(page)
+        return {
+            "title"      : extr('"og:title" content="', '"'),
+            "description": extr('"og:description" content="', '"'),
+            "post_num"   : text.parse_int(self.post_id),
+            "blog_num"   : text.parse_int(extr("var blogNo = '", "'")),
+            "blog_id"    : self.blog_id,
+            "user"       : extr("var nickName = '", "'"),
+            "date"       : text.parse_datetime(
+                extr('se_publishDate pcol2">', '<') or
+                extr('_postAddDate">', '<'), "%Y. %m. %d. %H:%M"),
+        }
+
+    def images(self, page):
+        return [
+            (url.replace("://post", "://blog", 1).partition("?")[0], None)
+            for url in text.extract_iter(page, 'data-lazy-src="', '"')
+        ]
+
+
+class NaverBlogExtractor(NaverBase, Extractor):
+    """Extractor for a user's blog on blog.naver.com"""
+    subcategory = "blog"
+    pattern = (r"(?:https?://)?blog\.naver\.com/"
+               r"(?:PostList.nhn\?(?:[^&#]+&)*blogId=([^&#]+)|(\w+)/?$)")
+    test = (
+        ("https://blog.naver.com/gukjung", {
+            "pattern": NaverPostExtractor.pattern,
+            "count": 12,
+            "range": "1-12",
+        }),
+        ("https://blog.naver.com/PostList.nhn?blogId=gukjung", {
+            "pattern": NaverPostExtractor.pattern,
+            "count": 12,
+            "range": "1-12",
+        }),
+    )
+
+    def __init__(self, match):
+        Extractor.__init__(self, match)
+        self.blog_id = match.group(1) or match.group(2)
+
+    def items(self):
+        yield Message.Version, 1
+
+        # fetch first post number
+        url = "{}/PostList.nhn?blogId={}".format(self.root, self.blog_id)
+        post_num = text.extract(
+            self.request(url).text, 'gnFirstLogNo = "', '"',
+        )[0]
+
+        # setup params for API calls
+        url = "{}/PostViewBottomTitleListAsync.nhn".format(self.root)
+        params = {
+            "blogId"             : self.blog_id,
+            "logNo"              : post_num or "0",
+            "viewDate"           : "",
+            "categoryNo"         : "",
+            "parentCategoryNo"   : "",
+            "showNextPage"       : "true",
+            "showPreviousPage"   : "false",
+            "sortDateInMilli"    : "",
+            "isThumbnailViewType": "false",
+            "countPerPage"       : "",
+        }
+
+        # loop over all posts
+        while True:
+            data = self.request(url, params=params).json()
+
+            for post in data["postList"]:
+                post["url"] = "{}/PostView.nhn?blogId={}&logNo={}".format(
+                    self.root, self.blog_id, post["logNo"])
+                post["_extractor"] = NaverPostExtractor
+                yield Message.Queue, post["url"], post
+
+            if not data["hasNextPage"]:
+                return
+            params["logNo"] = data["nextIndexLogNo"]
+            params["sortDateInMilli"] = data["nextIndexSortDate"]