diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index d5b324a3..95570a51 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -73,6 +73,7 @@ Mangapanda https://www.mangapanda.com/ Chapters, Manga MangaPark https://mangapark.me/ Chapters, Manga Mangareader https://www.mangareader.net/ Chapters, Manga Mangoxo https://www.mangoxo.com/ Albums, Channels Optional +Naver https://blog.naver.com/ Blogs, Posts Newgrounds https://www.newgrounds.com/ individual Images, User Profiles, Videos Ngomik http://ngomik.in/ Chapters nhentai https://nhentai.net/ Galleries, Search Results @@ -111,7 +112,7 @@ SmugMug https://www.smugmug.com/ |smugmug-C| The /b/ Archive https://thebarchive.com/ Threads Tsumino https://www.tsumino.com/ Galleries, Search Results Optional Tumblr https://www.tumblr.com/ Likes, Posts, Tag-Searches, User Profiles Optional (OAuth) -Twitter https://twitter.com/ Media Timelines, Timelines, Tweets Optional +Twitter https://twitter.com/ Media Timelines, Search Results, Timelines, Tweets Optional VSCO https://vsco.co/ Collections, individual Images, User Profiles Wallhaven https://wallhaven.cc/ individual Images, Search Results |wallhaven-A| Warosu https://warosu.org/ Threads diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index df3a573d..6bbd0b47 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -66,6 +66,7 @@ modules = [ "mangastream", "mangoxo", "myportfolio", + "naver", "newgrounds", "ngomik", "nhentai", diff --git a/gallery_dl/extractor/naver.py b/gallery_dl/extractor/naver.py new file mode 100644 index 00000000..13da67bf --- /dev/null +++ b/gallery_dl/extractor/naver.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- + +# Copyright 2019 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://blog.naver.com/""" + +from .common import GalleryExtractor, Extractor, Message +from .. import text + + +class NaverBase(): + """Base class for naver extractors""" + category = "naver" + root = "https://blog.naver.com" + + +class NaverPostExtractor(NaverBase, GalleryExtractor): + """Extractor for blog posts on blog.naver.com""" + subcategory = "post" + filename_fmt = "{num:>03}.{extension}" + directory_fmt = ("{category}", "{blog_id} {user}", "{post_num} {title}") + archive_fmt = "{blog_id}_{post_num}_{num}" + pattern = (r"(?:https?://)?blog\.naver\.com/" + r"(?:PostView\.nhn\?blogId=(\w+)&logNo=(\d+)|(\w+)/(\d+)/?$)") + test = ( + ("https://blog.naver.com/rlfqjxm0/221430673006", { + "url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "keyword": "f0cc292cb29da5692217fae75aa6384763ebe72c", + }), + (("https://blog.naver.com/PostView.nhn" + "?blogId=rlfqjxm0&logNo=221430673006"), { + "url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "keyword": "f0cc292cb29da5692217fae75aa6384763ebe72c", + }), + ) + + def __init__(self, match): + blog_id = match.group(1) + if blog_id: + self.blog_id = blog_id + self.post_id = match.group(2) + else: + self.blog_id = match.group(3) + self.post_id = match.group(4) + + url = "{}/PostView.nhn?blogId={}&logNo={}".format( + self.root, self.blog_id, self.post_id) + GalleryExtractor.__init__(self, match, url) + + def metadata(self, page): + extr = text.extract_from(page) + return { + "title" : extr('"og:title" content="', '"'), + "description": extr('"og:description" content="', '"'), + "post_num" : text.parse_int(self.post_id), + "blog_num" : text.parse_int(extr("var blogNo = '", "'")), + "blog_id" : self.blog_id, + "user" : extr("var nickName = '", "'"), + "date" : text.parse_datetime( + extr('se_publishDate pcol2">', '<') or + extr('_postAddDate">', '<'), "%Y. %m. %d. %H:%M"), + } + + def images(self, page): + return [ + (url.replace("://post", "://blog", 1).partition("?")[0], None) + for url in text.extract_iter(page, 'data-lazy-src="', '"') + ] + + +class NaverBlogExtractor(NaverBase, Extractor): + """Extractor for a user's blog on blog.naver.com""" + subcategory = "blog" + pattern = (r"(?:https?://)?blog\.naver\.com/" + r"(?:PostList.nhn\?(?:[^&#]+&)*blogId=([^&#]+)|(\w+)/?$)") + test = ( + ("https://blog.naver.com/gukjung", { + "pattern": NaverPostExtractor.pattern, + "count": 12, + "range": "1-12", + }), + ("https://blog.naver.com/PostList.nhn?blogId=gukjung", { + "pattern": NaverPostExtractor.pattern, + "count": 12, + "range": "1-12", + }), + ) + + def __init__(self, match): + Extractor.__init__(self, match) + self.blog_id = match.group(1) or match.group(2) + + def items(self): + yield Message.Version, 1 + + # fetch first post number + url = "{}/PostList.nhn?blogId={}".format(self.root, self.blog_id) + post_num = text.extract( + self.request(url).text, 'gnFirstLogNo = "', '"', + )[0] + + # setup params for API calls + url = "{}/PostViewBottomTitleListAsync.nhn".format(self.root) + params = { + "blogId" : self.blog_id, + "logNo" : post_num or "0", + "viewDate" : "", + "categoryNo" : "", + "parentCategoryNo" : "", + "showNextPage" : "true", + "showPreviousPage" : "false", + "sortDateInMilli" : "", + "isThumbnailViewType": "false", + "countPerPage" : "", + } + + # loop over all posts + while True: + data = self.request(url, params=params).json() + + for post in data["postList"]: + post["url"] = "{}/PostView.nhn?blogId={}&logNo={}".format( + self.root, self.blog_id, post["logNo"]) + post["_extractor"] = NaverPostExtractor + yield Message.Queue, post["url"], post + + if not data["hasNextPage"]: + return + params["logNo"] = data["nextIndexLogNo"] + params["sortDateInMilli"] = data["nextIndexSortDate"]