diff --git a/docs/supportedsites.md b/docs/supportedsites.md index f9afcae5..3ba8b2b2 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -535,6 +535,12 @@ Consider all listed sites to potentially be NSFW. Blogs, Posts + + LOFTER + https://www.lofter.com/ + Blog Posts, Posts + + Luscious https://members.luscious.net/ diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index d0fc1535..d003a61a 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -98,6 +98,7 @@ modules = [ "lexica", "lightroom", "livedoor", + "lofter", "luscious", "lynxchan", "mangadex", diff --git a/gallery_dl/extractor/lofter.py b/gallery_dl/extractor/lofter.py new file mode 100644 index 00000000..412b6b9f --- /dev/null +++ b/gallery_dl/extractor/lofter.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://www.lofter.com/""" + +from .common import Extractor, Message +from .. import text, util, exception + + +class LofterExtractor(Extractor): + """Base class for lofter extractors""" + category = "lofter" + root = "https://www.lofter.com" + directory_fmt = ("{category}", "{blog_name}") + filename_fmt = "{id}_{num}.{extension}" + archive_fmt = "{id}_{num}" + + def _init(self): + self.api = LofterAPI(self) + + def items(self): + for post in self.posts(): + if "post" in post: + post = post["post"] + + post["blog_name"] = post["blogInfo"]["blogName"] + post["date"] = text.parse_timestamp(post["publishTime"] // 1000) + post_type = post["type"] + + # Article + if post_type == 1: + content = post["content"] + image_urls = text.extract_iter(content, ' + r"www\.lofter\.com/front/blog/home-page/([\w-]+)|" + # https://.lofter.com/ + r"([\w-]+)\.lofter\.com" + r")/?(?:$|\?|#)") + example = "https://BLOG.lofter.com/" + + def posts(self): + blog_name = self.groups[0] or self.groups[1] + return self.api.blog_posts(blog_name) + + +class LofterAPI(): + + def __init__(self, extractor): + self.extractor = extractor + + def blog_posts(self, blog_name): + endpoint = "/v2.0/blogHomePage.api" + params = { + "method": "getPostLists", + "offset": 0, + "limit": 200, + "blogdomain": blog_name + ".lofter.com", + } + return self._pagination(endpoint, params) + + def post(self, blog_id, post_id): + endpoint = "/oldapi/post/detail.api" + params = { + "targetblogid": blog_id, + "postid": post_id, + } + return self._call(endpoint, params)["posts"][0] + + def _call(self, endpoint, data): + url = "https://api.lofter.com" + endpoint + params = { + 'product': 'lofter-android-7.9.10' + } + response = self.extractor.request( + url, method="POST", params=params, data=data) + info = response.json() + + if info["meta"]["status"] != 200: + self.extractor.log.debug("Server response: %s", info) + raise exception.StopExtraction("API request failed") + + return info["response"] + + def _pagination(self, endpoint, params): + while True: + data = self._call(endpoint, params) + posts = data["posts"] + + yield from posts + + if params["offset"] + len(posts) < data["offset"]: + break + params["offset"] = data["offset"] diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index aa2cdefe..2bbf9f6f 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -86,6 +86,7 @@ CATEGORY_MAP = { "kemonoparty" : "Kemono", "koharu" : "SchaleNetwork", "livedoor" : "livedoor Blog", + "lofter" : "LOFTER", "ohpolly" : "Oh Polly", "omgmiamiswimwear": "Omg Miami Swimwear", "mangadex" : "MangaDex", @@ -266,6 +267,9 @@ SUBCATEGORY_MAP = { "lensdump": { "albums": "", }, + "lofter": { + "blog-posts": "Blog Posts", + }, "mangadex": { "feed" : "Followed Feed", }, diff --git a/test/results/lofter.py b/test/results/lofter.py new file mode 100644 index 00000000..99a8f570 --- /dev/null +++ b/test/results/lofter.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import lofter + + +__tests__ = ( +{ + "#url" : "https://gengar563.lofter.com/post/1e82da8c_1c98dae1b", + "#class": lofter.LofterPostExtractor, + "#urls" : ( + "https://imglf3.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJQ1RxY0lYaU1UUE9tQ0NvUE9rVXFpOFFEVzMwbnQ4aEFnPT0.jpg", + "https://imglf3.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJRWlXYTRVOEpXTU9TSGt3TjBDQ0JFZVpZMEJtWjFneVNBPT0.png", + "https://imglf6.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJR1d3Y2VvbTNTQlIvdFU1WWlqZHEzbjI4MFVNZVdoN3VBPT0.png", + "https://imglf6.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJTi83NDRDUjNvd3hySGxEZFovd2hwbi9oaG9NQ1hOUkZ3PT0.png", + "https://imglf4.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJUFczb2RKSVlpMHJkNy9kc3BSQVQvQm5DNzB4eVhxay9nPT0.png", + "https://imglf4.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJSStJZE9RYnJURktHazdIVHNNMjQ5eFJldHVTQy9XbDB3PT0.png", + "https://imglf3.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJSzFCWFlnUWgzb01DcUdpT1lreG5yQjJVMkhGS09HNGR3PT0.png", + ), + + "blog_name": "gengar563", + "content" : "

发了三次发不出有毒……

\n

二部运动au  性转ac注意

\n

失去耐心.jpg

", + "date" : "dt:2020-06-04 12:51:42", + "id" : 7676472859, +}, + +{ + "#url" : "https://wooden-brain.lofter.com/post/1e60de5b_1c9bf8efb", + "#comment": "video", + "#class" : lofter.LofterPostExtractor, + "#urls" : ( + "https://vodm2lzexwq.vod.126.net/vodm2lzexwq/Pc5jg1nL_3039990631_sd.mp4?resId=254486990bfa2cd7aa860229db639341_3039990631_1&sign=4j02HTHXqNfhaF%2B%2FO14Ny%2F9SMNZj%2FIjpJDCqXfYa4aM%3D", + ), + + "blog_name": "wooden-brain", + "date" : "dt:2020-06-24 11:01:59", + "id" : 7679741691, +}, + +{ + "#url" : "https://gengar563.lofter.com/", + "#class": lofter.LofterBlogPostsExtractor, + "#range": "1-25", + "#count": 25, + + "blog_name": "gengar563", + "date" : "type:datetime", + "id" : int, +}, + +{ + "#url" : "https://www.lofter.com/front/blog/home-page/gengar563", + "#class": lofter.LofterBlogPostsExtractor, +}, + +)