[lofter] update

- add tests
- update docs/supportedsites
- provide 'date' metadata
- simplify/restructure some code

docs/supportedsites.md

@@ -547,6 +547,12 @@ Consider all listed sites to potentially be NSFW.
     <td>Blogs, Posts</td>
     <td></td>
 </tr>
+<tr>
+    <td>LOFTER</td>
+    <td>https://www.lofter.com/</td>
+    <td>Blog Posts, Posts</td>
+    <td></td>
+</tr>
 <tr>
     <td>Luscious</td>
     <td>https://members.luscious.net/</td>

gallery_dl/extractor/lofter.py

@@ -27,9 +27,8 @@ class LofterExtractor(Extractor):
                 post = post["post"]
 
             post["blog_name"] = post["blogInfo"]["blogName"]
-
+            post["date"] = text.parse_timestamp(post["publishTime"] // 1000)
             post_type = post["type"]
-            image_urls = []
 
             # Article
             if post_type == 1:
@@ -56,6 +55,7 @@ class LofterExtractor(Extractor):
                 image_urls = [x.partition("?")[0] for x in image_urls]
 
             else:
+                image_urls = ()
                 self.log.warning(
                     "%s: Unsupported post type '%s'.",
                     post["id"], post_type)
@@ -73,7 +73,7 @@ class LofterPostExtractor(LofterExtractor):
     """Extractor for a lofter post"""
     subcategory = "post"
     pattern = r"(?:https?://)?[\w-]+\.lofter\.com/post/([0-9a-f]+)_([0-9a-f]+)"
-    example = "https://blog_name.lofter.com/post/12345678_90abcdef"
+    example = "https://BLOG.lofter.com/post/12345678_90abcdef"
 
     def posts(self):
         blog_id, post_id = self.groups
@@ -89,21 +89,39 @@ class LofterBlogPostsExtractor(LofterExtractor):
                r"www\.lofter\.com/front/blog/home-page/([\w-]+)|"
                # https://<blog_name>.lofter.com/
                r"([\w-]+)\.lofter\.com"
-               r")")
+               r")/?(?:$|\?|#)")
-    example = "https://blog_name.lofter.com/"
+    example = "https://BLOG.lofter.com/"
 
     def posts(self):
         blog_name = self.groups[0] or self.groups[1]
-        posts = self.api.blog_posts(blog_name)
+        return self.api.blog_posts(blog_name)
-        return posts
 
 
 class LofterAPI():
+
     def __init__(self, extractor):
         self.extractor = extractor
 
+    def blog_posts(self, blog_name):
+        endpoint = "/v2.0/blogHomePage.api"
+        params = {
+            "method": "getPostLists",
+            "offset": 0,
+            "limit": 200,
+            "blogdomain": blog_name + ".lofter.com",
+        }
+        return self._pagination(endpoint, params)
+
+    def post(self, blog_id, post_id):
+        endpoint = "/oldapi/post/detail.api"
+        params = {
+            "targetblogid": blog_id,
+            "postid": post_id,
+        }
+        return self._call(endpoint, params)["posts"][0]
+
     def _call(self, endpoint, data):
-        url = "https://api.lofter.com{}".format(endpoint)
+        url = "https://api.lofter.com" + endpoint
         params = {
             'product': 'lofter-android-7.9.10'
         }
@@ -115,36 +133,15 @@ class LofterAPI():
             self.extractor.log.debug("Server response: %s", info)
             raise exception.StopExtraction("API request failed")
 
-        return info
+        return info["response"]
-
-    def blog_posts(self, blog_name):
-        endpoint = "/v2.0/blogHomePage.api"
-        params = {
-            "method": "getPostLists",
-            "offset": 0,
-            "limit": 200,
-            "blogdomain": "{}.lofter.com".format(blog_name),
-        }
 
+    def _pagination(self, endpoint, params):
         while True:
             data = self._call(endpoint, params)
-            posts = data["response"]["posts"]
+            posts = data["posts"]
 
-            for post in posts:
+            yield from posts
-                yield post
 
-            if params["offset"] + len(posts) < data["response"]["offset"]:
+            if params["offset"] + len(posts) < data["offset"]:
                 break
-
+            params["offset"] = data["offset"]
-            params["offset"] = data["response"]["offset"]
-
-    def post(self, blog_id, post_id):
-        endpoint = "/oldapi/post/detail.api"
-        params = {
-            "targetblogid": blog_id,
-            "postid": post_id,
-        }
-        data = self._call(endpoint, params)
-        posts = data["response"]["posts"]
-        post = posts[0]
-        return post

scripts/supportedsites.py

@@ -86,6 +86,7 @@ CATEGORY_MAP = {
     "kemonoparty"    : "Kemono",
     "koharu"         : "SchaleNetwork",
     "livedoor"       : "livedoor Blog",
+    "lofter"         : "LOFTER",
     "ohpolly"        : "Oh Polly",
     "omgmiamiswimwear": "Omg Miami Swimwear",
     "mangadex"       : "MangaDex",
@@ -265,6 +266,9 @@ SUBCATEGORY_MAP = {
     "lensdump": {
         "albums": "",
     },
+    "lofter": {
+        "blog-posts": "Blog Posts",
+    },
     "mangadex": {
         "feed" : "Followed Feed",
     },

test/results/lofter.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+from gallery_dl.extractor import lofter
+
+
+__tests__ = (
+{
+    "#url"  : "https://gengar563.lofter.com/post/1e82da8c_1c98dae1b",
+    "#class": lofter.LofterPostExtractor,
+    "#urls" : (
+        "https://imglf3.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJQ1RxY0lYaU1UUE9tQ0NvUE9rVXFpOFFEVzMwbnQ4aEFnPT0.jpg",
+        "https://imglf3.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJRWlXYTRVOEpXTU9TSGt3TjBDQ0JFZVpZMEJtWjFneVNBPT0.png",
+        "https://imglf6.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJR1d3Y2VvbTNTQlIvdFU1WWlqZHEzbjI4MFVNZVdoN3VBPT0.png",
+        "https://imglf6.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJTi83NDRDUjNvd3hySGxEZFovd2hwbi9oaG9NQ1hOUkZ3PT0.png",
+        "https://imglf4.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJUFczb2RKSVlpMHJkNy9kc3BSQVQvQm5DNzB4eVhxay9nPT0.png",
+        "https://imglf4.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJSStJZE9RYnJURktHazdIVHNNMjQ5eFJldHVTQy9XbDB3PT0.png",
+        "https://imglf3.lf127.net/img/S1d2QlVsWkJhSW1qcnpIS0ZSa3ZJSzFCWFlnUWgzb01DcUdpT1lreG5yQjJVMkhGS09HNGR3PT0.png",
+    ),
+
+    "blog_name": "gengar563",
+    "content"  : "<p>发了三次发不出有毒……</p> \n<p>二部运动au&nbsp;&nbsp;性转ac注意</p> \n<p>失去耐心.jpg</p>",
+    "date"     : "dt:2020-06-04 12:51:42",
+    "id"       : 7676472859,
+},
+
+{
+    "#url"    : "https://wooden-brain.lofter.com/post/1e60de5b_1c9bf8efb",
+    "#comment": "video",
+    "#class"  : lofter.LofterPostExtractor,
+    "#urls"   : (
+        "https://vodm2lzexwq.vod.126.net/vodm2lzexwq/Pc5jg1nL_3039990631_sd.mp4?resId=254486990bfa2cd7aa860229db639341_3039990631_1&sign=4j02HTHXqNfhaF%2B%2FO14Ny%2F9SMNZj%2FIjpJDCqXfYa4aM%3D",
+    ),
+
+    "blog_name": "wooden-brain",
+    "date"     : "dt:2020-06-24 11:01:59",
+    "id"       : 7679741691,
+},
+
+{
+    "#url"  : "https://gengar563.lofter.com/",
+    "#class": lofter.LofterBlogPostsExtractor,
+    "#range": "1-25",
+    "#count": 25,
+
+    "blog_name": "gengar563",
+    "date"     : "type:datetime",
+    "id"       : int,
+},
+
+{
+    "#url"  : "https://www.lofter.com/front/blog/home-page/gengar563",
+    "#class": lofter.LofterBlogPostsExtractor,
+},
+
+)