[skeb] refactor 'sent-requests' and add tests

gallery_dl/extractor/skeb.py

@@ -8,6 +8,7 @@
 
 from .common import Extractor, Message
 from .. import text
+import itertools
 
 
 class SkebExtractor(Extractor):
@@ -22,7 +23,6 @@ class SkebExtractor(Extractor):
         Extractor.__init__(self, match)
         self.user_name = match.group(1)
         self.thumbnails = self.config("thumbnails", False)
-        self.sent_requests = self.config("sent-requests", False)
 
     def items(self):
         for user_name, post_num in self.posts():
@@ -35,18 +35,18 @@ class SkebExtractor(Extractor):
     def posts(self):
         """Return post number"""
 
-    def _pagination(self):
+    def _pagination(self, url, params):
-        url = "{}/api/users/{}/works".format(self.root, self.user_name)
-        params = {"role": "creator", "sort": "date", "offset": 0}
         headers = {"Referer": self.root, "Authorization": "Bearer null"}
-        do_requests = self.sent_requests
+        params["offset"] = 0
 
         while True:
             posts = self.request(url, params=params, headers=headers).json()
 
             for post in posts:
-                post_num = post["path"].rpartition("/")[2]
+                parts = post["path"].split("/")
-                user_name = post["path"].split("/")[1][1:]
+                user_name = parts[1][1:]
+                post_num = parts[3]
+
                 if post["private"]:
                     self.log.debug("Skipping @%s/%s (private)",
                                    user_name, post_num)
@@ -54,13 +54,7 @@ class SkebExtractor(Extractor):
                 yield user_name, post_num
 
             if len(posts) < 30:
-                if do_requests:
+                return
-                    params["offset"] = 0
-                    params['role'] = "client"
-                    do_requests = False
-                    continue
-                else:
-                    return
             params["offset"] += 30
 
     def _get_post_data(self, user_name, post_num):
@@ -134,6 +128,54 @@ class SkebPostExtractor(SkebExtractor):
     """Extractor for a single skeb post"""
     subcategory = "post"
     pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/works/(\d+)"
+    test = ("https://skeb.jp/@kanade_cocotte/works/38", {
+        "count": 2,
+        "keyword": {
+            "anonymous": False,
+            "body": "re:はじめまして。私はYouTubeにてVTuberとして活動をしている湊ラ",
+            "client": {
+                "avatar_url": "https://pbs.twimg.com/profile_images"
+                              "/1471184042791895042/f0DcWFGl.jpg",
+                "header_url": None,
+                "id": 1196514,
+                "name": "湊ラギ",
+                "screen_name": "minato_ragi",
+            },
+            "completed_at": "2022-02-27T14:03:45.442Z",
+            "content_category": "preview",
+            "creator": {
+                "avatar_url": "https://pbs.twimg.com/profile_images"
+                              "/1225470417063645184/P8_SiB0V.jpg",
+                "header_url": "https://pbs.twimg.com/profile_banners"
+                              "/71243217/1647958329/1500x500",
+                "id": 159273,
+                "name": "イチノセ奏",
+                "screen_name": "kanade_cocotte",
+            },
+            "date": "dt:2022-02-27 14:03:45",
+            "file_id": int,
+            "file_url": str,
+            "genre": "art",
+            "nsfw": False,
+            "original": {
+                "byte_size": int,
+                "duration": None,
+                "extension": "re:psd|png",
+                "frame_rate": None,
+                "height": 3727,
+                "is_movie": False,
+                "width": 2810,
+            },
+            "post_num": "38",
+            "post_url": "https://skeb.jp/@kanade_cocotte/works/38",
+            "source_body": None,
+            "source_thanks": None,
+            "tags": list,
+            "thanks": None,
+            "translated_body": False,
+            "translated_thanks": None,
+        }
+    })
 
     def __init__(self, match):
         SkebExtractor.__init__(self, match)
@@ -146,7 +188,23 @@ class SkebPostExtractor(SkebExtractor):
 class SkebUserExtractor(SkebExtractor):
     """Extractor for all posts from a skeb user"""
     subcategory = "user"
-    pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)"
+    pattern = r"(?:https?://)?skeb\.jp/@([^/?#]+)/?$"
+    test = ("https://skeb.jp/@kanade_cocotte", {
+        "pattern": r"https://skeb\.imgix\.net/uploads/origins/[\w-]+"
+                   r"\?bg=%23fff&auto=format&txtfont=bold&txtshad=70"
+                   r"&txtclr=BFFFFFFF&txtalign=middle%2Ccenter&txtsize=150"
+                   r"&txt=SAMPLE&w=800&s=\w+",
+        "range": "1-5",
+    })
 
     def posts(self):
-        return self._pagination()
+        url = "{}/api/users/{}/works".format(self.root, self.user_name)
+
+        params = {"role": "creator", "sort": "date"}
+        posts = self._pagination(url, params)
+
+        if self.config("sent-requests", False):
+            params = {"role": "client", "sort": "date"}
+            posts = itertools.chain(posts, self._pagination(url, params))
+
+        return posts