[kemonoparty] various updates and fixes (#4676, #4681)

- fix pagination
- fix 'date' metadata
- fix discord channel API endpoint

gallery_dl/extractor/kemonoparty.py

@@ -70,8 +70,7 @@ class KemonopartyExtractor(Extractor):
                 self.root, post["service"], post["user"], post["id"])
             post["_http_headers"] = headers
             post["date"] = text.parse_datetime(
-                post["published"] or post["added"],
+                post["published"] or post["added"], "%Y-%m-%dT%H:%M:%S")
-                "%a, %d %b %Y %H:%M:%S %Z")
             if username:
                 post["username"] = username
             if comments:
@@ -208,7 +207,7 @@ class KemonopartyExtractor(Extractor):
 
     @memcache(keyarg=1)
     def _discord_channels(self, server):
-        url = "{}/api/v1/discord/channels/lookup/{}".format(
+        url = "{}/api/v1/discord/channel/lookup/{}".format(
             self.root, server)
         return self.request(url).json()
 
@@ -241,10 +240,9 @@ class KemonopartyUserExtractor(KemonopartyExtractor):
             posts = self.request(url, params=params).json()
             yield from posts
 
-            cnt = len(posts)
+            if len(posts) < 50:
-            if cnt < 25:
+                break
-                return
+            params["o"] += 50
-            params["o"] += cnt
 
 
 class KemonopartyPostExtractor(KemonopartyExtractor):
@@ -262,8 +260,7 @@ class KemonopartyPostExtractor(KemonopartyExtractor):
         self.user_url = "{}/{}/user/{}".format(self.root, service, user_id)
 
     def posts(self):
-        posts = self.request(self.api_url).json()
+        return (self.request(self.api_url).json(),)
-        return (posts[0],) if len(posts) > 1 else posts
 
 
 class KemonopartyDiscordExtractor(KemonopartyExtractor):
@@ -325,7 +322,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
 
             post["channel_name"] = self.channel_name
             post["date"] = text.parse_datetime(
-                post["published"], "%a, %d %b %Y %H:%M:%S %Z")
+                post["published"], "%Y-%m-%dT%H:%M:%S.%f")
             post["count"] = len(files)
             yield Message.Directory, post
 
@@ -348,15 +345,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
         url = "{}/api/v1/discord/channel/{}".format(
             self.root, self.channel_id)
         params = {"skip": 0}
-
+        return self.request(url, params=params).json()
-        while True:
-            posts = self.request(url, params=params).json()
-            yield from posts
-
-            cnt = len(posts)
-            if cnt < 25:
-                break
-            params["skip"] += cnt
 
 
 class KemonopartyDiscordServerExtractor(KemonopartyExtractor):

test/results/kemonoparty.py

@@ -12,8 +12,7 @@ __tests__ = (
     "#url"     : "https://kemono.party/fanbox/user/6993449",
     "#category": ("", "kemonoparty", "fanbox"),
     "#class"   : kemonoparty.KemonopartyUserExtractor,
-    "#range"   : "1-25",
+    "#count"   : 847,
-    "#count"   : 25,
 },
 
 {
@@ -21,8 +20,8 @@ __tests__ = (
     "#comment" : "'max-posts' option, 'o' query parameter (#1674)",
     "#category": ("", "kemonoparty", "patreon"),
     "#class"   : kemonoparty.KemonopartyUserExtractor,
-    "#options" : {"max-posts": 25},
+    "#options" : {"max-posts": 100},
-    "#count"   : "< 100",
+    "#count"   : range(200, 300),
 },
 
 {
@@ -44,7 +43,7 @@ __tests__ = (
     "#pattern"     : r"https://kemono.party/data/21/0f/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg",
     "#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd",
 
-    "added"      : "Wed, 06 May 2020 20:28:02 GMT",
+    "added"      : "2020-05-06T20:28:02.302000",
     "content"    : str,
     "count"      : 1,
     "date"       : "dt:2019-08-11 02:09:04",
@@ -55,7 +54,7 @@ __tests__ = (
     "hash"       : "210f35388e28bbcf756db18dd516e2d82ce758e0d32881eeee76d43e1716d382",
     "id"         : "506575",
     "num"        : 1,
-    "published"  : "Sun, 11 Aug 2019 02:09:04 GMT",
+    "published"  : "2019-08-11T02:09:04",
     "service"    : "fanbox",
     "shared_file": False,
     "subcategory": "fanbox",
@@ -183,6 +182,7 @@ __tests__ = (
 
     "channel"     : "608504710906904576",
     "channel_name": "finish-work",
+    "date"        : "type:datetime",
 },
 
 {