[kemonoparty] update to new site layout / API endpoints

(#6415, #6503, #6528, #6530, #6536)

… at least for the most part. Favorites are still broken, but the rest
should be functional again.

gallery_dl/extractor/kemonoparty.py

@@ -10,7 +10,7 @@
 
 from .common import Extractor, Message
 from .. import text, util, exception
-from ..cache import cache, memcache
+from ..cache import cache
 import itertools
 import json
 import re
@@ -38,6 +38,7 @@ class KemonopartyExtractor(Extractor):
         Extractor.__init__(self, match)
 
     def _init(self):
+        self.api = KemonoAPI(self)
         self.revisions = self.config("revisions")
         if self.revisions:
             self.revisions_unique = (self.revisions == "unique")
@@ -53,48 +54,53 @@ class KemonopartyExtractor(Extractor):
             sort_keys=True, separators=(",", ":")).encode
 
     def items(self):
+        service = self.groups[2]
+        creator_id = self.groups[3]
+
         find_hash = re.compile(HASH_PATTERN).match
         generators = self._build_file_generators(self.config("files"))
-        duplicates = self.config("duplicates")
+        announcements = True if self.config("announcements") else None
-        comments = self.config("comments")
+        comments = True if self.config("comments") else False
-        username = dms = announcements = None
+        duplicates = True if self.config("duplicates") else False
+        dms = True if self.config("dms") else None
+        profile = username = None
 
         # prevent files from being sent with gzip compression
         headers = {"Accept-Encoding": "identity"}
 
         if self.config("metadata"):
-            username = text.unescape(text.extract(
+            profile = self.api.creator_profile(service, creator_id)
-                self.request(self.user_url).text,
+            username = profile["name"]
-                '<meta name="artist_name" content="', '"')[0])
-        if self.config("dms"):
-            dms = True
-        if self.config("announcements"):
-            announcements = True
 
         posts = self.posts()
         max_posts = self.config("max-posts")
         if max_posts:
             posts = itertools.islice(posts, max_posts)
+        if self.revisions:
+            posts = self._revisions(posts)
 
         for post in posts:
-
             headers["Referer"] = "{}/{}/user/{}/post/{}".format(
                 self.root, post["service"], post["user"], post["id"])
             post["_http_headers"] = headers
             post["date"] = self._parse_datetime(
                 post.get("published") or post.get("added") or "")
 
-            if username:
+            if profile is not None:
                 post["username"] = username
+                post["user_profile"] = profile
             if comments:
-                post["comments"] = self._extract_comments(post)
+                post["comments"] = self.api.creator_post_comments(
+                    service, creator_id, post["id"])
             if dms is not None:
                 if dms is True:
-                    dms = self._extract_cards(post, "dms")
+                    dms = self.api.creator_dms(
+                        post["service"], post["user"])
                 post["dms"] = dms
             if announcements is not None:
                 if announcements is True:
-                    announcements = self._extract_cards(post, "announcements")
+                    announcements = self.api.creator_announcements(
+                        post["service"], post["user"])
                 post["announcements"] = announcements
 
             files = []
@@ -188,56 +194,21 @@ class KemonopartyExtractor(Extractor):
             filetypes = filetypes.split(",")
         return [genmap[ft] for ft in filetypes]
 
-    def _extract_comments(self, post):
-        url = "{}/{}/user/{}/post/{}".format(
-            self.root, post["service"], post["user"], post["id"])
-        page = self.request(url).text
-
-        comments = []
-        for comment in text.extract_iter(page, "<article", "</article>"):
-            extr = text.extract_from(comment)
-            cid = extr('id="', '"')
-            comments.append({
-                "id"  : cid,
-                "user": extr('href="#' + cid + '"', '</').strip(" \n\r>"),
-                "body": extr(
-                    '<section class="comment__body">', '</section>').strip(),
-                "date": extr('datetime="', '"'),
-            })
-        return comments
-
-    def _extract_cards(self, post, type):
-        url = "{}/{}/user/{}/{}".format(
-            self.root, post["service"], post["user"], type)
-        page = self.request(url).text
-
-        cards = []
-        for card in text.extract_iter(page, "<article", "</article>"):
-            footer = text.extr(card, "<footer", "</footer>")
-            cards.append({
-                "body": text.unescape(text.extr(
-                    card, "<pre>", "</pre></",
-                ).strip()),
-                "date": text.extr(footer, ': ', '\n'),
-            })
-        return cards
-
     def _parse_datetime(self, date_string):
         if len(date_string) > 19:
             date_string = date_string[:19]
         return text.parse_datetime(date_string, "%Y-%m-%dT%H:%M:%S")
 
-    @memcache(keyarg=1)
+    def _revisions(self, posts):
-    def _discord_channels(self, server):
+        return itertools.chain.from_iterable(
-        url = "{}/api/v1/discord/channel/lookup/{}".format(
+            self._revisions_post(post) for post in posts)
-            self.root, server)
-        return self.request(url).json()
 
-    def _revisions_post(self, post, url):
+    def _revisions_post(self, post):
         post["revision_id"] = 0
 
         try:
-            revs = self.request(url + "/revisions").json()
+            revs = self.api.creator_post_revisions(
+                post["service"], post["user"], post["id"])
         except exception.HttpError:
             post["revision_hash"] = self._revision_hash(post)
             post["revision_index"] = 1
@@ -268,8 +239,8 @@ class KemonopartyExtractor(Extractor):
 
         return revs
 
-    def _revisions_all(self, url):
+    def _revisions_all(self, service, creator_id, post_id):
-        revs = self.request(url + "/revisions").json()
+        revs = self.api.creator_post_revisions(service, creator_id, post_id)
 
         cnt = idx = len(revs)
         for rev in revs:
@@ -305,50 +276,30 @@ def _validate(response):
 class KemonopartyUserExtractor(KemonopartyExtractor):
     """Extractor for all posts from a kemono.su user listing"""
     subcategory = "user"
-    pattern = USER_PATTERN + r"/?(?:\?([^#]+))?(?:$|[?#])"
+    pattern = USER_PATTERN + r"/?(?:\?([^#]+))?(?:$|\?|#)"
     example = "https://kemono.su/SERVICE/user/12345"
 
     def __init__(self, match):
-        _, _, service, user_id, self.query = match.groups()
+        self.subcategory = match.group(3)
-        self.subcategory = service
         KemonopartyExtractor.__init__(self, match)
-        self.api_url = "{}/api/v1/{}/user/{}".format(
-            self.root, service, user_id)
-        self.user_url = "{}/{}/user/{}".format(self.root, service, user_id)
 
     def posts(self):
-        url = self.api_url
+        _, _, service, creator_id, query = self.groups
-        params = text.parse_query(self.query)
+        params = text.parse_query(query)
-        params["o"] = text.parse_int(params.get("o"))
+        return self.api.creator_posts(
-
+            service, creator_id, params.get("o"), params.get("q"))
-        while True:
-            posts = self.request(url, params=params).json()
-
-            if self.revisions:
-                for post in posts:
-                    post_url = "{}/api/v1/{}/user/{}/post/{}".format(
-                        self.root, post["service"], post["user"], post["id"])
-                    yield from self._revisions_post(post, post_url)
-            else:
-                yield from posts
-
-            if len(posts) < 50:
-                break
-            params["o"] += 50
 
 
 class KemonopartyPostsExtractor(KemonopartyExtractor):
     """Extractor for kemono.su post listings"""
     subcategory = "posts"
-    pattern = BASE_PATTERN + r"/posts(?:/?\?([^#]+))?"
+    pattern = BASE_PATTERN + r"/posts()()(?:/?\?([^#]+))?"
     example = "https://kemono.su/posts"
 
-    def __init__(self, match):
+    def posts(self):
-        KemonopartyExtractor.__init__(self, match)
+        params = text.parse_query(self.groups[4])
-        self.query = match.group(3)
+        return self.api.posts(
-        self.api_url = self.root + "/api/v1/posts"
+            params.get("o"), params.get("q"), params.get("tag"))
-
-    posts = KemonopartyUserExtractor.posts
 
 
 class KemonopartyPostExtractor(KemonopartyExtractor):
@@ -358,27 +309,23 @@ class KemonopartyPostExtractor(KemonopartyExtractor):
     example = "https://kemono.su/SERVICE/user/12345/post/12345"
 
     def __init__(self, match):
-        _, _, service, user_id, post_id, self.revision, self.revision_id = \
+        self.subcategory = match.group(3)
-            match.groups()
-        self.subcategory = service
         KemonopartyExtractor.__init__(self, match)
-        self.api_url = "{}/api/v1/{}/user/{}/post/{}".format(
-            self.root, service, user_id, post_id)
-        self.user_url = "{}/{}/user/{}".format(self.root, service, user_id)
 
     def posts(self):
-        if not self.revision:
+        _, _, service, creator_id, post_id, revision, revision_id = self.groups
-            post = self.request(self.api_url).json()
+        post = self.api.creator_post(service, creator_id, post_id)
-            if self.revisions:
+        if not revision:
-                return self._revisions_post(post, self.api_url)
+            return (post["post"],)
-            return (post,)
 
-        revs = self._revisions_all(self.api_url)
+        self.revisions = False
-        if not self.revision_id:
+
+        revs = self._revisions_all(service, creator_id, post_id)
+        if not revision_id:
             return revs
 
         for rev in revs:
-            if str(rev["revision_id"]) == self.revision_id:
+            if str(rev["revision_id"]) == revision_id:
                 return (rev,)
 
         raise exception.NotFoundError("revision")
@@ -394,37 +341,35 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
     pattern = BASE_PATTERN + r"/discord/server/(\d+)(?:/channel/(\d+))?#(.*)"
     example = "https://kemono.su/discord/server/12345#CHANNEL"
 
-    def __init__(self, match):
-        KemonopartyExtractor.__init__(self, match)
-        _, _, self.server, self.channel_id, self.channel = match.groups()
-        self.channel_name = ""
-
     def items(self):
         self._prepare_ddosguard_cookies()
 
-        if self.channel_id:
+        _, _, server_id, channel_id, channel = self.groups
-            self.channel_name = self.channel
+        channel_name = ""
+
+        if channel_id:
+            channel_name = channel
         else:
-            if self.channel.isdecimal() and len(self.channel) >= 16:
+            if channel.isdecimal() and len(channel) >= 16:
                 key = "id"
             else:
                 key = "name"
 
-            for channel in self._discord_channels(self.server):
+            for ch in self.api.discord_server(server_id):
-                if channel[key] == self.channel:
+                if ch[key] == channel:
                     break
             else:
                 raise exception.NotFoundError("channel")
 
-            self.channel_id = channel["id"]
+            channel_id = ch["id"]
-            self.channel_name = channel["name"]
+            channel_name = ch["name"]
 
         find_inline = re.compile(
             r"https?://(?:cdn\.discordapp.com|media\.discordapp\.net)"
             r"(/[A-Za-z0-9-._~:/?#\[\]@!$&'()*+,;%=]+)").findall
         find_hash = re.compile(HASH_PATTERN).match
 
-        posts = self.posts()
+        posts = self.api.discord_channel(channel_id)
         max_posts = self.config("max-posts")
         if max_posts:
             posts = itertools.islice(posts, max_posts)
@@ -441,7 +386,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
                 append({"path": "https://cdn.discordapp.com" + path,
                         "name": path, "type": "inline", "hash": ""})
 
-            post["channel_name"] = self.channel_name
+            post["channel_name"] = channel_name
             post["date"] = self._parse_datetime(post["published"])
             post["count"] = len(files)
             yield Message.Directory, post
@@ -461,33 +406,17 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
                     url = self.root + "/data" + url[20:]
                 yield Message.Url, url, post
 
-    def posts(self):
-        url = "{}/api/v1/discord/channel/{}".format(
-            self.root, self.channel_id)
-        params = {"o": 0}
-
-        while True:
-            posts = self.request(url, params=params).json()
-            yield from posts
-
-            if len(posts) < 150:
-                break
-            params["o"] += 150
-
 
 class KemonopartyDiscordServerExtractor(KemonopartyExtractor):
     subcategory = "discord-server"
     pattern = BASE_PATTERN + r"/discord/server/(\d+)$"
     example = "https://kemono.su/discord/server/12345"
 
-    def __init__(self, match):
-        KemonopartyExtractor.__init__(self, match)
-        self.server = match.group(3)
-
     def items(self):
-        for channel in self._discord_channels(self.server):
+        server_id = self.groups[2]
+        for channel in self.api.discord_server(server_id):
             url = "{}/discord/server/{}/channel/{}#{}".format(
-                self.root, self.server, channel["id"], channel["name"])
+                self.root, server_id, channel["id"], channel["name"])
             channel["_extractor"] = KemonopartyDiscordExtractor
             yield Message.Queue, url, channel
 
@@ -541,3 +470,100 @@ class KemonopartyFavoriteExtractor(KemonopartyExtractor):
                 url = "{}/{}/user/{}/post/{}".format(
                     self.root, post["service"], post["user"], post["id"])
                 yield Message.Queue, url, post
+
+
+class KemonoAPI():
+    """Interface for the Kemono API v1.1.0
+
+    https://kemono.su/documentation/api
+    """
+
+    def __init__(self, extractor):
+        self.extractor = extractor
+        self.root = extractor.root + "/api/v1"
+
+    def posts(self, offset=0, query=None, tags=None):
+        endpoint = "/posts"
+        params = {"q": query, "o": offset, "tags": tags}
+        return self._pagination(endpoint, params, 50, "posts")
+
+    def creator_posts(self, service, creator_id, offset=0, query=None):
+        endpoint = "/{}/user/{}".format(service, creator_id)
+        params = {"q": query, "o": offset}
+        return self._pagination(endpoint, params, 50)
+
+    def creator_announcements(self, service, creator_id):
+        endpoint = "/{}/user/{}/announcements".format(service, creator_id)
+        return self._call(endpoint)
+
+    def creator_dms(self, service, creator_id):
+        endpoint = "/{}/user/{}/dms".format(service, creator_id)
+        return self._call(endpoint)
+
+    def creator_fancards(self, service, creator_id):
+        endpoint = "/{}/user/{}/fancards".format(service, creator_id)
+        return self._call(endpoint)
+
+    def creator_post(self, service, creator_id, post_id):
+        endpoint = "/{}/user/{}/post/{}".format(service, creator_id, post_id)
+        return self._call(endpoint)
+
+    def creator_post_comments(self, service, creator_id, post_id):
+        endpoint = "/{}/user/{}/post/{}/comments".format(
+            service, creator_id, post_id)
+        return self._call(endpoint)
+
+    def creator_post_revisions(self, service, creator_id, post_id):
+        endpoint = "/{}/user/{}/post/{}/revisions".format(
+            service, creator_id, post_id)
+        return self._call(endpoint)
+
+    def creator_profile(self, service, creator_id):
+        endpoint = "/{}/user/{}/profile".format(service, creator_id)
+        return self._call(endpoint)
+
+    def creator_links(self, service, creator_id):
+        endpoint = "/{}/user/{}/links".format(service, creator_id)
+        return self._call(endpoint)
+
+    def creator_tags(self, service, creator_id):
+        endpoint = "/{}/user/{}/tags".format(service, creator_id)
+        return self._call(endpoint)
+
+    def discord_channel(self, channel_id):
+        endpoint = "/discord/channel/{}".format(channel_id)
+        return self._pagination(endpoint, {}, 150)
+
+    def discord_server(self, server_id):
+        endpoint = "/discord/channel/lookup/{}".format(server_id)
+        return self._call(endpoint)
+
+    def account_favorites(self, type):
+        endpoint = "/account/favorites"
+        params = {"type": type}
+        return self._call(endpoint, params)
+
+    def authentication_login(self, username, password):
+        endpoint = "/authentication/login"
+        params = {"username": username, "password": password}
+        return self._call(endpoint, params)
+
+    def _call(self, endpoint, params=None):
+        url = self.root + endpoint
+        response = self.extractor.request(url, params=params)
+        return response.json()
+
+    def _pagination(self, endpoint, params, batch=50, key=False):
+        params["o"] = text.parse_int(params.get("o")) % 50
+
+        while True:
+            data = self._call(endpoint, params)
+
+            if key:
+                yield from data[key]
+            else:
+                yield from data
+
+            if len(data) < batch:
+                return
+            params["o"] += batch

test/results/kemonoparty.py

@@ -23,7 +23,7 @@ __tests__ = (
     "#category": ("", "kemonoparty", "patreon"),
     "#class"   : kemonoparty.KemonopartyUserExtractor,
     "#options" : {"max-posts": 100},
-    "#count"   : range(200, 300),
+    "#count"   : range(200, 400),
 },
 
 {
@@ -92,7 +92,7 @@ __tests__ = (
     "#url"     : "https://kemono.su/gumroad/user/3101696181060/post/tOWyf",
     "#category": ("", "kemonoparty", "gumroad"),
     "#class"   : kemonoparty.KemonopartyPostExtractor,
-    "#urls"    : "https://kemono.su/data/6f/13/6f1394b19516396ea520254350662c254bbea30c1e111fd4b0f042c61c426d07.zip",
+    "#count"   : 12,
 },
 
 {
@@ -129,10 +129,19 @@ __tests__ = (
     "#class"   : kemonoparty.KemonopartyPostExtractor,
     "#options" : {"dms": True},
 
-    "dms": [{
+    "dms": [
-        "body": r"re:Hi! Thank you very much for supporting the work I did in May. Here's your reward pack! I hope you find something you enjoy in it. :\)\n\nhttps://www.mediafire.com/file/\w+/Set13_tier_2.zip/file",
+        {
-        "date": "2021-06",
+            "added"    : "2021-07-31T02:47:51.327865",
-    }],
+            "artist"   : None,
+            "content"  : "Hi! Thank you very much for supporting the work I did in May. Here's your reward pack! I hope you find something you enjoy in it. :)\n\nhttps://www.mediafire.com/file/n9ppjpip0r3f01v/Set13_tier_2.zip/file",
+            "embed"    : {},
+            "file"     : {},
+            "hash"     : "f8d4962fb7908614c9b7c8c0de1b5f8985f01b62a9b06d74d640c5b2bcedf758",
+            "published": "2021-06-09T03:28:51.431000",
+            "service"  : "patreon",
+            "user"     : "34134344",
+        },
+    ],
 },
 
 {
@@ -142,10 +151,16 @@ __tests__ = (
     "#class"   : kemonoparty.KemonopartyPostExtractor,
     "#options" : {"announcements": True},
 
-    "announcements": [{
+    "announcements": [
-        "body": "<div><strong>Thank you so much for the support!</strong><strong><br></strong>This Patreon is more of a tip jar for supporting what I make. I have to clarify that there are <strong>no exclusive Patreon animations</strong> because all are released for the public. You will get earlier access to WIPs. Direct downloads to my works are also available for $5 and $10 Tiers.</div>",
+        {
-        "date": "2023-02",
+            "added"    : "2023-02-01T22:44:34.670719",
-    }],
+            "content"  : "<div style=\"text-align: center;\"><strong>Thank you so much for the support!</strong><strong><br></strong>This Patreon is more of a tip jar for supporting what I make. I have to clarify that there are <strong>no exclusive Patreon animations</strong>&nbsp;because all are released for the public. You will get earlier access to WIPs. Direct downloads to my works are also available for $5 and $10 Tiers.</div>",
+            "hash"     : "815648d41c60d1d546437e475a0888fd4a77fd098b1ec61a3648ea6da30c1034",
+            "published": None,
+            "service"  : "patreon",
+            "user_id"  : "3161935",
+        },
+    ],
 },
 
 {
@@ -207,7 +222,7 @@ __tests__ = (
     "hash"          : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
     "revision_id"   : 142470,
     "revision_index": 2,
-    "revision_count": 9,
+    "revision_count": 10,
     "revision_hash" : "e0e93281495e151b11636c156e52bfe9234c2a40",
 },
 
@@ -218,13 +233,15 @@ __tests__ = (
     "#class"   : kemonoparty.KemonopartyPostExtractor,
     "#options" : {"revisions": "unique"},
     "#urls"    : "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg",
+    "#archive" : False,
 
     "filename"      : "wip update",
     "hash"          : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
-    "revision_id"   : 0,
+    "revision_id"   : {9277608, 0},
-    "revision_index": 1,
+    "revision_index": {1, 2},
-    "revision_count": 1,
+    "revision_count": 2,
-    "revision_hash" : "e0e93281495e151b11636c156e52bfe9234c2a40",
+    "revision_hash" : {"e0e93281495e151b11636c156e52bfe9234c2a40",
+                       "79d5967719583a6fa52b2fc143e6a80fcdf75fb8"},
 },
 
 {
@@ -233,12 +250,12 @@ __tests__ = (
     "#category": ("", "kemonoparty", "patreon"),
     "#class"   : kemonoparty.KemonopartyPostExtractor,
     "#pattern" : r"https://kemono\.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86\.jpg",
-    "#count"   : 9,
+    "#count"   : 10,
     "#archive" : False,
 
-    "revision_id": range(134996, 3052965),
+    "revision_id": range(134996, 9277608),
-    "revision_index": range(1, 9),
+    "revision_index": range(1, 10),
-    "revision_count": 9,
+    "revision_count": 10,
     "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40",
 },
 
@@ -341,15 +358,7 @@ __tests__ = (
     "#category": ("", "kemonoparty", "discord-server"),
     "#class"   : kemonoparty.KemonopartyDiscordServerExtractor,
     "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern,
-    "#count"   : 13,
+    "#count"   : 15,
-},
-
-{
-    "#url"     : "https://kemono.su/discord/server/488668827274444803",
-    "#category": ("", "kemonoparty", "discord-server"),
-    "#class"   : kemonoparty.KemonopartyDiscordServerExtractor,
-    "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern,
-    "#count"   : 13,
 },
 
 {