[kemonoparty:discord] update server & channel metadata (#7569)

to match the Discord Extractors' names

- rename
    server       -> server_id
    channel      -> channel_id
    channel_name -> channel
- add
    server
    channel_nsfw
    channel_type
    channel_topic
    parent_id

gallery_dl/extractor/kemonoparty.py

@@ -10,7 +10,7 @@
 
 from .common import Extractor, Message
 from .. import text, util, exception
-from ..cache import cache
+from ..cache import cache, memcache
 import itertools
 import json
 import re
@@ -381,34 +381,32 @@ class KemonopartyPostExtractor(KemonopartyExtractor):
 class KemonopartyDiscordExtractor(KemonopartyExtractor):
     """Extractor for kemono.su discord servers"""
     subcategory = "discord"
-    directory_fmt = ("{category}", "discord", "{server}",
+    directory_fmt = ("{category}", "discord",
-                     "{channel_name|channel}")
+                     "{server_id} {server}", "{channel_id} {channel}")
     filename_fmt = "{id}_{num:>02}_{filename}.{extension}"
-    archive_fmt = "discord_{server}_{id}_{num}"
+    archive_fmt = "discord_{server_id}_{id}_{num}"
-    pattern = (BASE_PATTERN + r"/discord/server/(\d+)"
+    pattern = BASE_PATTERN + r"/discord/server/(\d+)[/#](?:channel/)?(\d+)"
-               r"(?:/(?:channel/)?(\d+)(?:#(.+))?|#(.+))")
     example = "https://kemono.su/discord/server/12345/12345"
 
     def items(self):
-        _, _, server_id, channel_id, channel_name, channel = self.groups
+        _, _, server_id, channel_id = self.groups
 
-        if channel_id is None:
+        try:
-            if channel.isdecimal() and len(channel) >= 16:
+            server, channels = discord_server_info(self, server_id)
-                key = "id"
+            channel = channels[channel_id]
-            else:
+        except Exception:
-                key = "name"
+            raise exception.NotFoundError("channel")
-        else:
-            key = "id"
-            channel = channel_id
 
-        if not channel_name or not channel_id:
+        data = {
-            for ch in self.api.discord_server(server_id):
+            "server"       : server["name"],
-                if ch[key] == channel:
+            "server_id"    : server["id"],
-                    break
+            "channel"      : channel["name"],
-            else:
+            "channel_id"   : channel["id"],
-                raise exception.NotFoundError("channel")
+            "channel_nsfw" : channel["is_nsfw"],
-            channel_id = ch["id"]
+            "channel_type" : channel["type"],
-            channel_name = ch["name"]
+            "channel_topic": channel["topic"],
+            "parent_id"    : channel["parent_channel_id"],
+        }
 
         find_inline = re.compile(
             r"https?://(?:cdn\.discordapp.com|media\.discordapp\.net)"
@@ -432,7 +430,7 @@ class KemonopartyDiscordExtractor(KemonopartyExtractor):
                 append({"path": "https://cdn.discordapp.com" + path,
                         "name": path, "type": "inline", "hash": ""})
 
-            post["channel_name"] = channel_name
+            post.update(data)
             post["date"] = self._parse_datetime(post["published"])
             post["count"] = len(files)
             yield Message.Directory, post
@@ -460,11 +458,24 @@ class KemonopartyDiscordServerExtractor(KemonopartyExtractor):
 
     def items(self):
         server_id = self.groups[2]
-        for channel in self.api.discord_server(server_id):
+        server, channels = discord_server_info(self, server_id)
+        for channel in channels.values():
             url = "{}/discord/server/{}/{}#{}".format(
                 self.root, server_id, channel["id"], channel["name"])
-            channel["_extractor"] = KemonopartyDiscordExtractor
+            yield Message.Queue, url, {
-            yield Message.Queue, url, channel
+                "server"    : server,
+                "channel"   : channel,
+                "_extractor": KemonopartyDiscordExtractor,
+            }
+
+
+@memcache(keyarg=1)
+def discord_server_info(extr, server_id):
+    server = extr.api.discord_server(server_id)
+    return server, {
+        channel["id"]: channel
+        for channel in server.pop("channels")
+    }
 
 
 class KemonopartyFavoriteExtractor(KemonopartyExtractor):
@@ -590,10 +601,14 @@ class KemonoAPI():
         endpoint = "/discord/channel/{}".format(channel_id)
         return self._pagination(endpoint, {}, 150)
 
-    def discord_server(self, server_id):
+    def discord_channel_lookup(self, server_id):
         endpoint = "/discord/channel/lookup/{}".format(server_id)
         return self._call(endpoint)
 
+    def discord_server(self, server_id):
+        endpoint = "/discord/server/{}".format(server_id)
+        return self._call(endpoint)
+
     def account_favorites(self, type):
         endpoint = "/account/favorites"
         params = {"type": type}

test/results/kemonoparty.py

@@ -517,8 +517,13 @@ __tests__ = (
     "#class"   : kemonoparty.KemonopartyDiscordExtractor,
     "#count"   : 4,
 
-    "channel"     : "608504710906904576",
+    "channel"      : "finish-work",
-    "channel_name": "finish-work",
+    "channel_id"   : "608504710906904576",
+    "channel_nsfw" : False,
+    "channel_topic": None,
+    "channel_type" : 0,
+    "server"       : "ABFMMD NSFW Server",
+    "server_id"    : "488668827274444803",
 },
 
 {
@@ -527,18 +532,13 @@ __tests__ = (
     "#class"   : kemonoparty.KemonopartyDiscordExtractor,
     "#count"   : 4,
 
-    "channel"     : "608504710906904576",
+    "channel"      : "finish-work",
-    "channel_name": "finish-work",
+    "channel_id"   : "608504710906904576",
-},
+    "channel_nsfw" : False,
-
+    "channel_topic": None,
-{
+    "channel_type" : 0,
-    "#url"     : "https://kemono.su/discord/server/488668827274444803#finish-work",
+    "server"       : "ABFMMD NSFW Server",
-    "#category": ("", "kemonoparty", "discord"),
+    "server_id"    : "488668827274444803",
-    "#class"   : kemonoparty.KemonopartyDiscordExtractor,
-    "#count"   : 4,
-
-    "channel"     : "608504710906904576",
-    "channel_name": "finish-work",
 },
 
 {
@@ -547,21 +547,31 @@ __tests__ = (
     "#class"   : kemonoparty.KemonopartyDiscordExtractor,
     "#count"   : 4,
 
-    "channel"     : "608504710906904576",
+    "channel"      : "finish-work",
-    "channel_name": "finish-work",
+    "channel_id"   : "608504710906904576",
-    "date"        : "type:datetime",
+    "channel_nsfw" : False,
+    "channel_topic": None,
+    "channel_type" : 0,
+    "server"       : "ABFMMD NSFW Server",
+    "server_id"    : "488668827274444803",
+    "date"         : "type:datetime",
 },
 
 {
-    "#url"     : "https://kemono.su/discord/server/818188637329031199#818343747275456522",
+    "#url"     : "https://kemono.su/discord/server/818188637329031199/818343747275456522",
     "#comment" : "pagination",
     "#category": ("", "kemonoparty", "discord"),
     "#class"   : kemonoparty.KemonopartyDiscordExtractor,
     "#range"   : "1-250",
     "#count"   : 250,
 
-    "channel"     : "818343747275456522",
+    "channel"      : "wraith-sfw-gallery",
-    "channel_name": "wraith-sfw-gallery",
+    "channel_id"   : "818343747275456522",
+    "channel_nsfw" : False,
+    "channel_type" : 0,
+    "channel_topic": None,
+    "server"       : "The Ghost Zone",
+    "server_id"    : "818188637329031199",
 },
 
 {
@@ -571,7 +581,10 @@ __tests__ = (
     "#pattern" : r"https://kemono\.su/data/(e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)",
     "#count"   : ">= 2",
 
-    "hash": r"re:e377e3525164559484ace2e64425b0cec1db08|51453640a5e0a4d23fbf57fb85390f9c5ec154",
+    "hash": {
+        "51453640a5e0a4d23fbf57fb85390f9c5ec15459af0bb5ba65a83781056b68e2",
+        "e377e3525164559484ace2e64425b0cec1db0863b9398682b90a9af006d87758",
+    },
 },
 
 {