[kemono:discord] extract 'archives' metadata (#8898)

4a74bc6e30

gallery_dl/extractor/kemono.py

@@ -417,6 +417,8 @@ class KemonoDiscordExtractor(KemonoExtractor):
             r"https?://(?:cdn\.discordapp.com|media\.discordapp\.net)"
             r"(/[A-Za-z0-9-._~:/?#\[\]@!$&'()*+,;%=]+)").findall
         find_hash = text.re(HASH_PATTERN).match
+        archives = True if self.config("archives") else False
+        exts_archive = util.EXTS_ARCHIVE
 
         if (order := self.config("order-posts")) and order[0] in ("r", "d"):
             posts = self.api.discord_channel(channel_id, channel["post_count"])
@@ -440,6 +442,8 @@ class KemonoDiscordExtractor(KemonoExtractor):
             post.update(data)
             post["date"] = self._parse_datetime(post["published"])
             post["count"] = len(files)
+            post["archives"] = post_archives = ()
+
             yield Message.Directory, "", post
 
             for post["num"], file in enumerate(files, 1):
@@ -451,6 +455,23 @@ class KemonoDiscordExtractor(KemonoExtractor):
                 if not post["extension"]:
                     post["extension"] = text.ext_from_url(url)
 
+                if post["extension"] in exts_archive:
+                    if not post_archives:
+                        post["archives"] = post_archives = []
+                    post["type"] = "archive"
+                    if archives:
+                        try:
+                            post_archives.append({
+                                **self.api.file(file["hash"]), **file})
+                        except Exception as exc:
+                            self.log.warning(
+                                "%s: Failed to retrieve archive metadata of "
+                                "'%s' (%s: %s)", post["id"], file.get("name"),
+                                exc.__class__.__name__, exc)
+                            post_archives.append(file.copy())
+                    else:
+                        post_archives.append(file.copy())
+
                 if url[0] == "/":
                     url = f"{self.root}/data{url}"
                 elif url.startswith(self.root):

test/results/kemono.py

@@ -674,6 +674,85 @@ __tests__ = (
     "hash": "",
 },
 
+{
+    "#url"     : "https://kemono.cr/discord/server/814339508694155294/815230464306446346",
+    "#comment" : "discord archives (#8898)",
+    "#class"   : kemono.KemonoDiscordExtractor,
+    "#options" : {
+        "archives"   : True,
+        "order-posts": "asc",
+    },
+    "#range"   : "1",
+    "#results" : "https://kemono.cr/data/ae/16/ae16db15cc15cc250db003964d6cd3cf2590863d925d96730871b6e75db3e69a.zip",
+
+    "added"        : "2021-10-24T09:11:03.268740",
+    "channel"      : "宝箱エリア",
+    "channel_id"   : "815230464306446346",
+    "channel_nsfw" : True,
+    "channel_topic": None,
+    "channel_type" : 0,
+    "content"      : "",
+    "count"        : 1,
+    "date"         : "dt:2021-02-28 03:17:21",
+    "edited"       : None,
+    "embeds"       : [],
+    "extension"    : "zip",
+    "filename"     : "Hachikuji_F",
+    "hash"         : "ae16db15cc15cc250db003964d6cd3cf2590863d925d96730871b6e75db3e69a",
+    "id"           : "815422363813675048",
+    "mentions"     : [],
+    "num"          : 1,
+    "parent_id"    : None,
+    "published"    : "2021-02-28T03:17:21.348000",
+    "revisions"    : [],
+    "seq"          : 1,
+    "server"       : "隠しエリア",
+    "server_id"    : "814339508694155294",
+    "type"         : "archive",
+    "archives"     : [{
+        "hash"     : "ae16db15cc15cc250db003964d6cd3cf2590863d925d96730871b6e75db3e69a",
+        "name"     : "Hachikuji_F.zip",
+        "password" : "894F",
+        "path"     : "/ae/16/ae16db15cc15cc250db003964d6cd3cf2590863d925d96730871b6e75db3e69a.zip",
+        "type"     : "attachment",
+        "file_list": [
+            "Hachikuji1.png",
+            "Hachikuji2.png",
+            "Hachikuji3.png",
+            "Hachikuji4.png",
+            "Hachikuji5.png",
+            "Hachikuji6.png",
+            "Hachikuji7.png",
+            "Hachikuji8.png",
+        ],
+        "file"     : {
+            "added": "2021-10-24T09:11:44.375913",
+            "ctime": "2021-10-24T09:11:44.318451",
+            "ext"  : ".zip",
+            "flags": "00000011",
+            "hash" : "ae16db15cc15cc250db003964d6cd3cf2590863d925d96730871b6e75db3e69a",
+            "id"   : 7886691,
+            "ihash": None,
+            "mime" : "application/zip",
+            "mtime": "2021-10-24T09:11:44.318451",
+            "size" : 12572064,
+        },
+    }],
+    "attachments"  : [{
+        "hash": "ae16db15cc15cc250db003964d6cd3cf2590863d925d96730871b6e75db3e69a",
+        "name": "Hachikuji_F.zip",
+        "path": "/ae/16/ae16db15cc15cc250db003964d6cd3cf2590863d925d96730871b6e75db3e69a.zip",
+        "type": "attachment",
+    }],
+    "author"       : {
+        "avatar"       : "336bbb6864275a8ca05c3de4ec5d5984",
+        "discriminator": "6362",
+        "id"           : "798533277852893218",
+        "public_flags" : 0,
+        "username"     : "影おじ",
+    },
+},
+
 {
     "#url"     : "https://kemono.cr/discord/server/488668827274444803",
     "#category": ("", "kemono", "discord-server"),