[kemonoparty] extract 'archives' metadata (#7195)

add 'archives' option for additional data

docs/configuration.rst

@@ -2995,6 +2995,19 @@ Description
     Download video files.
 
 
+extractor.kemonoparty.archives
+------------------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    Extract additional metadata for ``archives`` files, including
+    ``file``, ``file_list``, and ``password``.
+
+    Note: This requires 1 additional HTTP request per ``archives`` file.
+
+
 extractor.kemonoparty.comments
 ------------------------------
 Type

docs/gallery-dl.conf

@@ -368,6 +368,7 @@
             "password": "",
 
             "announcements": false,
+            "archives"     : false,
             "comments"     : false,
             "dms"          : false,
             "duplicates"   : false,

gallery_dl/extractor/kemonoparty.py

@@ -57,11 +57,13 @@ class KemonopartyExtractor(Extractor):
         find_hash = re.compile(HASH_PATTERN).match
         generators = self._build_file_generators(self.config("files"))
         announcements = True if self.config("announcements") else None
+        archives = True if self.config("archives") else False
         comments = True if self.config("comments") else False
         duplicates = True if self.config("duplicates") else False
         dms = True if self.config("dms") else None
         max_posts = self.config("max-posts")
         creator_info = {} if self.config("metadata", True) else None
+        exts_archive = {"zip", "rar", "7z"}
 
         # prevent files from being sent with gzip compression
         headers = {"Accept-Encoding": "identity"}
@@ -115,6 +117,7 @@ class KemonopartyExtractor(Extractor):
 
             files = []
             hashes = set()
+            post_archives = post["archives"] = []
 
             for file in itertools.chain.from_iterable(
                     g(post) for g in generators):
@@ -129,31 +132,44 @@ class KemonopartyExtractor(Extractor):
                             continue
                         hashes.add(hash)
                 else:
-                    file["hash"] = ""
+                    file["hash"] = hash = ""
-
-                files.append(file)
-
-            post["count"] = len(files)
-            yield Message.Directory, post
-
-            for post["num"], file in enumerate(files, 1):
-                post["_http_validate"] = None
-                post["hash"] = file["hash"]
-                post["type"] = file["type"]
-                url = file["path"]
-
-                text.nameext_from_url(file.get("name", url), post)
-                ext = text.ext_from_url(url)
-                if not post["extension"]:
-                    post["extension"] = ext
-                elif ext == "txt" and post["extension"] != "txt":
-                    post["_http_validate"] = _validate
 
                 if url[0] == "/":
                     url = self.root + "/data" + url
                 elif url.startswith(self.root):
                     url = self.root + "/data" + url[20:]
-                yield Message.Url, url, post
+                file["url"] = url
+
+                text.nameext_from_url(file.get("name", url), file)
+                ext = text.ext_from_url(url)
+                if not file["extension"]:
+                    file["extension"] = ext
+                elif ext == "txt" and file["extension"] != "txt":
+                    file["_http_validate"] = _validate
+                elif ext in exts_archive:
+                    if archives:
+                        try:
+                            data = self.api.posts_archives(file["hash"])
+                            data.update(file)
+                            post_archives.append(data)
+                        except Exception as exc:
+                            self.log.warning(
+                                "%s: Failed to retrieve archive metadata of "
+                                "'%s' (%s: %s)", post["id"], file.get("name"),
+                                exc.__class__.__name__, exc)
+                            post_archives.append(file.copy())
+                    else:
+                        post_archives.append(file.copy())
+
+                files.append(file)
+
+            post["count"] = len(files)
+            yield Message.Directory, post
+            for post["num"], file in enumerate(files, 1):
+                if "id" in file:
+                    del file["id"]
+                post.update(file)
+                yield Message.Url, file["url"], post
 
     def login(self):
         username, password = self._get_auth_info()
@@ -503,6 +519,10 @@ class KemonoAPI():
         params = {"q": query, "o": offset, "tag": tags}
         return self._pagination(endpoint, params, 50, "posts")
 
+    def posts_archives(self, file_hash):
+        endpoint = "/posts/archives/" + file_hash
+        return self._call(endpoint)["archive"]
+
     def creator_posts(self, service, creator_id, offset=0, query=None):
         endpoint = "/{}/user/{}".format(service, creator_id)
         params = {"q": query, "o": offset}

test/results/kemonoparty.py

@@ -45,9 +45,12 @@ __tests__ = (
     "#category": ("", "kemonoparty", "patreon"),
     "#class"   : kemonoparty.KemonopartyUserExtractor,
     "#urls"    : (
+        "https://kemono.su/data/83/61/8361560887a09c7b828d326b3e1a2f0288673741569a09d74bcd01e602d20db1.png",
         "https://kemono.su/data/03/e6/03e62592c3b616b8906c1aaa130bd9ceaa24d7f601b31f90cc11956a57ca1d82.png",
+        "https://kemono.su/data/83/0d/830d017873157d2e6544a0f23a47622ec1e91be09b5d7795eb22e32b3150c837.png",
         "https://kemono.su/data/6a/9b/6a9b6d93dcb86c24a48def1bb93ce2a9ad77393941f3469d87d39400433cf825.png",
-        "https://kemono.su/data/2a/b8/2ab8ba30644249e9516afaea05d61c0de14591cb9d232a2dc249650eb1a9a759.jpg",
+        "https://kemono.su/data/96/43/9643ac03888f3b199f4e769242477b8d4d4f96025b10ab3f28affc3a1ae6bf52.jpg",
+        "https://kemono.su/data/f7/a8/f7a87ccac5736f46190a53a2bb1ff3828230e90f480776759895fcba28375909.jpg",
         "https://kemono.su/data/b0/38/b03882c8b0ab3b1cf9fc658a2bb2f9ac6ad4f3449015311dcd2d7ee7f748db31.png",
     ),
 
@@ -74,6 +77,7 @@ __tests__ = (
     "#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd",
 
     "added"      : "2020-05-06T20:28:02.302000",
+    "archives"   : [],
     "content"    : str,
     "count"      : 1,
     "date"       : "dt:2019-08-10 17:09:04",
@@ -229,7 +233,7 @@ __tests__ = (
     "#class"   : kemonoparty.KemonopartyPostExtractor,
     "#urls"    : "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg",
 
-    "file"          : {
+    "file": {
         "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
         "name": "wip update.jpg",
         "path": "/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg",
@@ -248,7 +252,7 @@ __tests__ = (
     "hash"          : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
     "revision_id"   : 142470,
     "revision_index": 2,
-    "revision_count": 10,
+    "revision_count": 11,
     "revision_hash" : "e0e93281495e151b11636c156e52bfe9234c2a40",
 },
 
@@ -258,16 +262,22 @@ __tests__ = (
     "#category": ("", "kemonoparty", "patreon"),
     "#class"   : kemonoparty.KemonopartyPostExtractor,
     "#options" : {"revisions": "unique"},
-    "#urls"    : "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg",
+    "#urls"    : "https://kemono.su/data/e3/e6/e3e6287dbc0468dd2a9d28ed276ae86788907143acf2ba10ab886a3add4c436c.jpg",
     "#archive" : False,
 
     "filename"      : "wip update",
-    "hash"          : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
+    "hash"          : {
-    "revision_id"   : {9277608, 0},
+        "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
-    "revision_index": {1, 2},
+        "e3e6287dbc0468dd2a9d28ed276ae86788907143acf2ba10ab886a3add4c436c",
-    "revision_count": 2,
+    },
-    "revision_hash" : {"e0e93281495e151b11636c156e52bfe9234c2a40",
+    "revision_id"   : {9277608, 10619155, 0},
-                       "79d5967719583a6fa52b2fc143e6a80fcdf75fb8"},
+    "revision_index": {1, 2, 3},
+    "revision_count": 3,
+    "revision_hash" : {
+        "e0e93281495e151b11636c156e52bfe9234c2a40",
+        "bc5713195e14799da40c525381216c5a1a340b0f",
+        "9872bfb536a47cc69d95d2f195cd5c825808f089",
+    },
 },
 
 {
@@ -276,13 +286,16 @@ __tests__ = (
     "#category": ("", "kemonoparty", "patreon"),
     "#class"   : kemonoparty.KemonopartyPostExtractor,
     "#pattern" : r"https://kemono\.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86\.jpg",
-    "#count"   : 10,
+    "#count"   : 11,
     "#archive" : False,
 
-    "revision_id": range(134996, 9277608),
+    "revision_id": range(134996, 10619155),
-    "revision_index": range(1, 10),
+    "revision_index": range(1, 11),
-    "revision_count": 10,
+    "revision_count": 11,
-    "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40",
+    "revision_hash": {
+        "9872bfb536a47cc69d95d2f195cd5c825808f089",
+        "e0e93281495e151b11636c156e52bfe9234c2a40",
+    },
 },
 
 
@@ -314,6 +327,86 @@ __tests__ = (
     "type"     : "attachment",
 },
 
+{
+    "#url"     : "https://kemono.su/fanbox/user/49494721/post/9457614",
+    "#comment" : "archives",
+    "#category": ("", "kemonoparty", "fanbox"),
+    "#class"   : kemonoparty.KemonopartyPostExtractor,
+    "#options" : {"archives": True},
+
+    "archives": [
+        {
+            "file": {
+                "added": "2025-03-03T02:11:28.153911",
+                "ctime": "2025-03-03T02:05:15.810201",
+                "ext"  : ".zip",
+                "hash" : "c22c7e979355f633aaae4929b010816895a47ec37a9cfc25186a0952ec6e5774",
+                "id"   : 190824068,
+                "ihash": None,
+                "mime" : "application/zip",
+                "mtime": "2025-03-03T02:11:28.807462",
+                "size" : 18634288,
+            },
+            "file_list": [
+                "モナmp4形式まとめ/",
+                "モナmp4形式まとめ/Movie_1.mp4",
+                "モナmp4形式まとめ/Movie_2.mp4",
+                "モナmp4形式まとめ/Movie_3.mp4",
+                "モナmp4形式まとめ/Movie_4.mp4",
+                "モナmp4形式まとめ/Movie_5.mp4",
+                "モナmp4形式まとめ/Movie_End_3.mp4",
+            ],
+            "filename": "モナmp4形式まとめ",
+            "extension": "zip",
+            "hash": "c22c7e979355f633aaae4929b010816895a47ec37a9cfc25186a0952ec6e5774",
+            "name": "モナmp4形式まとめ.zip",
+            "password": None,
+            "path": "/c2/2c/c22c7e979355f633aaae4929b010816895a47ec37a9cfc25186a0952ec6e5774.zip",
+            "type": "attachment",
+            "url": "https://kemono.su/data/c2/2c/c22c7e979355f633aaae4929b010816895a47ec37a9cfc25186a0952ec6e5774.zip",
+        },
+        {
+            "file": {
+                "added": "2025-03-03T02:11:00.541142",
+                "ctime": "2025-03-03T02:04:56.754326",
+                "ext"  : ".zip",
+                "hash" : "f7b4dedd9742aeb8da56dc6fe07deb7639880d0800ac0b7a6e91f64ff6b40178",
+                "id"   : 190824029,
+                "ihash": None,
+                "mime" : "application/zip",
+                "mtime": "2025-03-03T02:11:01.110281",
+                "size" : 84738158,
+            },
+            "file_list": "len:229",
+            "filename": "モナUnity",
+            "extension": "zip",
+            "hash": "f7b4dedd9742aeb8da56dc6fe07deb7639880d0800ac0b7a6e91f64ff6b40178",
+            "name": "モナUnity.zip",
+            "password": None,
+            "path": "/f7/b4/f7b4dedd9742aeb8da56dc6fe07deb7639880d0800ac0b7a6e91f64ff6b40178.zip",
+            "type": "attachment",
+            "url": "https://kemono.su/data/f7/b4/f7b4dedd9742aeb8da56dc6fe07deb7639880d0800ac0b7a6e91f64ff6b40178.zip"
+        },
+    ],
+
+    "title": "モナ（Live2Dアニメ）",
+    "user": "49494721",
+    "username": "soso",
+    "user_profile": {
+        "id": "49494721",
+        "indexed": "2021-04-02T23:50:57.138135",
+        "name": "soso",
+        "public_id": "soso",
+        "relation_id": None,
+        "service": "fanbox",
+        "updated": "2025-03-03T03:29:08.598883",
+    },
+    "tags": [
+        "うごイラ",
+        "原神",
+    ],
+},
+
 {
     "#url"     : "https://kemono.su/discord/server/488668827274444803/608504710906904576",
     "#category": ("", "kemonoparty", "discord"),