github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[kemonoparty] extract 'archives' metadata (#7195)

Browse Source 
add 'archives' option for additional data
This commit is contained in:
Mike Fährmann
2025-03-22 18:33:20 +01:00
parent 63a1052970
commit 4a74bc6e30
4 changed files with 162 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
docs/configuration.rst
View File

@@ -2995,6 +2995,19 @@ Description
Download video files.
extractor.kemonoparty.archives
------------------------------
Type
``bool``
Default
``false``
Description
Extract additional metadata for ``archives`` files, including
``file``, ``file_list``, and ``password``.
Note: This requires 1 additional HTTP request per ``archives`` file.
extractor.kemonoparty.comments
------------------------------
Type

1
docs/gallery-dl.conf
View File

@@ -368,6 +368,7 @@
"password": "",
"announcements": false,
"archives" : false,
"comments" : false,
"dms" : false,
"duplicates" : false,

60
gallery_dl/extractor/kemonoparty.py
View File

@@ -57,11 +57,13 @@ class KemonopartyExtractor(Extractor):
find_hash = re.compile(HASH_PATTERN).match
generators = self._build_file_generators(self.config("files"))
announcements = True if self.config("announcements") else None
archives = True if self.config("archives") else False
comments = True if self.config("comments") else False
duplicates = True if self.config("duplicates") else False
dms = True if self.config("dms") else None
max_posts = self.config("max-posts")
creator_info = {} if self.config("metadata", True) else None
exts_archive = {"zip", "rar", "7z"}
# prevent files from being sent with gzip compression
headers = {"Accept-Encoding": "identity"}
@@ -115,6 +117,7 @@ class KemonopartyExtractor(Extractor):
files = []
hashes = set()
post_archives = post["archives"] = []
for file in itertools.chain.from_iterable(
g(post) for g in generators):
@@ -129,31 +132,44 @@ class KemonopartyExtractor(Extractor):
continue
hashes.add(hash)
else:
file["hash"] = ""
files.append(file)
post["count"] = len(files)
yield Message.Directory, post
for post["num"], file in enumerate(files, 1):
post["_http_validate"] = None
post["hash"] = file["hash"]
post["type"] = file["type"]
url = file["path"]
text.nameext_from_url(file.get("name", url), post)
ext = text.ext_from_url(url)
if not post["extension"]:
post["extension"] = ext
elif ext == "txt" and post["extension"] != "txt":
post["_http_validate"] = _validate
file["hash"] = hash = ""
if url[0] == "/":
url = self.root + "/data" + url
elif url.startswith(self.root):
url = self.root + "/data" + url[20:]
yield Message.Url, url, post
file["url"] = url
text.nameext_from_url(file.get("name", url), file)
ext = text.ext_from_url(url)
if not file["extension"]:
file["extension"] = ext
elif ext == "txt" and file["extension"] != "txt":
file["_http_validate"] = _validate
elif ext in exts_archive:
if archives:
try:
data = self.api.posts_archives(file["hash"])
data.update(file)
post_archives.append(data)
except Exception as exc:
self.log.warning(
"%s: Failed to retrieve archive metadata of "
"'%s' (%s: %s)", post["id"], file.get("name"),
exc.__class__.__name__, exc)
post_archives.append(file.copy())
else:
post_archives.append(file.copy())
files.append(file)
post["count"] = len(files)
yield Message.Directory, post
for post["num"], file in enumerate(files, 1):
if "id" in file:
del file["id"]
post.update(file)
yield Message.Url, file["url"], post
def login(self):
username, password = self._get_auth_info()
@@ -503,6 +519,10 @@ class KemonoAPI():
params = {"q": query, "o": offset, "tag": tags}
return self._pagination(endpoint, params, 50, "posts")
def posts_archives(self, file_hash):
endpoint = "/posts/archives/" + file_hash
return self._call(endpoint)["archive"]
def creator_posts(self, service, creator_id, offset=0, query=None):
endpoint = "/{}/user/{}".format(service, creator_id)
params = {"q": query, "o": offset}

123
test/results/kemonoparty.py
View File

@@ -45,9 +45,12 @@ __tests__ = (
"#category": ("", "kemonoparty", "patreon"),
"#class" : kemonoparty.KemonopartyUserExtractor,
"#urls" : (
"https://kemono.su/data/83/61/8361560887a09c7b828d326b3e1a2f0288673741569a09d74bcd01e602d20db1.png",
"https://kemono.su/data/03/e6/03e62592c3b616b8906c1aaa130bd9ceaa24d7f601b31f90cc11956a57ca1d82.png",
"https://kemono.su/data/83/0d/830d017873157d2e6544a0f23a47622ec1e91be09b5d7795eb22e32b3150c837.png",
"https://kemono.su/data/6a/9b/6a9b6d93dcb86c24a48def1bb93ce2a9ad77393941f3469d87d39400433cf825.png",
"https://kemono.su/data/2a/b8/2ab8ba30644249e9516afaea05d61c0de14591cb9d232a2dc249650eb1a9a759.jpg",
"https://kemono.su/data/96/43/9643ac03888f3b199f4e769242477b8d4d4f96025b10ab3f28affc3a1ae6bf52.jpg",
"https://kemono.su/data/f7/a8/f7a87ccac5736f46190a53a2bb1ff3828230e90f480776759895fcba28375909.jpg",
"https://kemono.su/data/b0/38/b03882c8b0ab3b1cf9fc658a2bb2f9ac6ad4f3449015311dcd2d7ee7f748db31.png",
),
@@ -74,6 +77,7 @@ __tests__ = (
"#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd",
"added" : "2020-05-06T20:28:02.302000",
"archives" : [],
"content" : str,
"count" : 1,
"date" : "dt:2019-08-10 17:09:04",
@@ -229,7 +233,7 @@ __tests__ = (
"#class" : kemonoparty.KemonopartyPostExtractor,
"#urls" : "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg",
"file" : {
"file": {
"hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
"name": "wip update.jpg",
"path": "/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg",
@@ -248,7 +252,7 @@ __tests__ = (
"hash" : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
"revision_id" : 142470,
"revision_index": 2,
"revision_count": 10,
"revision_count": 11,
"revision_hash" : "e0e93281495e151b11636c156e52bfe9234c2a40",
},
@@ -258,16 +262,22 @@ __tests__ = (
"#category": ("", "kemonoparty", "patreon"),
"#class" : kemonoparty.KemonopartyPostExtractor,
"#options" : {"revisions": "unique"},
"#urls" : "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg",
"#urls" : "https://kemono.su/data/e3/e6/e3e6287dbc0468dd2a9d28ed276ae86788907143acf2ba10ab886a3add4c436c.jpg",
"#archive" : False,
"filename" : "wip update",
"hash" : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
"revision_id" : {9277608, 0},
"revision_index": {1, 2},
"revision_count": 2,
"revision_hash" : {"e0e93281495e151b11636c156e52bfe9234c2a40",
"79d5967719583a6fa52b2fc143e6a80fcdf75fb8"},
"hash" : {
"88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86",
"e3e6287dbc0468dd2a9d28ed276ae86788907143acf2ba10ab886a3add4c436c",
},
"revision_id" : {9277608, 10619155, 0},
"revision_index": {1, 2, 3},
"revision_count": 3,
"revision_hash" : {
"e0e93281495e151b11636c156e52bfe9234c2a40",
"bc5713195e14799da40c525381216c5a1a340b0f",
"9872bfb536a47cc69d95d2f195cd5c825808f089",
},
},
{
@@ -276,13 +286,16 @@ __tests__ = (
"#category": ("", "kemonoparty", "patreon"),
"#class" : kemonoparty.KemonopartyPostExtractor,
"#pattern" : r"https://kemono\.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86\.jpg",
"#count" : 10,
"#count" : 11,
"#archive" : False,
"revision_id": range(134996, 9277608),
"revision_index": range(1, 10),
"revision_count": 10,
"revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40",
"revision_id": range(134996, 10619155),
"revision_index": range(1, 11),
"revision_count": 11,
"revision_hash": {
"9872bfb536a47cc69d95d2f195cd5c825808f089",
"e0e93281495e151b11636c156e52bfe9234c2a40",
},
},
@@ -314,6 +327,86 @@ __tests__ = (
"type" : "attachment",
},
{
"#url" : "https://kemono.su/fanbox/user/49494721/post/9457614",
"#comment" : "archives",
"#category": ("", "kemonoparty", "fanbox"),
"#class" : kemonoparty.KemonopartyPostExtractor,
"#options" : {"archives": True},
"archives": [
{
"file": {
"added": "2025-03-03T02:11:28.153911",
"ctime": "2025-03-03T02:05:15.810201",
"ext" : ".zip",
"hash" : "c22c7e979355f633aaae4929b010816895a47ec37a9cfc25186a0952ec6e5774",
"id" : 190824068,
"ihash": None,
"mime" : "application/zip",
"mtime": "2025-03-03T02:11:28.807462",
"size" : 18634288,
},
"file_list": [
"モナmp4形式まとめ/",
"モナmp4形式まとめ/Movie_1.mp4",
"モナmp4形式まとめ/Movie_2.mp4",
"モナmp4形式まとめ/Movie_3.mp4",
"モナmp4形式まとめ/Movie_4.mp4",
"モナmp4形式まとめ/Movie_5.mp4",
"モナmp4形式まとめ/Movie_End_3.mp4",
],
"filename": "モナmp4形式まとめ",
"extension": "zip",
"hash": "c22c7e979355f633aaae4929b010816895a47ec37a9cfc25186a0952ec6e5774",
"name": "モナmp4形式まとめ.zip",
"password": None,
"path": "/c2/2c/c22c7e979355f633aaae4929b010816895a47ec37a9cfc25186a0952ec6e5774.zip",
"type": "attachment",
"url": "https://kemono.su/data/c2/2c/c22c7e979355f633aaae4929b010816895a47ec37a9cfc25186a0952ec6e5774.zip",
},
{
"file": {
"added": "2025-03-03T02:11:00.541142",
"ctime": "2025-03-03T02:04:56.754326",
"ext" : ".zip",
"hash" : "f7b4dedd9742aeb8da56dc6fe07deb7639880d0800ac0b7a6e91f64ff6b40178",
"id" : 190824029,
"ihash": None,
"mime" : "application/zip",
"mtime": "2025-03-03T02:11:01.110281",
"size" : 84738158,
},
"file_list": "len:229",
"filename": "モナUnity",
"extension": "zip",
"hash": "f7b4dedd9742aeb8da56dc6fe07deb7639880d0800ac0b7a6e91f64ff6b40178",
"name": "モナUnity.zip",
"password": None,
"path": "/f7/b4/f7b4dedd9742aeb8da56dc6fe07deb7639880d0800ac0b7a6e91f64ff6b40178.zip",
"type": "attachment",
"url": "https://kemono.su/data/f7/b4/f7b4dedd9742aeb8da56dc6fe07deb7639880d0800ac0b7a6e91f64ff6b40178.zip"
},
],
"title": "モナ（Live2Dアニメ）",
"user": "49494721",
"username": "soso",
"user_profile": {
"id": "49494721",
"indexed": "2021-04-02T23:50:57.138135",
"name": "soso",
"public_id": "soso",
"relation_id": None,
"service": "fanbox",
"updated": "2025-03-03T03:29:08.598883",
},
"tags": [
"うごイラ",
"原神",
],
},
{
"#url" : "https://kemono.su/discord/server/488668827274444803/608504710906904576",
"#category": ("", "kemonoparty", "discord"),