From ad84e72fda41c0a4ba448b6e95ca4a5c4f69412a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 7 Jul 2025 11:48:21 +0200 Subject: [PATCH] [civitai] implement retrieving 'post' metadata (#7548) --- docs/configuration.rst | 8 ++++---- gallery_dl/extractor/civitai.py | 15 +++++++++++++-- test/results/civitai.py | 15 ++++++++++++++- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index e19423c8..cb1b5fc9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1875,12 +1875,12 @@ Type Default ``false`` Example - * ``"generation,version"`` - * ``["generation", "version"]`` + * ``"generation,post,version"`` + * ``["version", "generation"]`` Description - Extract additional ``generation`` and ``version`` metadata. + Extract additional ``generation``, ``version``, and ``post`` metadata. - Note: This requires 1 additional HTTP request per image or video. + Note: This requires 1 or more additional API requests per image or video. extractor.civitai.nsfw diff --git a/gallery_dl/extractor/civitai.py b/gallery_dl/extractor/civitai.py index c191c5ab..1df91452 100644 --- a/gallery_dl/extractor/civitai.py +++ b/gallery_dl/extractor/civitai.py @@ -64,11 +64,13 @@ class CivitaiExtractor(Extractor): if isinstance(metadata, str): metadata = metadata.split(",") elif not isinstance(metadata, (list, tuple)): - metadata = ("generation", "version") + metadata = ("generation", "version", "post") self._meta_generation = ("generation" in metadata) self._meta_version = ("version" in metadata) + self._meta_post = ("post" in metadata) else: - self._meta_generation = self._meta_version = False + self._meta_generation = self._meta_version = self._meta_post = \ + False def items(self): models = self.models() @@ -122,6 +124,9 @@ class CivitaiExtractor(Extractor): self._extract_meta_version(file, False) if "post" in file: data["post"] = file.pop("post") + if self._meta_post and "post" not in data: + data["post"] = post = self._extract_meta_post(file) + post.pop("user", None) file["date"] = text.parse_datetime( file["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") @@ -204,6 +209,12 @@ class CivitaiExtractor(Extractor): except Exception as exc: return self.log.debug("", exc_info=exc) + def _extract_meta_post(self, image): + try: + return self.api.post(image["postId"]) + except Exception as exc: + return self.log.debug("", exc_info=exc) + def _extract_meta_version(self, item, is_post=True): try: version_id = self._extract_version_id(item, is_post) diff --git a/test/results/civitai.py b/test/results/civitai.py index acee2ac3..fd73305a 100644 --- a/test/results/civitai.py +++ b/test/results/civitai.py @@ -193,8 +193,9 @@ __tests__ = ( { "#url" : "https://civitai.com/images/44789630", - "#comment": "video", + "#comment": "video - 'post' metadata (#7548)", "#class" : civitai.CivitaiImageExtractor, + "#options": {"metadata": "post"}, "#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6a09ec54-6de4-4af1-b11d-2d0d8a66d651/quality=100/copy_C6C532CE-EC47-4A52-9138-AEF1D7756F16.Mp4", "extension": "mp4", @@ -221,6 +222,18 @@ __tests__ = ( "width" : 1080, }, }, + "post": { + "availability": "Public", + "collectionId": None, + "detail" : None, + "id" : 10151863, + "modelVersion": None, + "modelVersionId": None, + "nsfwLevel" : 2, + "publishedAt" : "2024-12-10T19:20:51.579Z", + "tags" : [], + "title" : None, + }, "user" : { "username": "jboogx_creative", },