diff --git a/gallery_dl/extractor/civitai.py b/gallery_dl/extractor/civitai.py index 91b1a267..6843b732 100644 --- a/gallery_dl/extractor/civitai.py +++ b/gallery_dl/extractor/civitai.py @@ -96,9 +96,8 @@ class CivitaiExtractor(Extractor): "user": post.pop("user"), } if self._meta_version: - data["version"] = version = self.api.model_version( - post["modelVersionId"]).copy() - data["model"] = version.pop("model") + data["model"], data["version"] = \ + self._extract_meta_version(post) yield Message.Directory, data for file in self._image_results(images): @@ -109,24 +108,17 @@ class CivitaiExtractor(Extractor): images = self.images() if images: for image in images: - url = self._url(image) - if self._meta_generation: - image["generation"] = self.api.image_generationdata( - image["id"]) - if self._meta_version: - if "modelVersionId" in image: - version_id = image["modelVersionId"] - else: - post = image["post"] = self.api.post( - image["postId"]) - post.pop("user", None) - version_id = post["modelVersionId"] - image["version"] = version = self.api.model_version( - version_id).copy() - image["model"] = version.pop("model") + if self._meta_generation: + image["generation"] = \ + self._extract_meta_generation(image) + if self._meta_version: + image["model"], image["version"] = \ + self._extract_meta_version(image, False) image["date"] = text.parse_datetime( image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + + url = self._url(image) text.nameext_from_url(url, image) if not image["extension"]: image["extension"] = ( @@ -155,8 +147,8 @@ class CivitaiExtractor(Extractor): image["uuid"] = parts[1] parts[2] = quality return "/".join(parts) - image["uuid"] = url + image["uuid"] = url name = image.get("name") if not name: mime = image.get("mimeType") or self._image_ext @@ -180,7 +172,7 @@ class CivitaiExtractor(Extractor): if "id" not in file and data["filename"].isdecimal(): file["id"] = text.parse_int(data["filename"]) if self._meta_generation: - file["generation"] = self.api.image_generationdata(file["id"]) + file["generation"] = self._extract_meta_generation(file) yield data def _parse_query(self, value): @@ -188,6 +180,32 @@ class CivitaiExtractor(Extractor): value, {"tags", "reactions", "baseModels", "tools", "techniques", "types", "fileFormats"}) + def _extract_meta_generation(self, image): + return self.api.image_generationdata(image["id"]) + + def _extract_meta_version(self, item, is_post=True): + version_id = self._extract_version_id(item, is_post) + if version_id is None: + return None, None + version = self.api.model_version(version_id).copy() + return version.pop("model", None), version + + def _extract_version_id(self, item, is_post=True): + version_id = item.get("modelVersionId") + if version_id: + return version_id + + version_ids = item.get("modelVersionIds") + if version_ids: + return version_ids[0] + + if is_post: + return None + + item["post"] = post = self.api.post(item["postId"]) + post.pop("user", None) + return self._extract_version_id(post) + class CivitaiModelExtractor(CivitaiExtractor): subcategory = "model" diff --git a/test/results/civitai.py b/test/results/civitai.py index 7d84da26..5e07ceb1 100644 --- a/test/results/civitai.py +++ b/test/results/civitai.py @@ -133,13 +133,67 @@ __tests__ = ( }, }, }, + "post": { + "id": 6030721, + "nsfwLevel": 1, + "title": "メイド クラシック/maid classic - v1.0 XL Showcase", + "detail": None, + "modelVersionId": 788385, + "modelVersion": { + "id": 788385, + }, + "publishedAt": "2024-08-31T01:11:52.175Z", + "availability": "Public", + "tags": [], + "collectionId": None, + }, + "model": { + "id": 703211, + "name": "メイド クラシック/maid classic", + "type": "LORA", + "status": "Published", + "publishedAt": "2024-08-30T15:38:14.770Z", + "nsfw": False, + "uploadType": "Created", + "availability": "Public", + }, + "version": { + "id": 788385, + "name": "v1.0 XL", + "description": None, + "baseModel": "SDXL 1.0", + "baseModelType": None, + "earlyAccessConfig": None, + "earlyAccessEndsAt": None, + "trainedWords": [ + "maid, maid apron, maid headdress, long sleeves", + ], + "epochs": None, + "steps": None, + "clipSkip": None, + "status": "Published", + "createdAt": "2024-08-31T01:11:08.841Z", + "vaeId": None, + "trainingDetails": None, + "trainingStatus": None, + "uploadType": "Created", + "usageControl": "Download", + "requireAuth": True, + "settings": { + "strength": 0.8, + }, + "recommendedResources": [], + "monetization": None, + "canGenerate": True, + "files": None, + }, }, { "#url" : "https://civitai.com/images/44789630", "#comment": "video", "#class" : civitai.CivitaiImageExtractor, - "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6a09ec54-6de4-4af1-b11d-2d0d8a66d651/original=true,transcode=true,quality=100/copy_C6C532CE-EC47-4A52-9138-AEF1D7756F16.Mp4", + "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6a09ec54-6de4-4af1-b11d-2d0d8a66d651/quality=100/copy_C6C532CE-EC47-4A52-9138-AEF1D7756F16.Mp4", "date" : "dt:2024-12-10 19:19:14", "extension": "mp4", @@ -172,7 +226,7 @@ __tests__ = ( "#url" : "https://civitai.com/images/74353746", "#comment": "video, rated 'R', WebP download (#7502)", "#class": civitai.CivitaiImageExtractor, - "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c7e3744b-8f0d-4124-94c1-75e2af00431d/original=true,transcode=true,quality=100/2025-04-25-23h40m21s_seed665048144_A man appears from off screen and spanks her butto_2.webm", + "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c7e3744b-8f0d-4124-94c1-75e2af00431d/quality=100/2025-04-25-23h40m21s_seed665048144_A man appears from off screen and spanks her butto_2.webm", "date" : "dt:2025-05-05 12:27:28", "extension": "webm", @@ -203,6 +257,44 @@ __tests__ = ( }, }, +{ + "#url" : "https://civitai.com/images/76635747", + "#comment": "no 'modelVersionId' (#7432)", + "#class" : civitai.CivitaiImageExtractor, + "#options": {"metadata": "version"}, + "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/65c1a01c-2583-4495-b4e9-bdb94218004e/original=true/5b5b95f8-9923-4c27-b50a-c801c0311375-0.jpg", + + "model" : None, + "version": None, +}, + +{ + "#url" : "https://civitai.com/images/68947296", + "#comment": "rated R / nsfwlevel 4", + "#class" : civitai.CivitaiImageExtractor, + "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2d1fbe1b-6038-479f-8c37-39d338198fb1/quality=100/received_687641707052140.mp4", + + "nsfwLevel": 4, +}, + +{ + "#url" : "https://civitai.com/images/68852050", + "#comment": "rated X / nsfwlevel 8", + "#class" : civitai.CivitaiImageExtractor, + "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/1107208c-14cc-46fd-848d-2efa14fa6180/original=true/QRQC7HE5DFW3QZ85R3MXQXY440.jpeg", + + "nsfwLevel": 8, +}, + +{ + "#url" : "https://civitai.com/images/68851932", + "#comment": "rated XXX / nsfwlevel 16", + "#class" : civitai.CivitaiImageExtractor, + "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/fdbaa27d-4278-496b-8209-21591e5dc6fe/original=true/Q8AE16QCMCYCCBX49PG8VVWWD0.jpeg", + + "nsfwLevel": 16, +}, + { "#url" : "https://civitai.com/posts/6877551", "#class" : civitai.CivitaiPostExtractor, @@ -234,6 +326,16 @@ __tests__ = ( }, }, +{ + "#url" : "https://civitai.com/posts/17021768", + "#comment": "no 'modelVersionId' (#7432)", + "#class" : civitai.CivitaiPostExtractor, + "#options": {"metadata": "version"}, + + "model" : None, + "version": None, +}, + { "#url" : "https://civitai.com/tag/mecha", "#class": civitai.CivitaiTagExtractor,