[civitai] fix exception for images without modelVersionId (#7432)

gallery_dl/extractor/civitai.py

@@ -96,9 +96,8 @@ class CivitaiExtractor(Extractor):
                     "user": post.pop("user"),
                 }
                 if self._meta_version:
-                    data["version"] = version = self.api.model_version(
+                    data["model"], data["version"] = \
-                        post["modelVersionId"]).copy()
+                        self._extract_meta_version(post)
-                    data["model"] = version.pop("model")
 
                 yield Message.Directory, data
                 for file in self._image_results(images):
@@ -109,24 +108,17 @@ class CivitaiExtractor(Extractor):
         images = self.images()
         if images:
             for image in images:
-                url = self._url(image)
-                if self._meta_generation:
-                    image["generation"] = self.api.image_generationdata(
-                        image["id"])
-                if self._meta_version:
-                    if "modelVersionId" in image:
-                        version_id = image["modelVersionId"]
-                    else:
-                        post = image["post"] = self.api.post(
-                            image["postId"])
-                        post.pop("user", None)
-                        version_id = post["modelVersionId"]
-                    image["version"] = version = self.api.model_version(
-                        version_id).copy()
-                    image["model"] = version.pop("model")
 
+                if self._meta_generation:
+                    image["generation"] = \
+                        self._extract_meta_generation(image)
+                if self._meta_version:
+                    image["model"], image["version"] = \
+                        self._extract_meta_version(image, False)
                 image["date"] = text.parse_datetime(
                     image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
+
+                url = self._url(image)
                 text.nameext_from_url(url, image)
                 if not image["extension"]:
                     image["extension"] = (
@@ -155,8 +147,8 @@ class CivitaiExtractor(Extractor):
             image["uuid"] = parts[1]
             parts[2] = quality
             return "/".join(parts)
-        image["uuid"] = url
 
+        image["uuid"] = url
         name = image.get("name")
         if not name:
             mime = image.get("mimeType") or self._image_ext
@@ -180,7 +172,7 @@ class CivitaiExtractor(Extractor):
             if "id" not in file and data["filename"].isdecimal():
                 file["id"] = text.parse_int(data["filename"])
             if self._meta_generation:
-                file["generation"] = self.api.image_generationdata(file["id"])
+                file["generation"] = self._extract_meta_generation(file)
             yield data
 
     def _parse_query(self, value):
@@ -188,6 +180,32 @@ class CivitaiExtractor(Extractor):
             value, {"tags", "reactions", "baseModels", "tools", "techniques",
                     "types", "fileFormats"})
 
+    def _extract_meta_generation(self, image):
+        return self.api.image_generationdata(image["id"])
+
+    def _extract_meta_version(self, item, is_post=True):
+        version_id = self._extract_version_id(item, is_post)
+        if version_id is None:
+            return None, None
+        version = self.api.model_version(version_id).copy()
+        return version.pop("model", None), version
+
+    def _extract_version_id(self, item, is_post=True):
+        version_id = item.get("modelVersionId")
+        if version_id:
+            return version_id
+
+        version_ids = item.get("modelVersionIds")
+        if version_ids:
+            return version_ids[0]
+
+        if is_post:
+            return None
+
+        item["post"] = post = self.api.post(item["postId"])
+        post.pop("user", None)
+        return self._extract_version_id(post)
+
 
 class CivitaiModelExtractor(CivitaiExtractor):
     subcategory = "model"

test/results/civitai.py

@@ -133,13 +133,67 @@ __tests__ = (
             },
         },
     },
+    "post": {
+        "id": 6030721,
+        "nsfwLevel": 1,
+        "title": "メイド　クラシック/maid classic - v1.0 XL Showcase",
+        "detail": None,
+        "modelVersionId": 788385,
+        "modelVersion": {
+            "id": 788385,
+        },
+        "publishedAt": "2024-08-31T01:11:52.175Z",
+        "availability": "Public",
+        "tags": [],
+        "collectionId": None,
+    },
+    "model": {
+        "id": 703211,
+        "name": "メイド　クラシック/maid classic",
+        "type": "LORA",
+        "status": "Published",
+        "publishedAt": "2024-08-30T15:38:14.770Z",
+        "nsfw": False,
+        "uploadType": "Created",
+        "availability": "Public",
+    },
+    "version": {
+        "id": 788385,
+        "name": "v1.0 XL",
+        "description": None,
+        "baseModel": "SDXL 1.0",
+        "baseModelType": None,
+        "earlyAccessConfig": None,
+        "earlyAccessEndsAt": None,
+        "trainedWords": [
+            "maid, maid apron, maid headdress, long sleeves",
+        ],
+        "epochs": None,
+        "steps": None,
+        "clipSkip": None,
+        "status": "Published",
+        "createdAt": "2024-08-31T01:11:08.841Z",
+        "vaeId": None,
+        "trainingDetails": None,
+        "trainingStatus": None,
+        "uploadType": "Created",
+        "usageControl": "Download",
+        "requireAuth": True,
+        "settings": {
+            "strength": 0.8,
+        },
+        "recommendedResources": [],
+        "monetization": None,
+        "canGenerate": True,
+        "files": None,
+    },
 },
 
 {
     "#url"    : "https://civitai.com/images/44789630",
     "#comment": "video",
     "#class"  : civitai.CivitaiImageExtractor,
-    "#urls"   : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6a09ec54-6de4-4af1-b11d-2d0d8a66d651/original=true,transcode=true,quality=100/copy_C6C532CE-EC47-4A52-9138-AEF1D7756F16.Mp4",
+    "#urls"   : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6a09ec54-6de4-4af1-b11d-2d0d8a66d651/quality=100/copy_C6C532CE-EC47-4A52-9138-AEF1D7756F16.Mp4",
 
     "date"     : "dt:2024-12-10 19:19:14",
     "extension": "mp4",
@@ -172,7 +226,7 @@ __tests__ = (
     "#url"  : "https://civitai.com/images/74353746",
     "#comment": "video, rated 'R', WebP download (#7502)",
     "#class": civitai.CivitaiImageExtractor,
-    "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c7e3744b-8f0d-4124-94c1-75e2af00431d/original=true,transcode=true,quality=100/2025-04-25-23h40m21s_seed665048144_A man appears from off screen and spanks her butto_2.webm",
+    "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c7e3744b-8f0d-4124-94c1-75e2af00431d/quality=100/2025-04-25-23h40m21s_seed665048144_A man appears from off screen and spanks her butto_2.webm",
 
     "date"     : "dt:2025-05-05 12:27:28",
     "extension": "webm",
@@ -203,6 +257,44 @@ __tests__ = (
     },
 },
 
+{
+    "#url"    : "https://civitai.com/images/76635747",
+    "#comment": "no 'modelVersionId' (#7432)",
+    "#class"  : civitai.CivitaiImageExtractor,
+    "#options": {"metadata": "version"},
+    "#urls"   : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/65c1a01c-2583-4495-b4e9-bdb94218004e/original=true/5b5b95f8-9923-4c27-b50a-c801c0311375-0.jpg",
+
+    "model"  : None,
+    "version": None,
+},
+
+{
+    "#url"    : "https://civitai.com/images/68947296",
+    "#comment": "rated R / nsfwlevel 4",
+    "#class"  : civitai.CivitaiImageExtractor,
+    "#urls"   : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2d1fbe1b-6038-479f-8c37-39d338198fb1/quality=100/received_687641707052140.mp4",
+
+    "nsfwLevel": 4,
+},
+
+{
+    "#url"    : "https://civitai.com/images/68852050",
+    "#comment": "rated X / nsfwlevel 8",
+    "#class"  : civitai.CivitaiImageExtractor,
+    "#urls"   : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/1107208c-14cc-46fd-848d-2efa14fa6180/original=true/QRQC7HE5DFW3QZ85R3MXQXY440.jpeg",
+
+    "nsfwLevel": 8,
+},
+
+{
+    "#url"    : "https://civitai.com/images/68851932",
+    "#comment": "rated XXX / nsfwlevel 16",
+    "#class"  : civitai.CivitaiImageExtractor,
+    "#urls"   : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/fdbaa27d-4278-496b-8209-21591e5dc6fe/original=true/Q8AE16QCMCYCCBX49PG8VVWWD0.jpeg",
+
+    "nsfwLevel": 16,
+},
+
 {
     "#url"    : "https://civitai.com/posts/6877551",
     "#class"  : civitai.CivitaiPostExtractor,
@@ -234,6 +326,16 @@ __tests__ = (
     },
 },
 
+{
+    "#url"    : "https://civitai.com/posts/17021768",
+    "#comment": "no 'modelVersionId' (#7432)",
+    "#class"  : civitai.CivitaiPostExtractor,
+    "#options": {"metadata": "version"},
+
+    "model"  : None,
+    "version": None,
+},
+
 {
     "#url"  : "https://civitai.com/tag/mecha",
     "#class": civitai.CivitaiTagExtractor,