[civitai] always provide 'file[…]' metadata (#7548)

wrap most top-level values of (single) image results into a 'file' object

gallery_dl/extractor/civitai.py

@@ -22,9 +22,9 @@ class CivitaiExtractor(Extractor):
     """Base class for civitai extractors"""
     category = "civitai"
     root = "https://civitai.com"
-    directory_fmt = ("{category}", "{username|user[username]}", "images")
+    directory_fmt = ("{category}", "{user[username]}", "images")
-    filename_fmt = "{file[id]|id|filename}.{extension}"
+    filename_fmt = "{file[id]}.{extension}"
-    archive_fmt = "{file[uuid]|uuid}"
+    archive_fmt = "{file[uuid]}"
     request_interval = (0.5, 1.5)
 
     def _init(self):
@@ -107,25 +107,32 @@ class CivitaiExtractor(Extractor):
 
         images = self.images()
         if images:
-            for image in images:
+            for file in images:
+
+                data = {
+                    "file": file,
+                    "user": file.pop("user"),
+                }
 
                 if self._meta_generation:
-                    image["generation"] = \
+                    data["generation"] = \
-                        self._extract_meta_generation(image)
+                        self._extract_meta_generation(file)
                 if self._meta_version:
-                    image["model"], image["version"] = \
+                    data["model"], data["version"] = \
-                        self._extract_meta_version(image, False)
+                        self._extract_meta_version(file, False)
-                image["date"] = text.parse_datetime(
+                    if "post" in file:
-                    image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
+                        data["post"] = file.pop("post")
+                file["date"] = text.parse_datetime(
+                    file["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
 
-                url = self._url(image)
+                data["url"] = url = self._url(file)
-                text.nameext_from_url(url, image)
+                text.nameext_from_url(url, data)
-                if not image["extension"]:
+                if not data["extension"]:
-                    image["extension"] = (
+                    data["extension"] = (
-                        self._video_ext if image.get("type") == "video" else
+                        self._video_ext if file.get("type") == "video" else
                         self._image_ext)
-                yield Message.Directory, image
+                yield Message.Directory, data
-                yield Message.Url, url, image
+                yield Message.Url, url, data
             return
 
     def models(self):

test/results/civitai.py

@@ -84,19 +84,21 @@ __tests__ = (
     "#results"     : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/w/00014-3150861441.png",
     "#sha1_content": "a9a9d08f5fcdbc1e1eec7f203717f9df97b7a671",
 
-    "createdAt": "2024-08-31T01:11:47.021Z",
-    "date"     : "dt:2024-08-31 01:11:47",
     "extension": "png",
     "filename" : "00014-3150861441",
-    "hash"     : "ULN0-w?b4nRjxGM{-;t7M_t7NGae~qRjMyt7",
+    "file": {
-    "width"    : 1152,
+        "createdAt": "2024-08-31T01:11:47.021Z",
-    "height"   : 1536,
+        "date"     : "dt:2024-08-31 01:11:47",
-    "id"       : 26962948,
+        "hash"     : "ULN0-w?b4nRjxGM{-;t7M_t7NGae~qRjMyt7",
-    "nsfwLevel": 1,
+        "width"    : 1152,
-    "postId"   : 6030721,
+        "height"   : 1536,
-    "stats"    : dict,
+        "id"       : 26962948,
-    "url"      : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
+        "nsfwLevel": 1,
-    "uuid"     : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
+        "postId"   : 6030721,
+        "stats"    : dict,
+        "url"      : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
+        "uuid"     : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
+    },
     "user"     : {
         "username": "bolero537",
     },
@@ -162,7 +164,7 @@ __tests__ = (
         "name": "v1.0 XL",
         "description": None,
         "baseModel": "SDXL 1.0",
-        "baseModelType": None,
+        "baseModelType": "Standard",
         "earlyAccessConfig": None,
         "earlyAccessEndsAt": None,
         "trainedWords": [
@@ -195,27 +197,29 @@ __tests__ = (
     "#class"  : civitai.CivitaiImageExtractor,
     "#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6a09ec54-6de4-4af1-b11d-2d0d8a66d651/quality=100/copy_C6C532CE-EC47-4A52-9138-AEF1D7756F16.Mp4",
 
-    "date"     : "dt:2024-12-10 19:19:14",
     "extension": "mp4",
     "filename" : "copy_C6C532CE-EC47-4A52-9138-AEF1D7756F16",
-    "hash"     : "U9D8%cIU03Rk02?F$$WE0gs,?GSg~B9ut6sl",
+    "file"     : {
-    "width"    : 1080,
+        "date"     : "dt:2024-12-10 19:19:14",
-    "height"   : 1920,
+        "hash"     : "U9D8%cIU03Rk02?F$$WE0gs,?GSg~B9ut6sl",
-    "id"       : 44789630,
+        "width"    : 1080,
-    "mimeType" : "video/mp4",
+        "height"   : 1920,
-    "nsfwLevel": 2,
+        "id"       : 44789630,
-    "postId"   : 10151863,
+        "mimeType" : "video/mp4",
-    "stats"    : dict,
+        "nsfwLevel": 2,
-    "type"     : "video",
+        "postId"   : 10151863,
-    "url"      : "6a09ec54-6de4-4af1-b11d-2d0d8a66d651",
+        "stats"    : dict,
-    "uuid"     : "6a09ec54-6de4-4af1-b11d-2d0d8a66d651",
+        "type"     : "video",
-    "metadata" : {
+        "url"      : "6a09ec54-6de4-4af1-b11d-2d0d8a66d651",
-        "audio"   : True,
+        "uuid"     : "6a09ec54-6de4-4af1-b11d-2d0d8a66d651",
-        "duration": 15.033,
+        "metadata" : {
-        "hash"    : "U9D8%cIU03Rk02?F$$WE0gs,?GSg~B9ut6sl",
+            "audio"   : True,
-        "height"  : 1920,
+            "duration": 15.033,
-        "size"    : 23984479,
+            "hash"    : "U9D8%cIU03Rk02?F$$WE0gs,?GSg~B9ut6sl",
-        "width"   : 1080,
+            "height"  : 1920,
+            "size"    : 23984479,
+            "width"   : 1080,
+        },
     },
     "user"     : {
         "username": "jboogx_creative",
@@ -228,28 +232,30 @@ __tests__ = (
     "#class": civitai.CivitaiImageExtractor,
     "#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c7e3744b-8f0d-4124-94c1-75e2af00431d/quality=100/2025-04-25-23h40m21s_seed665048144_A man appears from off screen and spanks her butto_2.webm",
 
-    "date"     : "dt:2025-05-05 12:27:28",
     "extension": "webm",
     "filename" : "2025-04-25-23h40m21s_seed665048144_A man appears from off screen and spanks her butto_2",
-    "hash"     : "UMCsEoRPivxY~VjuWBoenMWBx]WrxvV?xvbb",
+    "file"     : {
-    "width"    : 512,
+        "date"     : "dt:2025-05-05 12:27:28",
-    "height"   : 752,
+        "hash"     : "UMCsEoRPivxY~VjuWBoenMWBx]WrxvV?xvbb",
-    "id"       : 74353746,
+        "width"    : 512,
-    "mimeType" : "video/webm",
+        "height"   : 752,
-    "nsfwLevel": 4,
+        "id"       : 74353746,
-    "postId"   : 16509805,
+        "mimeType" : "video/webm",
-    "stats"    : dict,
+        "nsfwLevel": 4,
-    "type"     : "video",
+        "postId"   : 16509805,
-    "url"      : "c7e3744b-8f0d-4124-94c1-75e2af00431d",
+        "stats"    : dict,
-    "uuid"     : "c7e3744b-8f0d-4124-94c1-75e2af00431d",
+        "type"     : "video",
-    "metadata" : {
+        "url"      : "c7e3744b-8f0d-4124-94c1-75e2af00431d",
-        "audio"   : False,
+        "uuid"     : "c7e3744b-8f0d-4124-94c1-75e2af00431d",
-        "duration": 5.016,
+        "metadata" : {
-        "hash"    : "UMCsEoRPivxY~VjuWBoenMWBx]WrxvV?xvbb",
+            "audio"   : False,
-        "height"  : 752,
+            "duration": 5.016,
-        "size"    : 6011344,
+            "hash"    : "UMCsEoRPivxY~VjuWBoenMWBx]WrxvV?xvbb",
-        "skipScannedAtReassignment": True,
+            "height"  : 752,
-        "width"   : 512,
+            "size"    : 6011344,
+            "skipScannedAtReassignment": True,
+            "width"   : 512,
+        },
     },
     "user"     : {
         "id"      : 4856161,
@@ -274,7 +280,9 @@ __tests__ = (
     "#class"  : civitai.CivitaiImageExtractor,
     "#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2d1fbe1b-6038-479f-8c37-39d338198fb1/quality=100/received_687641707052140.mp4",
 
-    "nsfwLevel": 4,
+    "file": {
+        "nsfwLevel": 4,
+    },
 },
 
 {
@@ -283,7 +291,9 @@ __tests__ = (
     "#class"  : civitai.CivitaiImageExtractor,
     "#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/1107208c-14cc-46fd-848d-2efa14fa6180/original=true/QRQC7HE5DFW3QZ85R3MXQXY440.jpeg",
 
-    "nsfwLevel": 8,
+    "file": {
+        "nsfwLevel": 8,
+    },
 },
 
 {
@@ -292,7 +302,9 @@ __tests__ = (
     "#class"  : civitai.CivitaiImageExtractor,
     "#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/fdbaa27d-4278-496b-8209-21591e5dc6fe/original=true/Q8AE16QCMCYCCBX49PG8VVWWD0.jpeg",
 
-    "nsfwLevel": 16,
+    "file": {
+        "nsfwLevel": 16,
+    },
 },
 
 {