[civitai] always provide 'file[…]' metadata (#7548)
wrap most top-level values of (single) image results into a 'file' object
This commit is contained in:
Mike Fährmann
@@ -22,9 +22,9 @@ class CivitaiExtractor(Extractor):
|
||||
"""Base class for civitai extractors"""
|
||||
category = "civitai"
|
||||
root = "https://civitai.com"
|
||||
directory_fmt = ("{category}", "{username|user[username]}", "images")
|
||||
filename_fmt = "{file[id]|id|filename}.{extension}"
|
||||
archive_fmt = "{file[uuid]|uuid}"
|
||||
directory_fmt = ("{category}", "{user[username]}", "images")
|
||||
filename_fmt = "{file[id]}.{extension}"
|
||||
archive_fmt = "{file[uuid]}"
|
||||
request_interval = (0.5, 1.5)
|
||||
|
||||
def _init(self):
|
||||
@@ -107,25 +107,32 @@ class CivitaiExtractor(Extractor):
|
||||
|
||||
images = self.images()
|
||||
if images:
|
||||
for image in images:
|
||||
for file in images:
|
||||
|
||||
data = {
|
||||
"file": file,
|
||||
"user": file.pop("user"),
|
||||
}
|
||||
|
||||
if self._meta_generation:
|
||||
image["generation"] = \
|
||||
self._extract_meta_generation(image)
|
||||
data["generation"] = \
|
||||
self._extract_meta_generation(file)
|
||||
if self._meta_version:
|
||||
image["model"], image["version"] = \
|
||||
self._extract_meta_version(image, False)
|
||||
image["date"] = text.parse_datetime(
|
||||
image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
data["model"], data["version"] = \
|
||||
self._extract_meta_version(file, False)
|
||||
if "post" in file:
|
||||
data["post"] = file.pop("post")
|
||||
file["date"] = text.parse_datetime(
|
||||
file["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
url = self._url(image)
|
||||
text.nameext_from_url(url, image)
|
||||
if not image["extension"]:
|
||||
image["extension"] = (
|
||||
self._video_ext if image.get("type") == "video" else
|
||||
data["url"] = url = self._url(file)
|
||||
text.nameext_from_url(url, data)
|
||||
if not data["extension"]:
|
||||
data["extension"] = (
|
||||
self._video_ext if file.get("type") == "video" else
|
||||
self._image_ext)
|
||||
yield Message.Directory, image
|
||||
yield Message.Url, url, image
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, url, data
|
||||
return
|
||||
|
||||
def models(self):
|
||||
|
||||
@@ -84,19 +84,21 @@ __tests__ = (
|
||||
"#results" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/w/00014-3150861441.png",
|
||||
"#sha1_content": "a9a9d08f5fcdbc1e1eec7f203717f9df97b7a671",
|
||||
|
||||
"createdAt": "2024-08-31T01:11:47.021Z",
|
||||
"date" : "dt:2024-08-31 01:11:47",
|
||||
"extension": "png",
|
||||
"filename" : "00014-3150861441",
|
||||
"hash" : "ULN0-w?b4nRjxGM{-;t7M_t7NGae~qRjMyt7",
|
||||
"width" : 1152,
|
||||
"height" : 1536,
|
||||
"id" : 26962948,
|
||||
"nsfwLevel": 1,
|
||||
"postId" : 6030721,
|
||||
"stats" : dict,
|
||||
"url" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
|
||||
"uuid" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
|
||||
"file": {
|
||||
"createdAt": "2024-08-31T01:11:47.021Z",
|
||||
"date" : "dt:2024-08-31 01:11:47",
|
||||
"hash" : "ULN0-w?b4nRjxGM{-;t7M_t7NGae~qRjMyt7",
|
||||
"width" : 1152,
|
||||
"height" : 1536,
|
||||
"id" : 26962948,
|
||||
"nsfwLevel": 1,
|
||||
"postId" : 6030721,
|
||||
"stats" : dict,
|
||||
"url" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
|
||||
"uuid" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b",
|
||||
},
|
||||
"user" : {
|
||||
"username": "bolero537",
|
||||
},
|
||||
@@ -162,7 +164,7 @@ __tests__ = (
|
||||
"name": "v1.0 XL",
|
||||
"description": None,
|
||||
"baseModel": "SDXL 1.0",
|
||||
"baseModelType": None,
|
||||
"baseModelType": "Standard",
|
||||
"earlyAccessConfig": None,
|
||||
"earlyAccessEndsAt": None,
|
||||
"trainedWords": [
|
||||
@@ -195,27 +197,29 @@ __tests__ = (
|
||||
"#class" : civitai.CivitaiImageExtractor,
|
||||
"#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6a09ec54-6de4-4af1-b11d-2d0d8a66d651/quality=100/copy_C6C532CE-EC47-4A52-9138-AEF1D7756F16.Mp4",
|
||||
|
||||
"date" : "dt:2024-12-10 19:19:14",
|
||||
"extension": "mp4",
|
||||
"filename" : "copy_C6C532CE-EC47-4A52-9138-AEF1D7756F16",
|
||||
"hash" : "U9D8%cIU03Rk02?F$$WE0gs,?GSg~B9ut6sl",
|
||||
"width" : 1080,
|
||||
"height" : 1920,
|
||||
"id" : 44789630,
|
||||
"mimeType" : "video/mp4",
|
||||
"nsfwLevel": 2,
|
||||
"postId" : 10151863,
|
||||
"stats" : dict,
|
||||
"type" : "video",
|
||||
"url" : "6a09ec54-6de4-4af1-b11d-2d0d8a66d651",
|
||||
"uuid" : "6a09ec54-6de4-4af1-b11d-2d0d8a66d651",
|
||||
"metadata" : {
|
||||
"audio" : True,
|
||||
"duration": 15.033,
|
||||
"hash" : "U9D8%cIU03Rk02?F$$WE0gs,?GSg~B9ut6sl",
|
||||
"height" : 1920,
|
||||
"size" : 23984479,
|
||||
"width" : 1080,
|
||||
"file" : {
|
||||
"date" : "dt:2024-12-10 19:19:14",
|
||||
"hash" : "U9D8%cIU03Rk02?F$$WE0gs,?GSg~B9ut6sl",
|
||||
"width" : 1080,
|
||||
"height" : 1920,
|
||||
"id" : 44789630,
|
||||
"mimeType" : "video/mp4",
|
||||
"nsfwLevel": 2,
|
||||
"postId" : 10151863,
|
||||
"stats" : dict,
|
||||
"type" : "video",
|
||||
"url" : "6a09ec54-6de4-4af1-b11d-2d0d8a66d651",
|
||||
"uuid" : "6a09ec54-6de4-4af1-b11d-2d0d8a66d651",
|
||||
"metadata" : {
|
||||
"audio" : True,
|
||||
"duration": 15.033,
|
||||
"hash" : "U9D8%cIU03Rk02?F$$WE0gs,?GSg~B9ut6sl",
|
||||
"height" : 1920,
|
||||
"size" : 23984479,
|
||||
"width" : 1080,
|
||||
},
|
||||
},
|
||||
"user" : {
|
||||
"username": "jboogx_creative",
|
||||
@@ -228,28 +232,30 @@ __tests__ = (
|
||||
"#class": civitai.CivitaiImageExtractor,
|
||||
"#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c7e3744b-8f0d-4124-94c1-75e2af00431d/quality=100/2025-04-25-23h40m21s_seed665048144_A man appears from off screen and spanks her butto_2.webm",
|
||||
|
||||
"date" : "dt:2025-05-05 12:27:28",
|
||||
"extension": "webm",
|
||||
"filename" : "2025-04-25-23h40m21s_seed665048144_A man appears from off screen and spanks her butto_2",
|
||||
"hash" : "UMCsEoRPivxY~VjuWBoenMWBx]WrxvV?xvbb",
|
||||
"width" : 512,
|
||||
"height" : 752,
|
||||
"id" : 74353746,
|
||||
"mimeType" : "video/webm",
|
||||
"nsfwLevel": 4,
|
||||
"postId" : 16509805,
|
||||
"stats" : dict,
|
||||
"type" : "video",
|
||||
"url" : "c7e3744b-8f0d-4124-94c1-75e2af00431d",
|
||||
"uuid" : "c7e3744b-8f0d-4124-94c1-75e2af00431d",
|
||||
"metadata" : {
|
||||
"audio" : False,
|
||||
"duration": 5.016,
|
||||
"hash" : "UMCsEoRPivxY~VjuWBoenMWBx]WrxvV?xvbb",
|
||||
"height" : 752,
|
||||
"size" : 6011344,
|
||||
"skipScannedAtReassignment": True,
|
||||
"width" : 512,
|
||||
"file" : {
|
||||
"date" : "dt:2025-05-05 12:27:28",
|
||||
"hash" : "UMCsEoRPivxY~VjuWBoenMWBx]WrxvV?xvbb",
|
||||
"width" : 512,
|
||||
"height" : 752,
|
||||
"id" : 74353746,
|
||||
"mimeType" : "video/webm",
|
||||
"nsfwLevel": 4,
|
||||
"postId" : 16509805,
|
||||
"stats" : dict,
|
||||
"type" : "video",
|
||||
"url" : "c7e3744b-8f0d-4124-94c1-75e2af00431d",
|
||||
"uuid" : "c7e3744b-8f0d-4124-94c1-75e2af00431d",
|
||||
"metadata" : {
|
||||
"audio" : False,
|
||||
"duration": 5.016,
|
||||
"hash" : "UMCsEoRPivxY~VjuWBoenMWBx]WrxvV?xvbb",
|
||||
"height" : 752,
|
||||
"size" : 6011344,
|
||||
"skipScannedAtReassignment": True,
|
||||
"width" : 512,
|
||||
},
|
||||
},
|
||||
"user" : {
|
||||
"id" : 4856161,
|
||||
@@ -274,7 +280,9 @@ __tests__ = (
|
||||
"#class" : civitai.CivitaiImageExtractor,
|
||||
"#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2d1fbe1b-6038-479f-8c37-39d338198fb1/quality=100/received_687641707052140.mp4",
|
||||
|
||||
"nsfwLevel": 4,
|
||||
"file": {
|
||||
"nsfwLevel": 4,
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
@@ -283,7 +291,9 @@ __tests__ = (
|
||||
"#class" : civitai.CivitaiImageExtractor,
|
||||
"#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/1107208c-14cc-46fd-848d-2efa14fa6180/original=true/QRQC7HE5DFW3QZ85R3MXQXY440.jpeg",
|
||||
|
||||
"nsfwLevel": 8,
|
||||
"file": {
|
||||
"nsfwLevel": 8,
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
@@ -292,7 +302,9 @@ __tests__ = (
|
||||
"#class" : civitai.CivitaiImageExtractor,
|
||||
"#results": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/fdbaa27d-4278-496b-8209-21591e5dc6fe/original=true/Q8AE16QCMCYCCBX49PG8VVWWD0.jpeg",
|
||||
|
||||
"nsfwLevel": 16,
|
||||
"file": {
|
||||
"nsfwLevel": 16,
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user