[civitai] add 'collection' extractor (#8005)
This commit is contained in:
Mike Fährmann
@@ -190,7 +190,7 @@ Consider all listed sites to potentially be NSFW.
|
||||
<tr>
|
||||
<td>Civitai</td>
|
||||
<td>https://www.civitai.com/</td>
|
||||
<td>Generated Files, individual Images, Image Listings, Models, Model Listings, Posts, Post Listings, Image Searches, Model Searches, Tag Searches, User Profiles, User Images, Image Reactions, User Models, User Posts, User Videos, Video Reactions</td>
|
||||
<td>Collections, Generated Files, individual Images, Image Listings, Models, Model Listings, Posts, Post Listings, Image Searches, Model Searches, Tag Searches, User Profiles, User Images, Image Reactions, User Models, User Posts, User Videos, Video Reactions</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
||||
@@ -377,6 +377,28 @@ class CivitaiImageExtractor(CivitaiExtractor):
|
||||
return self.api.image(self.groups[0])
|
||||
|
||||
|
||||
class CivitaiCollectionExtractor(CivitaiExtractor):
|
||||
subcategory = "collection"
|
||||
directory_fmt = ("{category}", "{user_collection[username]}",
|
||||
"collections", "{collection[id]}{collection[name]:? //}")
|
||||
pattern = BASE_PATTERN + r"/collections/(\d+)"
|
||||
example = "https://civitai.com/collections/12345"
|
||||
|
||||
def images(self):
|
||||
cid = int(self.groups[0])
|
||||
self.kwdict["collection"] = col = self.api.collection(cid)
|
||||
self.kwdict["user_collection"] = col.pop("user", None)
|
||||
|
||||
params = {
|
||||
"collectionId" : cid,
|
||||
"period" : "AllTime",
|
||||
"sort" : "Newest",
|
||||
"browsingLevel" : self.api.nsfw,
|
||||
"include" : ("cosmetics",),
|
||||
}
|
||||
return self.api.images(params, defaults=False)
|
||||
|
||||
|
||||
class CivitaiPostExtractor(CivitaiExtractor):
|
||||
subcategory = "post"
|
||||
directory_fmt = ("{category}", "{username|user[username]}", "posts",
|
||||
@@ -635,7 +657,7 @@ class CivitaiTrpcAPI():
|
||||
self.root = extractor.root + "/api/trpc/"
|
||||
self.headers = {
|
||||
"content-type" : "application/json",
|
||||
"x-client-version": "5.0.920",
|
||||
"x-client-version": "5.0.954",
|
||||
"x-client-date" : "",
|
||||
"x-client" : "web",
|
||||
"x-fingerprint" : "undefined",
|
||||
@@ -758,6 +780,11 @@ class CivitaiTrpcAPI():
|
||||
params = self._type_params(params)
|
||||
return self._pagination(endpoint, params, meta)
|
||||
|
||||
def collection(self, collection_id):
|
||||
endpoint = "collection.getById"
|
||||
params = {"id": int(collection_id)}
|
||||
return self._call(endpoint, params)["collection"]
|
||||
|
||||
def user(self, username):
|
||||
endpoint = "user.getCreator"
|
||||
params = {"username": username}
|
||||
@@ -783,9 +810,8 @@ class CivitaiTrpcAPI():
|
||||
|
||||
params = {"input": util.json_dumps(input)}
|
||||
headers["x-client-date"] = str(int(time.time() * 1000))
|
||||
response = self.extractor.request(url, params=params, headers=headers)
|
||||
|
||||
return response.json()["result"]["data"]["json"]
|
||||
return self.extractor.request_json(
|
||||
url, params=params, headers=headers)["result"]["data"]["json"]
|
||||
|
||||
def _pagination(self, endpoint, params, meta=None):
|
||||
if "cursor" not in params:
|
||||
|
||||
@@ -529,4 +529,135 @@ __tests__ = (
|
||||
"#auth" : True,
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://civitai.com/collections/11035869",
|
||||
"#class" : civitai.CivitaiCollectionExtractor,
|
||||
"#results" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/9b9c64b4-60de-4a9c-becd-a386ecf3fa7a/original=true/DailyWorldMorphChallenge_Base_0003.png",
|
||||
|
||||
"filename" : "DailyWorldMorphChallenge_Base_0003",
|
||||
"extension" : "png",
|
||||
"url" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/9b9c64b4-60de-4a9c-becd-a386ecf3fa7a/original=true/DailyWorldMorphChallenge_Base_0003.png",
|
||||
"collection" : {
|
||||
"availability": "Public",
|
||||
"description" : "",
|
||||
"id" : 11035869,
|
||||
"image" : None,
|
||||
"metadata" : {},
|
||||
"mode" : None,
|
||||
"name" : "Trees, Bonsai, and so one",
|
||||
"nsfw" : False,
|
||||
"nsfwLevel" : 1,
|
||||
"read" : "Public",
|
||||
"tags" : [],
|
||||
"type" : "Image",
|
||||
"userId" : 4831516,
|
||||
"write" : "Private",
|
||||
},
|
||||
"file" : {
|
||||
"acceptableMinor": False,
|
||||
"availability" : "Public",
|
||||
"blockedFor" : None,
|
||||
"cosmetic" : None,
|
||||
"createdAt" : "2025-04-30T20:20:44.015Z",
|
||||
"date" : "dt:2025-04-30 20:20:44",
|
||||
"hasMeta" : True,
|
||||
"hasPositivePrompt": True,
|
||||
"hash" : "UHEfvNWC?dof00oc4Uae$,ofV}WFxeWCxwWV",
|
||||
"height" : 1152,
|
||||
"hideMeta" : False,
|
||||
"id" : 73339178,
|
||||
"index" : 1,
|
||||
"ingestion" : "Scanned",
|
||||
"metadata" : {
|
||||
"hash" : "UHEfvNWC?dof00oc4Uae$,ofV}WFxeWCxwWV",
|
||||
"height": 1152,
|
||||
"size" : 1523677,
|
||||
"width" : 896,
|
||||
},
|
||||
"mimeType" : "image/png",
|
||||
"minor" : False,
|
||||
"modelVersionId" : None,
|
||||
"modelVersionIds": [],
|
||||
"modelVersionIdsManual": [],
|
||||
"name" : "DailyWorldMorphChallenge_Base_0003.png",
|
||||
"needsReview" : None,
|
||||
"nsfwLevel" : 1,
|
||||
"onSite" : False,
|
||||
"poi" : False,
|
||||
"postId" : 16290779,
|
||||
"postTitle" : None,
|
||||
"publishedAt" : "2025-04-30T20:23:40.409Z",
|
||||
"reactions" : [],
|
||||
"remixOfId" : None,
|
||||
"scannedAt" : "2025-04-30T20:20:48.072Z",
|
||||
"sortAt" : "2025-04-30T20:23:40.409Z",
|
||||
"stats" : {
|
||||
"collectedCountAllTime": 1,
|
||||
"commentCountAllTime": 0,
|
||||
"cryCountAllTime" : 1,
|
||||
"dislikeCountAllTime": 0,
|
||||
"heartCountAllTime": 1,
|
||||
"laughCountAllTime": 0,
|
||||
"likeCountAllTime": 5,
|
||||
"tippedAmountCountAllTime": 0,
|
||||
"viewCountAllTime": 0,
|
||||
},
|
||||
"tagIds" : [
|
||||
5248,
|
||||
9143,
|
||||
111839,
|
||||
112019,
|
||||
116352,
|
||||
120250,
|
||||
161904,
|
||||
234268,
|
||||
],
|
||||
"tags" : None,
|
||||
"thumbnailUrl" : None,
|
||||
"type" : "image",
|
||||
"url" : "9b9c64b4-60de-4a9c-becd-a386ecf3fa7a",
|
||||
"uuid" : "9b9c64b4-60de-4a9c-becd-a386ecf3fa7a",
|
||||
"width" : 896,
|
||||
},
|
||||
"user" : {
|
||||
"cosmetics" : list,
|
||||
"deletedAt" : None,
|
||||
"id" : 2624648,
|
||||
"image" : "ce0f7d5e-cc4a-41e2-8587-75d823c85ce9",
|
||||
"profilePicture": None,
|
||||
"username" : "AIArtsChannel",
|
||||
},
|
||||
"user_collection": {
|
||||
"cosmetics" : [],
|
||||
"deletedAt" : None,
|
||||
"id" : 4831516,
|
||||
"image" : "https://lh3.googleusercontent.com/a/ACg8ocKeClAsD6kmHOATnC4Li1PLYw9-J41LCaVHdzcLLGZi9ElNUQ=s96-c",
|
||||
"profilePicture": None,
|
||||
"username" : "TettyCo",
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://civitai.com/collections/11453135",
|
||||
"#class" : civitai.CivitaiCollectionExtractor,
|
||||
"#count" : 12,
|
||||
|
||||
"collection" : {
|
||||
"availability": "Public",
|
||||
"description" : "",
|
||||
"id" : 11453135,
|
||||
"image" : None,
|
||||
"metadata" : {},
|
||||
"mode" : None,
|
||||
"name" : "Sakura Trees",
|
||||
"nsfw" : False,
|
||||
"nsfwLevel" : 3,
|
||||
"read" : "Public",
|
||||
"tags" : [],
|
||||
"type" : "Image",
|
||||
"userId" : 8511981,
|
||||
"write" : "Private",
|
||||
},
|
||||
},
|
||||
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user