From 2b03843092cdb08daf21e683e0df9a24c8c66dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 10 Aug 2025 17:37:48 +0200 Subject: [PATCH] [civitai] add 'collection' extractor (#8005) --- docs/supportedsites.md | 2 +- gallery_dl/extractor/civitai.py | 34 ++++++++- test/results/civitai.py | 131 ++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 5 deletions(-) diff --git a/docs/supportedsites.md b/docs/supportedsites.md index ae94ba77..8b2ee746 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -190,7 +190,7 @@ Consider all listed sites to potentially be NSFW. Civitai https://www.civitai.com/ - Generated Files, individual Images, Image Listings, Models, Model Listings, Posts, Post Listings, Image Searches, Model Searches, Tag Searches, User Profiles, User Images, Image Reactions, User Models, User Posts, User Videos, Video Reactions + Collections, Generated Files, individual Images, Image Listings, Models, Model Listings, Posts, Post Listings, Image Searches, Model Searches, Tag Searches, User Profiles, User Images, Image Reactions, User Models, User Posts, User Videos, Video Reactions diff --git a/gallery_dl/extractor/civitai.py b/gallery_dl/extractor/civitai.py index 52b952e8..557993e9 100644 --- a/gallery_dl/extractor/civitai.py +++ b/gallery_dl/extractor/civitai.py @@ -377,6 +377,28 @@ class CivitaiImageExtractor(CivitaiExtractor): return self.api.image(self.groups[0]) +class CivitaiCollectionExtractor(CivitaiExtractor): + subcategory = "collection" + directory_fmt = ("{category}", "{user_collection[username]}", + "collections", "{collection[id]}{collection[name]:? //}") + pattern = BASE_PATTERN + r"/collections/(\d+)" + example = "https://civitai.com/collections/12345" + + def images(self): + cid = int(self.groups[0]) + self.kwdict["collection"] = col = self.api.collection(cid) + self.kwdict["user_collection"] = col.pop("user", None) + + params = { + "collectionId" : cid, + "period" : "AllTime", + "sort" : "Newest", + "browsingLevel" : self.api.nsfw, + "include" : ("cosmetics",), + } + return self.api.images(params, defaults=False) + + class CivitaiPostExtractor(CivitaiExtractor): subcategory = "post" directory_fmt = ("{category}", "{username|user[username]}", "posts", @@ -635,7 +657,7 @@ class CivitaiTrpcAPI(): self.root = extractor.root + "/api/trpc/" self.headers = { "content-type" : "application/json", - "x-client-version": "5.0.920", + "x-client-version": "5.0.954", "x-client-date" : "", "x-client" : "web", "x-fingerprint" : "undefined", @@ -758,6 +780,11 @@ class CivitaiTrpcAPI(): params = self._type_params(params) return self._pagination(endpoint, params, meta) + def collection(self, collection_id): + endpoint = "collection.getById" + params = {"id": int(collection_id)} + return self._call(endpoint, params)["collection"] + def user(self, username): endpoint = "user.getCreator" params = {"username": username} @@ -783,9 +810,8 @@ class CivitaiTrpcAPI(): params = {"input": util.json_dumps(input)} headers["x-client-date"] = str(int(time.time() * 1000)) - response = self.extractor.request(url, params=params, headers=headers) - - return response.json()["result"]["data"]["json"] + return self.extractor.request_json( + url, params=params, headers=headers)["result"]["data"]["json"] def _pagination(self, endpoint, params, meta=None): if "cursor" not in params: diff --git a/test/results/civitai.py b/test/results/civitai.py index dc40b62d..b4c31cdf 100644 --- a/test/results/civitai.py +++ b/test/results/civitai.py @@ -529,4 +529,135 @@ __tests__ = ( "#auth" : True, }, +{ + "#url" : "https://civitai.com/collections/11035869", + "#class" : civitai.CivitaiCollectionExtractor, + "#results" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/9b9c64b4-60de-4a9c-becd-a386ecf3fa7a/original=true/DailyWorldMorphChallenge_Base_0003.png", + + "filename" : "DailyWorldMorphChallenge_Base_0003", + "extension" : "png", + "url" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/9b9c64b4-60de-4a9c-becd-a386ecf3fa7a/original=true/DailyWorldMorphChallenge_Base_0003.png", + "collection" : { + "availability": "Public", + "description" : "", + "id" : 11035869, + "image" : None, + "metadata" : {}, + "mode" : None, + "name" : "Trees, Bonsai, and so one", + "nsfw" : False, + "nsfwLevel" : 1, + "read" : "Public", + "tags" : [], + "type" : "Image", + "userId" : 4831516, + "write" : "Private", + }, + "file" : { + "acceptableMinor": False, + "availability" : "Public", + "blockedFor" : None, + "cosmetic" : None, + "createdAt" : "2025-04-30T20:20:44.015Z", + "date" : "dt:2025-04-30 20:20:44", + "hasMeta" : True, + "hasPositivePrompt": True, + "hash" : "UHEfvNWC?dof00oc4Uae$,ofV}WFxeWCxwWV", + "height" : 1152, + "hideMeta" : False, + "id" : 73339178, + "index" : 1, + "ingestion" : "Scanned", + "metadata" : { + "hash" : "UHEfvNWC?dof00oc4Uae$,ofV}WFxeWCxwWV", + "height": 1152, + "size" : 1523677, + "width" : 896, + }, + "mimeType" : "image/png", + "minor" : False, + "modelVersionId" : None, + "modelVersionIds": [], + "modelVersionIdsManual": [], + "name" : "DailyWorldMorphChallenge_Base_0003.png", + "needsReview" : None, + "nsfwLevel" : 1, + "onSite" : False, + "poi" : False, + "postId" : 16290779, + "postTitle" : None, + "publishedAt" : "2025-04-30T20:23:40.409Z", + "reactions" : [], + "remixOfId" : None, + "scannedAt" : "2025-04-30T20:20:48.072Z", + "sortAt" : "2025-04-30T20:23:40.409Z", + "stats" : { + "collectedCountAllTime": 1, + "commentCountAllTime": 0, + "cryCountAllTime" : 1, + "dislikeCountAllTime": 0, + "heartCountAllTime": 1, + "laughCountAllTime": 0, + "likeCountAllTime": 5, + "tippedAmountCountAllTime": 0, + "viewCountAllTime": 0, + }, + "tagIds" : [ + 5248, + 9143, + 111839, + 112019, + 116352, + 120250, + 161904, + 234268, + ], + "tags" : None, + "thumbnailUrl" : None, + "type" : "image", + "url" : "9b9c64b4-60de-4a9c-becd-a386ecf3fa7a", + "uuid" : "9b9c64b4-60de-4a9c-becd-a386ecf3fa7a", + "width" : 896, + }, + "user" : { + "cosmetics" : list, + "deletedAt" : None, + "id" : 2624648, + "image" : "ce0f7d5e-cc4a-41e2-8587-75d823c85ce9", + "profilePicture": None, + "username" : "AIArtsChannel", + }, + "user_collection": { + "cosmetics" : [], + "deletedAt" : None, + "id" : 4831516, + "image" : "https://lh3.googleusercontent.com/a/ACg8ocKeClAsD6kmHOATnC4Li1PLYw9-J41LCaVHdzcLLGZi9ElNUQ=s96-c", + "profilePicture": None, + "username" : "TettyCo", + }, +}, + +{ + "#url" : "https://civitai.com/collections/11453135", + "#class" : civitai.CivitaiCollectionExtractor, + "#count" : 12, + + "collection" : { + "availability": "Public", + "description" : "", + "id" : 11453135, + "image" : None, + "metadata" : {}, + "mode" : None, + "name" : "Sakura Trees", + "nsfw" : False, + "nsfwLevel" : 3, + "read" : "Public", + "tags" : [], + "type" : "Image", + "userId" : 8511981, + "write" : "Private", + }, +}, + )