diff --git a/docs/supportedsites.md b/docs/supportedsites.md index f3699147..d8916af4 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -166,7 +166,7 @@ Consider all listed sites to potentially be NSFW. Civitai https://www.civitai.com/ - individual Images, Models, Posts, Search Results, Tag Searches (Images), Tag Searches (Models), User Profiles, User Images, User Models, User Posts + individual Images, Image Listings, Models, Model Listings, Posts, Search Results, Tag Searches, User Profiles, User Images, User Models, User Posts diff --git a/gallery_dl/extractor/civitai.py b/gallery_dl/extractor/civitai.py index 1a54f4d1..6e9006f9 100644 --- a/gallery_dl/extractor/civitai.py +++ b/gallery_dl/extractor/civitai.py @@ -269,24 +269,14 @@ class CivitaiPostExtractor(CivitaiExtractor): return ({"id": int(self.groups[0])},) -class CivitaiTagModelsExtractor(CivitaiExtractor): - subcategory = "tag-models" - pattern = BASE_PATTERN + r"/(?:tag/|models\?tag=)([^/?&#]+)" +class CivitaiTagExtractor(CivitaiExtractor): + subcategory = "tag" + pattern = BASE_PATTERN + r"/tag/([^/?&#]+)" example = "https://civitai.com/tag/TAG" def models(self): tag = text.unquote(self.groups[0]) - return self.api.models({"tag": tag}) - - -class CivitaiTagImagesExtractor(CivitaiExtractor): - subcategory = "tag-images" - pattern = BASE_PATTERN + r"/images\?tags=([^&#]+)" - example = "https://civitai.com/images?tags=12345" - - def images(self): - tag = text.unquote(self.groups[0]) - return self.api.images({"tag": tag}) + return self.api.models_tag(tag) class CivitaiSearchExtractor(CivitaiExtractor): @@ -299,6 +289,26 @@ class CivitaiSearchExtractor(CivitaiExtractor): return self.api.models(params) +class CivitaiModelsExtractor(CivitaiExtractor): + subcategory = "models" + pattern = BASE_PATTERN + r"/models(?:/?\?([^#]+))?(?:$|#)" + example = "https://civitai.com/models" + + def models(self): + params = text.parse_query(self.groups[0]) + return self.api.models(params) + + +class CivitaiImagesExtractor(CivitaiExtractor): + subcategory = "images" + pattern = BASE_PATTERN + r"/images(?:/?\?([^#]+))?(?:$|#)" + example = "https://civitai.com/images" + + def images(self): + params = text.parse_query(self.groups[0]) + return self.api.images(params) + + class CivitaiUserExtractor(CivitaiExtractor): subcategory = "user" pattern = USER_PATTERN + r"/?(?:$|\?|#)" @@ -402,6 +412,9 @@ class CivitaiRestAPI(): def models(self, params): return self._pagination("/v1/models", params) + def models_tag(self, tag): + return self.models({"tag": tag}) + def _call(self, endpoint, params=None): if endpoint[0] == "/": url = self.root + endpoint @@ -469,6 +482,7 @@ class CivitaiTrpcAPI(): "include" : ["cosmetics"], }) + params = self._type_params(params) return self._pagination(endpoint, params) def images_gallery(self, model, version, user): @@ -522,6 +536,9 @@ class CivitaiTrpcAPI(): return self._pagination(endpoint, params) + def models_tag(self, tag): + return self.models({"tagname": tag}) + def post(self, post_id): endpoint = "post.get" params = {"id": int(post_id)} @@ -586,3 +603,13 @@ class CivitaiTrpcAPI(): def _merge_params(self, params_user, params_default): params_default.update(params_user) return params_default + + def _type_params(self, params): + for key, type in ( + ("tags" , int), + ("modelId" , int), + ("modelVersionId", int), + ): + if key in params: + params[key] = type(params[key]) + return params diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 1e565a9a..5c27e16e 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -198,11 +198,11 @@ SUBCATEGORY_MAP = { "feed": "Subscriptions Feed", }, "civitai": { - "tag-models": "Tag Searches (Models)", - "tag-images": "Tag Searches (Images)", + "models": "Model Listings", + "images": "Image Listings", "user-models": "User Models", "user-images": "User Images", - "user-posts": "User Posts", + "user-posts" : "User Posts", }, "coomerparty": { "discord" : "", diff --git a/test/results/civitai.py b/test/results/civitai.py index 65982eed..927d6b0a 100644 --- a/test/results/civitai.py +++ b/test/results/civitai.py @@ -121,12 +121,22 @@ __tests__ = ( { "#url" : "https://civitai.com/tag/mecha", - "#class": civitai.CivitaiTagModelsExtractor, + "#class": civitai.CivitaiTagExtractor, }, { "#url" : "https://civitai.com/images?tags=482", - "#class": civitai.CivitaiTagImagesExtractor, + "#class": civitai.CivitaiImagesExtractor, +}, + +{ + "#url" : "https://civitai.com/images?modelVersionId=786644", + "#class": civitai.CivitaiImagesExtractor, +}, + +{ + "#url" : "https://civitai.com/models", + "#class": civitai.CivitaiModelsExtractor, }, {