[civitai] fix/improve query parameter handling (#7138)
This commit is contained in:
Mike Fährmann
@@ -144,6 +144,11 @@ class CivitaiExtractor(Extractor):
|
||||
file["generation"] = self.api.image_generationdata(file["id"])
|
||||
yield data
|
||||
|
||||
def _parse_query(self, value):
|
||||
return text.parse_query_list(
|
||||
value, {"tags", "reactions", "baseModels", "tools", "techniques",
|
||||
"types", "fileFormats"})
|
||||
|
||||
|
||||
class CivitaiModelExtractor(CivitaiExtractor):
|
||||
subcategory = "model"
|
||||
@@ -348,8 +353,9 @@ class CivitaiUserModelsExtractor(CivitaiExtractor):
|
||||
example = "https://civitai.com/user/USER/models"
|
||||
|
||||
def models(self):
|
||||
params = text.parse_query(self.groups[1])
|
||||
params["username"] = text.unquote(self.groups[0])
|
||||
user, query = self.groups
|
||||
params = self._parse_query(query)
|
||||
params["username"] = text.unquote(user)
|
||||
return self.api.models(params)
|
||||
|
||||
|
||||
@@ -361,8 +367,9 @@ class CivitaiUserPostsExtractor(CivitaiExtractor):
|
||||
example = "https://civitai.com/user/USER/posts"
|
||||
|
||||
def posts(self):
|
||||
params = text.parse_query(self.groups[1])
|
||||
params["username"] = text.unquote(self.groups[0])
|
||||
user, query = self.groups
|
||||
params = self._parse_query(query)
|
||||
params["username"] = text.unquote(user)
|
||||
return self.api.posts(params)
|
||||
|
||||
|
||||
@@ -372,8 +379,7 @@ class CivitaiUserImagesExtractor(CivitaiExtractor):
|
||||
example = "https://civitai.com/user/USER/images"
|
||||
|
||||
def __init__(self, match):
|
||||
self.params = text.parse_query_list(
|
||||
match.group(2), {"reactions"})
|
||||
self.params = self._parse_query(match.group(2))
|
||||
if self.params.get("section") == "reactions":
|
||||
self.subcategory = "reactions"
|
||||
self.images = self.images_reactions
|
||||
@@ -406,9 +412,11 @@ class CivitaiUserVideosExtractor(CivitaiExtractor):
|
||||
|
||||
def images(self):
|
||||
self._image_ext = "mp4"
|
||||
params = text.parse_query(self.groups[1])
|
||||
|
||||
user, query = self.groups
|
||||
params = self._parse_query(query)
|
||||
params["types"] = ["video"]
|
||||
params["username"] = text.unquote(self.groups[0])
|
||||
params["username"] = text.unquote(user)
|
||||
return self.api.images(params)
|
||||
|
||||
|
||||
@@ -496,7 +504,7 @@ class CivitaiTrpcAPI():
|
||||
self.root = extractor.root + "/api/trpc/"
|
||||
self.headers = {
|
||||
"content-type" : "application/json",
|
||||
"x-client-version": "5.0.394",
|
||||
"x-client-version": "5.0.542",
|
||||
"x-client-date" : "",
|
||||
"x-client" : "web",
|
||||
"x-fingerprint" : "undefined",
|
||||
@@ -657,15 +665,35 @@ class CivitaiTrpcAPI():
|
||||
meta_ = meta
|
||||
|
||||
def _merge_params(self, params_user, params_default):
|
||||
"""Combine 'params_user' with 'params_default'"""
|
||||
params_default.update(params_user)
|
||||
return params_default
|
||||
|
||||
def _type_params(self, params):
|
||||
for key, type in (
|
||||
("tags" , int),
|
||||
("modelId" , int),
|
||||
("modelVersionId", int),
|
||||
):
|
||||
if key in params:
|
||||
params[key] = type(params[key])
|
||||
"""Convert 'params' values to expected types"""
|
||||
types = {
|
||||
"tags" : int,
|
||||
"tools" : int,
|
||||
"techniques" : int,
|
||||
"modelId" : int,
|
||||
"modelVersionId": int,
|
||||
"remixesOnly" : _bool,
|
||||
"nonRemixesOnly": _bool,
|
||||
"withMeta" : _bool,
|
||||
"fromPlatform" : _bool,
|
||||
"supportsGeneration": _bool,
|
||||
}
|
||||
|
||||
for name, value in params.items():
|
||||
if name not in types:
|
||||
continue
|
||||
elif isinstance(value, str):
|
||||
params[name] = types[name](value)
|
||||
elif isinstance(value, list):
|
||||
type = types[name]
|
||||
params[name] = [type(item) for item in value]
|
||||
return params
|
||||
|
||||
|
||||
def _bool(value):
|
||||
return True if value == "true" else False
|
||||
|
||||
@@ -13,12 +13,15 @@ __tests__ = (
|
||||
"#url" : "https://civitai.com/models/703211/maid-classic",
|
||||
"#class": civitai.CivitaiModelExtractor,
|
||||
"#urls" : [
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2dd1dc69-45a6-4beb-b36b-2e2bc65e3cda/original=true/00015-2885514572.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/51ea6a54-762c-46cf-9588-726461193c96/original=true/00019-2944604798.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/aaa474a8-5a4d-4003-819f-79df2935ad78/original=true/00020-1919126538.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/1bb22783-1c29-405e-9d7e-7c98b5a53d65/original=true/00021-2415646212.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2dd1dc69-45a6-4beb-b36b-2e2bc65e3cda/original=true/00015-2885514572.png",
|
||||
],
|
||||
|
||||
"model" : {
|
||||
@@ -205,6 +208,20 @@ __tests__ = (
|
||||
"#count" : ">= 8",
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://civitai.com/user/waomodder/models?tag=character&types=Checkpoint&types=TextualInversion&types=Hypernetwork&types=LORA&checkpointType=Trained&fileFormats=SafeTensor&fileFormats=PickleTensor",
|
||||
"#comment": "various filters (#7138)",
|
||||
"#class" : civitai.CivitaiUserModelsExtractor,
|
||||
"#urls" : (
|
||||
"https://civitai.com/models/42166",
|
||||
"https://civitai.com/models/79845",
|
||||
"https://civitai.com/models/81424",
|
||||
"https://civitai.com/models/75925",
|
||||
"https://civitai.com/models/65818",
|
||||
"https://civitai.com/models/64272",
|
||||
),
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://civitai.com/user/waomodder/posts",
|
||||
"#class": civitai.CivitaiUserPostsExtractor,
|
||||
@@ -221,6 +238,25 @@ __tests__ = (
|
||||
"#count" : 50,
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://civitai.com/user/waomodder/images?tags=5132",
|
||||
"#comment": "tags (#7138)",
|
||||
"#class" : civitai.CivitaiUserImagesExtractor,
|
||||
"#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/8cc7c513-ba77-4444-a21f-7e3907d29a4e/original=true/982824.png",
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://civitai.com/user/waomodder/images?sort=Most+Collected&period=AllTime&tags=6594&baseModels=Illustrious&baseModels=PixArt+a&baseModels=Other&baseModels=Pony&remixesOnly=false",
|
||||
"#comment": "various filters (#7138)",
|
||||
"#class" : civitai.CivitaiUserImagesExtractor,
|
||||
"#range" : "1-3",
|
||||
"#urls" : (
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c44c116a-263b-457d-8fa8-cc3d7716a0aa/original=true/36800924.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/0f6cf303-8b12-4401-914e-bff33371e9c6/original=true/36801099.png",
|
||||
"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/9a682316-e451-4b98-8873-cc6c2e2d39bb/original=true/36801079.png",
|
||||
),
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://civitai.com/user/USER/images?section=reactions",
|
||||
"#category": ("", "civitai", "reactions"),
|
||||
|
||||
Reference in New Issue
Block a user