[tungsten] add support (#8061)
This commit is contained in:
Mike Fährmann
@@ -991,6 +991,12 @@ Consider all listed sites to potentially be NSFW.
|
||||
<td>Posts, Search Results, Tumblrblogs</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr id="tungsten" title="tungsten">
|
||||
<td>Tungsten</td>
|
||||
<td>https://tungsten.run/</td>
|
||||
<td>Models, Posts</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr id="twibooru" title="twibooru">
|
||||
<td>Twibooru</td>
|
||||
<td>https://twibooru.org/</td>
|
||||
|
||||
@@ -190,6 +190,7 @@ modules = [
|
||||
"tsumino",
|
||||
"tumblr",
|
||||
"tumblrgallery",
|
||||
"tungsten",
|
||||
"twibooru",
|
||||
"twitter",
|
||||
"urlgalleries",
|
||||
|
||||
100
gallery_dl/extractor/tungsten.py
Normal file
100
gallery_dl/extractor/tungsten.py
Normal file
@@ -0,0 +1,100 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
"""Extractors for https://tungsten.run/"""
|
||||
|
||||
from .common import Extractor, Message
|
||||
from .. import text
|
||||
|
||||
BASE_PATTERN = r"(?:https?://)?(?:www\.)?tungsten\.run"
|
||||
|
||||
|
||||
class TungstenExtractor(Extractor):
|
||||
"""Base class for tungsten extractors"""
|
||||
category = "tungsten"
|
||||
root = "https://tungsten.run"
|
||||
directory_fmt = ("{category}", "{user[username]}")
|
||||
filename_fmt = "{date} {title:?/ /}{uuid}.{extension}"
|
||||
archive_fmt = "{uuid}"
|
||||
|
||||
def items(self):
|
||||
for post in self.posts():
|
||||
url = post["original_url"]
|
||||
post["date"] = text.parse_datetime(post["created_at"])
|
||||
post["filename"] = url[url.rfind("/")+1:]
|
||||
post["extension"] = "webp"
|
||||
yield Message.Directory, post
|
||||
yield Message.Url, url, post
|
||||
|
||||
def _pagination(self, url, params):
|
||||
params["page"] = 1
|
||||
params["per_page"] = 40
|
||||
|
||||
headers = {
|
||||
"Origin": self.root,
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-site",
|
||||
}
|
||||
|
||||
while True:
|
||||
data = self.request_json(url, params=params, headers=headers)
|
||||
|
||||
yield from data
|
||||
|
||||
if len(data) < params["per_page"]:
|
||||
break
|
||||
params["page"] += 1
|
||||
|
||||
|
||||
class TungstenPostExtractor(TungstenExtractor):
|
||||
subcategory = "post"
|
||||
pattern = rf"{BASE_PATTERN}/post/(\w+)"
|
||||
example = "https://tungsten.run/post/AbCdEfGhIjKlMnOp"
|
||||
|
||||
def posts(self):
|
||||
url = f"{self.root}/post/{self.groups[0]}"
|
||||
page = self.request(url).text
|
||||
data = self._extract_nextdata(page)
|
||||
return (data["props"]["pageProps"]["post"],)
|
||||
|
||||
|
||||
class TungstenModelExtractor(TungstenExtractor):
|
||||
subcategory = "model"
|
||||
pattern = rf"{BASE_PATTERN}/model/(\w+)(?:/?\?model_version=(\w+))?"
|
||||
example = "https://tungsten.run/model/AbCdEfGhIjKlM"
|
||||
|
||||
def posts(self):
|
||||
uuid_model, uuid_version = self.groups
|
||||
|
||||
if uuid_version is None:
|
||||
url = f"{self.root}/model/{uuid_model}/"
|
||||
page = self.request(url).text
|
||||
uuid_version = text.extr(page, '"modelVersionUUID":"', '"')
|
||||
|
||||
url = "https://api.tungsten.run/v1/posts"
|
||||
params = {
|
||||
"sort" : "top_all_time",
|
||||
"tweakable_only": "false",
|
||||
"following" : "false",
|
||||
"model_version_uuid": uuid_version,
|
||||
}
|
||||
return self._pagination(url, params)
|
||||
|
||||
|
||||
class TungstenUserExtractor(TungstenExtractor):
|
||||
subcategory = "user"
|
||||
pattern = rf"{BASE_PATTERN}/user/([^/?#]+)"
|
||||
example = "https://tungsten.run/user/USER/posts"
|
||||
|
||||
def posts(self):
|
||||
url = f"{self.root}/user/{self.groups[0]}"
|
||||
page = self.request(url).text
|
||||
uuid_user = text.extr(page, '"uuid":"', '"')
|
||||
|
||||
url = f"https://api.tungsten.run/v1/users/{uuid_user}/posts"
|
||||
params = {"sort": "top_all_time"}
|
||||
return self._pagination(url, params)
|
||||
147
test/results/tungsten.py
Normal file
147
test/results/tungsten.py
Normal file
@@ -0,0 +1,147 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
|
||||
from gallery_dl.extractor import tungsten
|
||||
|
||||
|
||||
__tests__ = (
|
||||
{
|
||||
"#url" : "https://tungsten.run/post/VuXCdyw9vohCAFyN",
|
||||
"#class" : tungsten.TungstenPostExtractor,
|
||||
"#results" : "https://api.tungsten.run/v1/upload/aPg2RYeezA9wZ8uQ",
|
||||
|
||||
"comment_count" : int,
|
||||
"created_at" : "2024-12-25T00:05:39Z",
|
||||
"date" : "dt:2024-12-25 00:05:39",
|
||||
"description" : None,
|
||||
"extension" : "webp",
|
||||
"filename" : "aPg2RYeezA9wZ8uQ",
|
||||
"generation_postprocessing": None,
|
||||
"generation_type": "flux1d",
|
||||
"is_tweakable" : True,
|
||||
"like_count" : range(50, 200),
|
||||
"model_versions" : [{
|
||||
"cover_url" : "https://api.tungsten.run/v1/model_version_cover/48M6hN3HsQ/1726596449",
|
||||
"created_at" : "2024-09-17T18:05:05Z",
|
||||
"fullname" : "Flux.1 [Dev] - v1",
|
||||
"model_base" : "flux1d",
|
||||
"model_name" : "Flux.1 [Dev]",
|
||||
"model_resolved_nsfw_score": 0,
|
||||
"model_type" : "checkpoint",
|
||||
"model_uuid" : "9fThn2WaVP",
|
||||
"name" : "v1",
|
||||
"sm_cover_url": "https://api.tungsten.run/v1/model_version_cover/48M6hN3HsQ/1726596449",
|
||||
"status" : "approved",
|
||||
"uuid" : "48M6hN3HsQ",
|
||||
"extra" : {"dtype": "fp16"},
|
||||
}],
|
||||
"nsfw" : False,
|
||||
"original_url" : "https://api.tungsten.run/v1/upload/aPg2RYeezA9wZ8uQ",
|
||||
"original_width" : 832,
|
||||
"original_height": 1216,
|
||||
"rating" : None,
|
||||
"resized_url" : "https://api.tungsten.run/v1/upload/jRJMBAD5zG3NFar6",
|
||||
"resized_width" : 832,
|
||||
"resized_height" : 1216,
|
||||
"title" : "Moon",
|
||||
"uuid" : "VuXCdyw9vohCAFyN",
|
||||
"view_count" : int,
|
||||
"tags" : [
|
||||
"art",
|
||||
"moon",
|
||||
"surreal",
|
||||
],
|
||||
"generation_data": {
|
||||
"cfg" : 4,
|
||||
"height" : 1216,
|
||||
"img2img" : None,
|
||||
"loras" : [],
|
||||
"model_version_uuid": "48M6hN3HsQ",
|
||||
"num_images" : 2,
|
||||
"postprocess": [],
|
||||
"prompt" : "low angle photograph of a woman, standing on top of a small mountain, hands in the air holding up the full moon late at night, long brunette hair, loose flowing spaghetti strap dress, hair and dress blowing in the wind, cler sky",
|
||||
"sampler" : "euler",
|
||||
"seed" : 7147106691,
|
||||
"steps" : 20,
|
||||
"width" : 832,
|
||||
},
|
||||
"user" : {
|
||||
"avatar_url" : "https://api.tungsten.run/v1/avatar/512x512/HDiix5cCtzg2VrZU/1726485265",
|
||||
"badges" : list,
|
||||
"bio" : "Hello, world! 😋",
|
||||
"created_at" : "2024-09-16T17:18:58Z",
|
||||
"display_name" : "Survivor",
|
||||
"sm_avatar_url": "https://api.tungsten.run/v1/avatar/64x64/HDiix5cCtzg2VrZU/1726485265",
|
||||
"type" : "admin",
|
||||
"username" : "survivor",
|
||||
"uuid" : "HDiix5cCtzg2VrZU",
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://tungsten.run/model/9vHB2hNUdg/chroma",
|
||||
"#class" : tungsten.TungstenModelExtractor,
|
||||
"#pattern" : r"https://api\.tungsten\.run/v1/upload/\w+",
|
||||
"#count" : 17,
|
||||
|
||||
"comment_count" : int,
|
||||
"created_at" : "iso:8601",
|
||||
"date" : "type:datetime",
|
||||
"extension" : "webp",
|
||||
"filename" : str,
|
||||
"like_count" : int,
|
||||
"nsfw" : bool,
|
||||
"original_url" : str,
|
||||
"original_width" : int,
|
||||
"original_height": int,
|
||||
"rating" : None,
|
||||
"resized_url" : str,
|
||||
"resized_width" : int,
|
||||
"resized_height" : int,
|
||||
"title" : str,
|
||||
"uuid" : "len:str:16",
|
||||
"view_count" : int,
|
||||
"user" : {
|
||||
"avatar_url" : str,
|
||||
"badges" : list,
|
||||
"bio" : {str, None},
|
||||
"created_at" : str,
|
||||
"display_name" : str,
|
||||
"sm_avatar_url": str,
|
||||
"type" : "regular",
|
||||
"username" : str,
|
||||
"uuid" : "len:str:16",
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://tungsten.run/model/9vHB2hNUdg?model_version=CxvEH6esrG",
|
||||
"#class" : tungsten.TungstenModelExtractor,
|
||||
"#range" : "1-50",
|
||||
"#pattern" : r"https://api\.tungsten\.run/v1/upload/\w+",
|
||||
"#count" : 50,
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://tungsten.run/user/lynodie",
|
||||
"#class" : tungsten.TungstenUserExtractor,
|
||||
"#range" : "1-50",
|
||||
"#pattern" : r"https://api\.tungsten\.run/v1/upload/\w+",
|
||||
"#count" : 50,
|
||||
|
||||
"user" : {
|
||||
"avatar_url" : "https://api.tungsten.run/v1/avatar/512x512/EYJsXQDD69u26NYT/1751499148",
|
||||
"bio" : "Just a normal lesbian girl from Germany.",
|
||||
"created_at" : "2024-09-16T18:01:20Z",
|
||||
"display_name" : "Sarah",
|
||||
"sm_avatar_url": "https://api.tungsten.run/v1/avatar/64x64/EYJsXQDD69u26NYT/1751499148",
|
||||
"type" : "regular",
|
||||
"username" : "lynodie",
|
||||
"uuid" : "EYJsXQDD69u26NYT",
|
||||
},
|
||||
},
|
||||
|
||||
)
|
||||
Reference in New Issue
Block a user