From b4e94e6d7556101c5be7eec5f558730c1eb80dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 28 Aug 2025 18:41:51 +0200 Subject: [PATCH] [tungsten] add support (#8061) --- docs/supportedsites.md | 6 ++ gallery_dl/extractor/__init__.py | 1 + gallery_dl/extractor/tungsten.py | 100 +++++++++++++++++++++ test/results/tungsten.py | 147 +++++++++++++++++++++++++++++++ 4 files changed, 254 insertions(+) create mode 100644 gallery_dl/extractor/tungsten.py create mode 100644 test/results/tungsten.py diff --git a/docs/supportedsites.md b/docs/supportedsites.md index e511993f..86b4732e 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -991,6 +991,12 @@ Consider all listed sites to potentially be NSFW. Posts, Search Results, Tumblrblogs + + Tungsten + https://tungsten.run/ + Models, Posts + + Twibooru https://twibooru.org/ diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index aabaa933..c2822f3e 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -190,6 +190,7 @@ modules = [ "tsumino", "tumblr", "tumblrgallery", + "tungsten", "twibooru", "twitter", "urlgalleries", diff --git a/gallery_dl/extractor/tungsten.py b/gallery_dl/extractor/tungsten.py new file mode 100644 index 00000000..22699c74 --- /dev/null +++ b/gallery_dl/extractor/tungsten.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://tungsten.run/""" + +from .common import Extractor, Message +from .. import text + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?tungsten\.run" + + +class TungstenExtractor(Extractor): + """Base class for tungsten extractors""" + category = "tungsten" + root = "https://tungsten.run" + directory_fmt = ("{category}", "{user[username]}") + filename_fmt = "{date} {title:?/ /}{uuid}.{extension}" + archive_fmt = "{uuid}" + + def items(self): + for post in self.posts(): + url = post["original_url"] + post["date"] = text.parse_datetime(post["created_at"]) + post["filename"] = url[url.rfind("/")+1:] + post["extension"] = "webp" + yield Message.Directory, post + yield Message.Url, url, post + + def _pagination(self, url, params): + params["page"] = 1 + params["per_page"] = 40 + + headers = { + "Origin": self.root, + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-site", + } + + while True: + data = self.request_json(url, params=params, headers=headers) + + yield from data + + if len(data) < params["per_page"]: + break + params["page"] += 1 + + +class TungstenPostExtractor(TungstenExtractor): + subcategory = "post" + pattern = rf"{BASE_PATTERN}/post/(\w+)" + example = "https://tungsten.run/post/AbCdEfGhIjKlMnOp" + + def posts(self): + url = f"{self.root}/post/{self.groups[0]}" + page = self.request(url).text + data = self._extract_nextdata(page) + return (data["props"]["pageProps"]["post"],) + + +class TungstenModelExtractor(TungstenExtractor): + subcategory = "model" + pattern = rf"{BASE_PATTERN}/model/(\w+)(?:/?\?model_version=(\w+))?" + example = "https://tungsten.run/model/AbCdEfGhIjKlM" + + def posts(self): + uuid_model, uuid_version = self.groups + + if uuid_version is None: + url = f"{self.root}/model/{uuid_model}/" + page = self.request(url).text + uuid_version = text.extr(page, '"modelVersionUUID":"', '"') + + url = "https://api.tungsten.run/v1/posts" + params = { + "sort" : "top_all_time", + "tweakable_only": "false", + "following" : "false", + "model_version_uuid": uuid_version, + } + return self._pagination(url, params) + + +class TungstenUserExtractor(TungstenExtractor): + subcategory = "user" + pattern = rf"{BASE_PATTERN}/user/([^/?#]+)" + example = "https://tungsten.run/user/USER/posts" + + def posts(self): + url = f"{self.root}/user/{self.groups[0]}" + page = self.request(url).text + uuid_user = text.extr(page, '"uuid":"', '"') + + url = f"https://api.tungsten.run/v1/users/{uuid_user}/posts" + params = {"sort": "top_all_time"} + return self._pagination(url, params) diff --git a/test/results/tungsten.py b/test/results/tungsten.py new file mode 100644 index 00000000..b523698d --- /dev/null +++ b/test/results/tungsten.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tungsten + + +__tests__ = ( +{ + "#url" : "https://tungsten.run/post/VuXCdyw9vohCAFyN", + "#class" : tungsten.TungstenPostExtractor, + "#results" : "https://api.tungsten.run/v1/upload/aPg2RYeezA9wZ8uQ", + + "comment_count" : int, + "created_at" : "2024-12-25T00:05:39Z", + "date" : "dt:2024-12-25 00:05:39", + "description" : None, + "extension" : "webp", + "filename" : "aPg2RYeezA9wZ8uQ", + "generation_postprocessing": None, + "generation_type": "flux1d", + "is_tweakable" : True, + "like_count" : range(50, 200), + "model_versions" : [{ + "cover_url" : "https://api.tungsten.run/v1/model_version_cover/48M6hN3HsQ/1726596449", + "created_at" : "2024-09-17T18:05:05Z", + "fullname" : "Flux.1 [Dev] - v1", + "model_base" : "flux1d", + "model_name" : "Flux.1 [Dev]", + "model_resolved_nsfw_score": 0, + "model_type" : "checkpoint", + "model_uuid" : "9fThn2WaVP", + "name" : "v1", + "sm_cover_url": "https://api.tungsten.run/v1/model_version_cover/48M6hN3HsQ/1726596449", + "status" : "approved", + "uuid" : "48M6hN3HsQ", + "extra" : {"dtype": "fp16"}, + }], + "nsfw" : False, + "original_url" : "https://api.tungsten.run/v1/upload/aPg2RYeezA9wZ8uQ", + "original_width" : 832, + "original_height": 1216, + "rating" : None, + "resized_url" : "https://api.tungsten.run/v1/upload/jRJMBAD5zG3NFar6", + "resized_width" : 832, + "resized_height" : 1216, + "title" : "Moon", + "uuid" : "VuXCdyw9vohCAFyN", + "view_count" : int, + "tags" : [ + "art", + "moon", + "surreal", + ], + "generation_data": { + "cfg" : 4, + "height" : 1216, + "img2img" : None, + "loras" : [], + "model_version_uuid": "48M6hN3HsQ", + "num_images" : 2, + "postprocess": [], + "prompt" : "low angle photograph of a woman, standing on top of a small mountain, hands in the air holding up the full moon late at night, long brunette hair, loose flowing spaghetti strap dress, hair and dress blowing in the wind, cler sky", + "sampler" : "euler", + "seed" : 7147106691, + "steps" : 20, + "width" : 832, + }, + "user" : { + "avatar_url" : "https://api.tungsten.run/v1/avatar/512x512/HDiix5cCtzg2VrZU/1726485265", + "badges" : list, + "bio" : "Hello, world! 😋", + "created_at" : "2024-09-16T17:18:58Z", + "display_name" : "Survivor", + "sm_avatar_url": "https://api.tungsten.run/v1/avatar/64x64/HDiix5cCtzg2VrZU/1726485265", + "type" : "admin", + "username" : "survivor", + "uuid" : "HDiix5cCtzg2VrZU", + }, +}, + +{ + "#url" : "https://tungsten.run/model/9vHB2hNUdg/chroma", + "#class" : tungsten.TungstenModelExtractor, + "#pattern" : r"https://api\.tungsten\.run/v1/upload/\w+", + "#count" : 17, + + "comment_count" : int, + "created_at" : "iso:8601", + "date" : "type:datetime", + "extension" : "webp", + "filename" : str, + "like_count" : int, + "nsfw" : bool, + "original_url" : str, + "original_width" : int, + "original_height": int, + "rating" : None, + "resized_url" : str, + "resized_width" : int, + "resized_height" : int, + "title" : str, + "uuid" : "len:str:16", + "view_count" : int, + "user" : { + "avatar_url" : str, + "badges" : list, + "bio" : {str, None}, + "created_at" : str, + "display_name" : str, + "sm_avatar_url": str, + "type" : "regular", + "username" : str, + "uuid" : "len:str:16", + }, +}, + +{ + "#url" : "https://tungsten.run/model/9vHB2hNUdg?model_version=CxvEH6esrG", + "#class" : tungsten.TungstenModelExtractor, + "#range" : "1-50", + "#pattern" : r"https://api\.tungsten\.run/v1/upload/\w+", + "#count" : 50, +}, + +{ + "#url" : "https://tungsten.run/user/lynodie", + "#class" : tungsten.TungstenUserExtractor, + "#range" : "1-50", + "#pattern" : r"https://api\.tungsten\.run/v1/upload/\w+", + "#count" : 50, + + "user" : { + "avatar_url" : "https://api.tungsten.run/v1/avatar/512x512/EYJsXQDD69u26NYT/1751499148", + "bio" : "Just a normal lesbian girl from Germany.", + "created_at" : "2024-09-16T18:01:20Z", + "display_name" : "Sarah", + "sm_avatar_url": "https://api.tungsten.run/v1/avatar/64x64/EYJsXQDD69u26NYT/1751499148", + "type" : "regular", + "username" : "lynodie", + "uuid" : "EYJsXQDD69u26NYT", + }, +}, + +)