diff --git a/docs/supportedsites.md b/docs/supportedsites.md index e511993f..86b4732e 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -991,6 +991,12 @@ Consider all listed sites to potentially be NSFW. Posts, Search Results, Tumblrblogs + + Tungsten + https://tungsten.run/ + Models, Posts + + Twibooru https://twibooru.org/ diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index aabaa933..c2822f3e 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -190,6 +190,7 @@ modules = [ "tsumino", "tumblr", "tumblrgallery", + "tungsten", "twibooru", "twitter", "urlgalleries", diff --git a/gallery_dl/extractor/tungsten.py b/gallery_dl/extractor/tungsten.py new file mode 100644 index 00000000..22699c74 --- /dev/null +++ b/gallery_dl/extractor/tungsten.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://tungsten.run/""" + +from .common import Extractor, Message +from .. import text + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?tungsten\.run" + + +class TungstenExtractor(Extractor): + """Base class for tungsten extractors""" + category = "tungsten" + root = "https://tungsten.run" + directory_fmt = ("{category}", "{user[username]}") + filename_fmt = "{date} {title:?/ /}{uuid}.{extension}" + archive_fmt = "{uuid}" + + def items(self): + for post in self.posts(): + url = post["original_url"] + post["date"] = text.parse_datetime(post["created_at"]) + post["filename"] = url[url.rfind("/")+1:] + post["extension"] = "webp" + yield Message.Directory, post + yield Message.Url, url, post + + def _pagination(self, url, params): + params["page"] = 1 + params["per_page"] = 40 + + headers = { + "Origin": self.root, + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-site", + } + + while True: + data = self.request_json(url, params=params, headers=headers) + + yield from data + + if len(data) < params["per_page"]: + break + params["page"] += 1 + + +class TungstenPostExtractor(TungstenExtractor): + subcategory = "post" + pattern = rf"{BASE_PATTERN}/post/(\w+)" + example = "https://tungsten.run/post/AbCdEfGhIjKlMnOp" + + def posts(self): + url = f"{self.root}/post/{self.groups[0]}" + page = self.request(url).text + data = self._extract_nextdata(page) + return (data["props"]["pageProps"]["post"],) + + +class TungstenModelExtractor(TungstenExtractor): + subcategory = "model" + pattern = rf"{BASE_PATTERN}/model/(\w+)(?:/?\?model_version=(\w+))?" + example = "https://tungsten.run/model/AbCdEfGhIjKlM" + + def posts(self): + uuid_model, uuid_version = self.groups + + if uuid_version is None: + url = f"{self.root}/model/{uuid_model}/" + page = self.request(url).text + uuid_version = text.extr(page, '"modelVersionUUID":"', '"') + + url = "https://api.tungsten.run/v1/posts" + params = { + "sort" : "top_all_time", + "tweakable_only": "false", + "following" : "false", + "model_version_uuid": uuid_version, + } + return self._pagination(url, params) + + +class TungstenUserExtractor(TungstenExtractor): + subcategory = "user" + pattern = rf"{BASE_PATTERN}/user/([^/?#]+)" + example = "https://tungsten.run/user/USER/posts" + + def posts(self): + url = f"{self.root}/user/{self.groups[0]}" + page = self.request(url).text + uuid_user = text.extr(page, '"uuid":"', '"') + + url = f"https://api.tungsten.run/v1/users/{uuid_user}/posts" + params = {"sort": "top_all_time"} + return self._pagination(url, params) diff --git a/test/results/tungsten.py b/test/results/tungsten.py new file mode 100644 index 00000000..b523698d --- /dev/null +++ b/test/results/tungsten.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import tungsten + + +__tests__ = ( +{ + "#url" : "https://tungsten.run/post/VuXCdyw9vohCAFyN", + "#class" : tungsten.TungstenPostExtractor, + "#results" : "https://api.tungsten.run/v1/upload/aPg2RYeezA9wZ8uQ", + + "comment_count" : int, + "created_at" : "2024-12-25T00:05:39Z", + "date" : "dt:2024-12-25 00:05:39", + "description" : None, + "extension" : "webp", + "filename" : "aPg2RYeezA9wZ8uQ", + "generation_postprocessing": None, + "generation_type": "flux1d", + "is_tweakable" : True, + "like_count" : range(50, 200), + "model_versions" : [{ + "cover_url" : "https://api.tungsten.run/v1/model_version_cover/48M6hN3HsQ/1726596449", + "created_at" : "2024-09-17T18:05:05Z", + "fullname" : "Flux.1 [Dev] - v1", + "model_base" : "flux1d", + "model_name" : "Flux.1 [Dev]", + "model_resolved_nsfw_score": 0, + "model_type" : "checkpoint", + "model_uuid" : "9fThn2WaVP", + "name" : "v1", + "sm_cover_url": "https://api.tungsten.run/v1/model_version_cover/48M6hN3HsQ/1726596449", + "status" : "approved", + "uuid" : "48M6hN3HsQ", + "extra" : {"dtype": "fp16"}, + }], + "nsfw" : False, + "original_url" : "https://api.tungsten.run/v1/upload/aPg2RYeezA9wZ8uQ", + "original_width" : 832, + "original_height": 1216, + "rating" : None, + "resized_url" : "https://api.tungsten.run/v1/upload/jRJMBAD5zG3NFar6", + "resized_width" : 832, + "resized_height" : 1216, + "title" : "Moon", + "uuid" : "VuXCdyw9vohCAFyN", + "view_count" : int, + "tags" : [ + "art", + "moon", + "surreal", + ], + "generation_data": { + "cfg" : 4, + "height" : 1216, + "img2img" : None, + "loras" : [], + "model_version_uuid": "48M6hN3HsQ", + "num_images" : 2, + "postprocess": [], + "prompt" : "low angle photograph of a woman, standing on top of a small mountain, hands in the air holding up the full moon late at night, long brunette hair, loose flowing spaghetti strap dress, hair and dress blowing in the wind, cler sky", + "sampler" : "euler", + "seed" : 7147106691, + "steps" : 20, + "width" : 832, + }, + "user" : { + "avatar_url" : "https://api.tungsten.run/v1/avatar/512x512/HDiix5cCtzg2VrZU/1726485265", + "badges" : list, + "bio" : "Hello, world! 😋", + "created_at" : "2024-09-16T17:18:58Z", + "display_name" : "Survivor", + "sm_avatar_url": "https://api.tungsten.run/v1/avatar/64x64/HDiix5cCtzg2VrZU/1726485265", + "type" : "admin", + "username" : "survivor", + "uuid" : "HDiix5cCtzg2VrZU", + }, +}, + +{ + "#url" : "https://tungsten.run/model/9vHB2hNUdg/chroma", + "#class" : tungsten.TungstenModelExtractor, + "#pattern" : r"https://api\.tungsten\.run/v1/upload/\w+", + "#count" : 17, + + "comment_count" : int, + "created_at" : "iso:8601", + "date" : "type:datetime", + "extension" : "webp", + "filename" : str, + "like_count" : int, + "nsfw" : bool, + "original_url" : str, + "original_width" : int, + "original_height": int, + "rating" : None, + "resized_url" : str, + "resized_width" : int, + "resized_height" : int, + "title" : str, + "uuid" : "len:str:16", + "view_count" : int, + "user" : { + "avatar_url" : str, + "badges" : list, + "bio" : {str, None}, + "created_at" : str, + "display_name" : str, + "sm_avatar_url": str, + "type" : "regular", + "username" : str, + "uuid" : "len:str:16", + }, +}, + +{ + "#url" : "https://tungsten.run/model/9vHB2hNUdg?model_version=CxvEH6esrG", + "#class" : tungsten.TungstenModelExtractor, + "#range" : "1-50", + "#pattern" : r"https://api\.tungsten\.run/v1/upload/\w+", + "#count" : 50, +}, + +{ + "#url" : "https://tungsten.run/user/lynodie", + "#class" : tungsten.TungstenUserExtractor, + "#range" : "1-50", + "#pattern" : r"https://api\.tungsten\.run/v1/upload/\w+", + "#count" : 50, + + "user" : { + "avatar_url" : "https://api.tungsten.run/v1/avatar/512x512/EYJsXQDD69u26NYT/1751499148", + "bio" : "Just a normal lesbian girl from Germany.", + "created_at" : "2024-09-16T18:01:20Z", + "display_name" : "Sarah", + "sm_avatar_url": "https://api.tungsten.run/v1/avatar/64x64/EYJsXQDD69u26NYT/1751499148", + "type" : "regular", + "username" : "lynodie", + "uuid" : "EYJsXQDD69u26NYT", + }, +}, + +)