104 lines
3.2 KiB
Python
104 lines
3.2 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
"""Extractors for https://tungsten.run/"""
|
|
|
|
from .common import Extractor, Message
|
|
from .. import text
|
|
|
|
BASE_PATTERN = r"(?:https?://)?(?:www\.)?tungsten\.run"
|
|
|
|
|
|
class TungstenExtractor(Extractor):
|
|
"""Base class for tungsten extractors"""
|
|
category = "tungsten"
|
|
root = "https://tungsten.run"
|
|
directory_fmt = ("{category}", "{user[username]}")
|
|
filename_fmt = "{date} {title:?/ /}{uuid}.{extension}"
|
|
archive_fmt = "{uuid}"
|
|
|
|
def items(self):
|
|
for post in self.posts():
|
|
url = post["original_url"]
|
|
post["date"] = self.parse_datetime_iso(post["created_at"])
|
|
post["filename"] = url[url.rfind("/")+1:]
|
|
post["extension"] = "webp"
|
|
yield Message.Directory, "", post
|
|
yield Message.Url, url, post
|
|
|
|
def _pagination(self, url, params):
|
|
params["page"] = 1
|
|
params["per_page"] = 40
|
|
|
|
headers = {
|
|
"Origin": self.root,
|
|
"Sec-Fetch-Dest": "empty",
|
|
"Sec-Fetch-Mode": "cors",
|
|
"Sec-Fetch-Site": "same-site",
|
|
}
|
|
|
|
while True:
|
|
data = self.request_json(url, params=params, headers=headers)
|
|
|
|
yield from data
|
|
|
|
if len(data) < params["per_page"]:
|
|
break
|
|
params["page"] += 1
|
|
|
|
|
|
class TungstenPostExtractor(TungstenExtractor):
|
|
subcategory = "post"
|
|
pattern = BASE_PATTERN + r"/post/(\w+)"
|
|
example = "https://tungsten.run/post/AbCdEfGhIjKlMnOp"
|
|
|
|
def posts(self):
|
|
url = f"{self.root}/post/{self.groups[0]}"
|
|
page = self.request(url).text
|
|
data = self._extract_nextdata(page)
|
|
return (data["props"]["pageProps"]["post"],)
|
|
|
|
|
|
class TungstenModelExtractor(TungstenExtractor):
|
|
subcategory = "model"
|
|
pattern = BASE_PATTERN + r"/model/(\w+)(?:/?\?model_version=(\w+))?"
|
|
example = "https://tungsten.run/model/AbCdEfGhIjKlM"
|
|
|
|
def posts(self):
|
|
uuid_model, uuid_version = self.groups
|
|
|
|
if uuid_version is None:
|
|
url = f"{self.root}/model/{uuid_model}/"
|
|
page = self.request(url).text
|
|
uuid_version = text.extr(page, '"modelVersionUUID":"', '"')
|
|
|
|
url = "https://api.tungsten.run/v1/posts"
|
|
params = {
|
|
"sort" : "top_all_time",
|
|
"tweakable_only": "false",
|
|
"following" : "false",
|
|
"model_version_uuid": uuid_version,
|
|
}
|
|
return self._pagination(url, params)
|
|
|
|
|
|
class TungstenUserExtractor(TungstenExtractor):
|
|
subcategory = "user"
|
|
pattern = BASE_PATTERN + r"/user/([^/?#]+)(?:/posts)?/?(?:\?([^#]+))?"
|
|
example = "https://tungsten.run/user/USER"
|
|
|
|
def posts(self):
|
|
user, qs = self.groups
|
|
url = f"{self.root}/user/{user}"
|
|
page = self.request(url).text
|
|
uuid_user = text.extr(page, '"user":{"uuid":"', '"')
|
|
|
|
url = f"https://api.tungsten.run/v1/users/{uuid_user}/posts"
|
|
params = text.parse_query(qs)
|
|
params.setdefault("sort", "top_all_time")
|
|
self.kwdict["search_tags"] = params.get("tag", "")
|
|
return self._pagination(url, params)
|