diff --git a/docs/supportedsites.md b/docs/supportedsites.md index a33e7248..f109b260 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1081,6 +1081,12 @@ Consider all listed sites to potentially be NSFW. Galleries, User Profiles + + Yiff verse + https://yiffverse.com/ + Playlists, Posts, Tag Searches + + Zerochan https://www.zerochan.net/ diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 8d5f3d0b..d0fc1535 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -195,6 +195,7 @@ modules = [ "wikimedia", "xhamster", "xvideos", + "yiffverse", "zerochan", "zzup", "booru", diff --git a/gallery_dl/extractor/yiffverse.py b/gallery_dl/extractor/yiffverse.py new file mode 100644 index 00000000..2b143419 --- /dev/null +++ b/gallery_dl/extractor/yiffverse.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://yiffverse.com/""" + +from .booru import BooruExtractor +from .. import text +import collections + +BASE_PATTERN = r"(?:https?://)?(?:www\.)?yiffverse\.com" + + +class YiffverseExtractor(BooruExtractor): + category = "yiffverse" + root = "https://yiffverse.com" + root_cdn = "https://furry34com.b-cdn.net" + filename_fmt = "{category}_{id}.{extension}" + per_page = 30 + + TAG_TYPES = { + None: "general", + 1 : "general", + 2 : "copyright", + 4 : "character", + 8 : "artist", + } + FORMATS = ( + ("100", "mov.mp4"), + ("101", "mov720.mp4"), + ("102", "mov480.mp4"), + ("10" , "pic.jpg"), + ) + + def _file_url(self, post): + files = post["files"] + for fmt, extension in self.FORMATS: + if fmt in files: + break + else: + fmt = next(iter(files)) + + post_id = post["id"] + root = self.root_cdn if files[fmt][0] else self.root + post["file_url"] = url = "{}/posts/{}/{}/{}.{}".format( + root, post_id // 1000, post_id, post_id, extension) + post["format_id"] = fmt + post["format"] = extension.partition(".")[0] + + return url + + def _prepare(self, post): + post.pop("files", None) + post["date"] = text.parse_datetime( + post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["filename"], _, post["format"] = post["filename"].rpartition(".") + if "tags" in post: + post["tags"] = [t["value"] for t in post["tags"]] + + def _tags(self, post, _): + if "tags" not in post: + post.update(self._fetch_post(post["id"])) + + tags = collections.defaultdict(list) + for tag in post["tags"]: + tags[tag["type"]].append(tag["value"]) + types = self.TAG_TYPES + for type, values in tags.items(): + post["tags_" + types[type]] = values + + def _fetch_post(self, post_id): + url = "{}/api/v2/post/{}".format(self.root, post_id) + return self.request(url).json() + + def _pagination(self, endpoint, params=None): + url = "{}/api{}".format(self.root, endpoint) + + if params is None: + params = {} + params["sortOrder"] = 1 + params["status"] = 2 + params["take"] = self.per_page + threshold = self.per_page + + while True: + data = self.request(url, method="POST", json=params).json() + + yield from data["items"] + + if len(data["items"]) < threshold: + return + params["cursor"] = data.get("cursor") + + +class YiffversePostExtractor(YiffverseExtractor): + subcategory = "post" + archive_fmt = "{id}" + pattern = BASE_PATTERN + r"/post/(\d+)" + example = "https://yiffverse.com/post/12345" + + def posts(self): + return (self._fetch_post(self.groups[0]),) + + +class YiffversePlaylistExtractor(YiffverseExtractor): + subcategory = "playlist" + directory_fmt = ("{category}", "{playlist_id}") + archive_fmt = "p_{playlist_id}_{id}" + pattern = BASE_PATTERN + r"/playlist/(\d+)" + example = "https://yiffverse.com/playlist/12345" + + def metadata(self): + return {"playlist_id": self.groups[0]} + + def posts(self): + endpoint = "/v2/post/search/playlist/" + self.groups[0] + return self._pagination(endpoint) + + +class YiffverseTagExtractor(YiffverseExtractor): + subcategory = "tag" + directory_fmt = ("{category}", "{search_tags}") + archive_fmt = "t_{search_tags}_{id}" + pattern = BASE_PATTERN + r"/(?:tag/([^/?#]+))?(?:/?\?([^#]+))?(?:$|#)" + example = "https://yiffverse.com/tag/TAG" + + def _init(self): + tag, query = self.groups + params = text.parse_query(query) + + self.tags = tags = [] + if tag: + tags.append(text.unquote(tag)) + if "tags" in params: + tags.extend(params["tags"].split("|")) + + type = params.get("type") + if type == "video": + self.type = 1 + elif type == "image": + self.type = 0 + else: + self.type = None + + def metadata(self): + return {"search_tags": " ".join(self.tags)} + + def posts(self): + endpoint = "/v2/post/search/root" + params = {"includeTags": [t.replace("_", " ") for t in self.tags]} + if self.type is not None: + params["type"] = self.type + return self._pagination(endpoint, params) diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index bfb2e2a4..aa2cdefe 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -160,6 +160,7 @@ CATEGORY_MAP = { "xhamster" : "xHamster", "xvideos" : "XVideos", "yandere" : "yande.re", + "yiffverse" : "Yiff verse", } SUBCATEGORY_MAP = { diff --git a/test/results/yiffverse.py b/test/results/yiffverse.py new file mode 100644 index 00000000..59df47cd --- /dev/null +++ b/test/results/yiffverse.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import yiffverse + + +__tests__ = ( +{ + "#url" : "https://yiffverse.com/post/574342", + "#comment": "image", + "#class" : yiffverse.YiffversePostExtractor, + "#options" : {"tags": True}, + "#urls" : "https://yiffverse.com/posts/574/574342/574342.pic.jpg", + "#sha1_content": "0f169fddbd320eae904508f83a722bb3633ad507", + + "created" : "2024-12-06T13:55:24.483002Z", + "date" : "dt:2024-12-06 13:55:24", + "extension": "jpg", + "file_url" : "https://yiffverse.com/posts/574/574342/574342.pic.jpg", + "filename" : "574342", + "format" : "pic", + "format_id": "10", + "height" : 862, + "id" : 574342, + "likes" : range(5, 100), + "posted" : "2024-12-06T13:55:55.299953Z", + "status" : 2, + "type" : 0, + "tags" : list, + "tags_general": list, + "tags_artist" : ["imanika"], + "uploaderId" : 2, + "width" : 950, + "data" : { + "sources": [ + "https://www.furaffinity.net/view/59071676/", + "https://www.furaffinity.net/user/imanika/", + "https://d.furaffinity.net/art/imanika/1733430246/1733430246.imanika_dream_girl_ych_slot1_web.jpg", + ], + }, + "uploader" : { + "created" : "2021-07-04T15:01:03.110916Z", + "data" : None, + "displayName" : "agent.e621-uploader", + "emailVerified": False, + "id" : 2, + "role" : 3, + "userName" : "agent.e621-uploader", + }, +}, + +{ + "#url" : "https://yiffverse.com/post/575680", + "#comment": "video", + "#class" : yiffverse.YiffversePostExtractor, + "#urls" : "https://yiffverse.com/posts/575/575680/575680.mov.mp4", + "#sha1_content": "8952fc794e58c531b4e3b01cfe9e14b1c59ad9ef", +}, + +{ + "#url" : "https://yiffverse.com/tag/tree", + "#class": yiffverse.YiffverseTagExtractor, + "#pattern": r"https://yiffverse\.com/posts/\d+/\d+/\d+\.(pic\.jpg|mov\d*\.mp4)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://yiffverse.com/playlist/6842", + "#class": yiffverse.YiffversePlaylistExtractor, + "#pattern": r"https://(yiffverse\.com|furry34com\.b-cdn\.net)/posts/\d+/\d+/\d+\.mov(720)?\.mp4", + "#count" : 25, +}, + +)