From 3d0263b3abf09555fdd60d278e2fc84c896a91b6 Mon Sep 17 00:00:00 2001 From: ssdaniel24 Date: Thu, 26 Sep 2024 22:13:42 +0300 Subject: [PATCH 1/2] [rule34vault] Added initial support for rule34vault.com - Added playlists support for rule34vault - Added support for posts in rule34vault - Fixed supported sites with script - Fixed posts pattern in rule34vault - Added tests for rule34vault - Clean - Fixed lint warnings --- docs/supportedsites.md | 6 ++ gallery_dl/extractor/__init__.py | 1 + gallery_dl/extractor/rule34vault.py | 130 ++++++++++++++++++++++++++++ scripts/supportedsites.py | 1 + test/results/rule34vault.py | 38 ++++++++ 5 files changed, 176 insertions(+) create mode 100644 gallery_dl/extractor/rule34vault.py create mode 100644 test/results/rule34vault.py diff --git a/docs/supportedsites.md b/docs/supportedsites.md index f3699147..c4f56094 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -757,6 +757,12 @@ Consider all listed sites to potentially be NSFW. Galleries, individual Images + + R34 Vault + https://rule34vault.com/ + Playlists, Posts, Tag Searches + + Read Comic Online https://readcomiconline.li/ diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 98851957..99427d87 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -139,6 +139,7 @@ modules = [ "reddit", "redgifs", "rule34us", + "rule34vault", "sankaku", "sankakucomplex", "seiga", diff --git a/gallery_dl/extractor/rule34vault.py b/gallery_dl/extractor/rule34vault.py new file mode 100644 index 00000000..df8404ea --- /dev/null +++ b/gallery_dl/extractor/rule34vault.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://rule34vault.com/""" + +from .booru import BooruExtractor +from .. import text + + +class Rule34vaultExtractor(BooruExtractor): + category = "rule34vault" + root = "https://rule34vault.com" + cdn_root = "https://r34xyz.b-cdn.net" + filename_fmt = "{category}_{id}.{extension}" + per_page = 100 + + def _get_file_url(self, post_id, filetype): + extension = "jpg" if filetype == "image" else "mp4" + return "{}/posts/{}/{}/{}.{}".format(self.cdn_root, + post_id // 1000, + post_id, + post_id, + extension) + + def _parse_post(self, post_id): + url = "{}/api/v2/post/{}".format(self.root, post_id) + data = self.request(url).json() + + tags = " ".join(t["value"].replace(" ", "_") for t in data["tags"]) + post = { + "id": post_id, + "tags": tags, + "uploader": data["uploader"]["userName"], + "score": data.get("likes") or 0, + "width": data["width"], + "height": data["height"], + } + post["file_url"] = self._get_file_url(post_id, + "image" if data['type'] == 0 + else "video") + + return post + + +class Rule34vaultPostExtractor(Rule34vaultExtractor): + subcategory = "post" + archive_fmt = "{id}" + pattern = r"(?:https?://)?rule34vault\.com/post/(\d+)" + example = "https://rule34vault.com/post/399437" + + def __init__(self, match): + Rule34vaultExtractor.__init__(self, match) + self.post_id = int(match.group(1)) + + def posts(self): + return (self._parse_post(self.post_id),) + + +class Rule34vaultPlaylistExtractor(Rule34vaultExtractor): + subcategory = "playlist" + directory_fmt = ("{category}", "{playlist_id}") + archive_fmt = "t_{playlist_id}_{id}" + pattern = r"(?:https?://)?rule34vault\.com/playlists/view/(\d+)" + example = "https://rule34vault.com/playlists/view/2" + + def __init__(self, match): + Rule34vaultExtractor.__init__(self, match) + self.playlist_id = match.group(1) + + def metadata(self): + return {"playlist_id": self.playlist_id} + + def posts(self): + url = "{}/api/v2/post/search/playlist/{}".format(self.root, + self.playlist_id) + current_page = self.page_start + + while True: + payload = { + "CountTotal": True, + "Skip": current_page * self.per_page, + "take": self.per_page, + } + data = self.request(url, method="POST", json=payload).json() + + for post in data["items"]: + yield self._parse_post(post["id"]) + + if current_page * self.per_page > data["totalCount"]: + return + current_page += 1 + + +class Rule34vaultTagExtractor(Rule34vaultExtractor): + subcategory = "tag" + directory_fmt = ("{category}", "{search_tags}") + archive_fmt = "t_{search_tags}_{id}" + pattern = r"(?:https?://)?rule34vault\.com/([^/?#]+)$" + example = "https://rule34vault.com/not_porn" + + def __init__(self, match): + Rule34vaultExtractor.__init__(self, match) + self.tags_ = text.unquote(match.group(1)).split("%7C") + self.tags = [t.replace("_", " ") for t in self.tags_] + + def metadata(self): + return {"search_tags": " ".join(self.tags_)} + + def posts(self): + url = '{}/api/v2/post/search/root'.format(self.root) + current_page = self.page_start + + while True: + payload = { + "CountTotal": True, + "Skip": current_page * self.per_page, + "take": self.per_page, + "includeTags": self.tags, + } + data = self.request(url, method="POST", json=payload).json() + + for post in data["items"]: + yield self._parse_post(post["id"]) + + if current_page * self.per_page > data['totalCount']: + return + current_page += 1 diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 1e565a9a..6e37dd59 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -119,6 +119,7 @@ CATEGORY_MAP = { "rule34" : "Rule 34", "rule34hentai" : "Rule34Hentai", "rule34us" : "Rule 34", + "rule34vault" : "R34 Vault", "sankaku" : "Sankaku Channel", "sankakucomplex" : "Sankaku Complex", "seiga" : "Niconico Seiga", diff --git a/test/results/rule34vault.py b/test/results/rule34vault.py new file mode 100644 index 00000000..81980dac --- /dev/null +++ b/test/results/rule34vault.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import rule34vault + + +__tests__ = ( + { + "#url" : "https://rule34vault.com/sfw", + "#category": ("booru", "rule34vault", "tag"), + "#class" : rule34vault.Rule34vaultTagExtractor, + "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + "#count" : 10, + }, + { + "#url" : "https://rule34vault.com/post/486545", + "#category": ("booru", "rule34vault", "post"), + "#class" : rule34vault.Rule34vaultPostExtractor, + "#pattern" : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg", + "#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07", + }, + { + "#url" : "https://rule34vault.com/post/382937", + "#category": ("booru", "rule34vault", "post"), + "#class" : rule34vault.Rule34vaultPostExtractor, + "#pattern" : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4", + "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af", + }, + { + "#url" : "https://rule34vault.com/playlists/view/20164", + "#category": ("booru", "rule34vault", "playlist"), + "#class" : rule34vault.Rule34vaultPlaylistExtractor, + "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + }, +) From a974c30057b73f1889ee60c38e2e2e8ee47633e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 28 Oct 2024 21:58:54 +0100 Subject: [PATCH 2/2] [rule34vault] update - use more of BooruExtractor's interface - use BASE_PATTERN - define generic pagination method - parse 'date' metadata - make tags & extended metadata optional --- gallery_dl/extractor/rule34vault.py | 133 +++++++++++----------------- test/results/rule34vault.py | 57 ++++++------ 2 files changed, 83 insertions(+), 107 deletions(-) diff --git a/gallery_dl/extractor/rule34vault.py b/gallery_dl/extractor/rule34vault.py index df8404ea..de2c0bf6 100644 --- a/gallery_dl/extractor/rule34vault.py +++ b/gallery_dl/extractor/rule34vault.py @@ -9,122 +9,95 @@ from .booru import BooruExtractor from .. import text +BASE_PATTERN = r"(?:https?://)?rule34vault\.com" + class Rule34vaultExtractor(BooruExtractor): category = "rule34vault" root = "https://rule34vault.com" - cdn_root = "https://r34xyz.b-cdn.net" + root_cdn = "https://r34xyz.b-cdn.net" filename_fmt = "{category}_{id}.{extension}" per_page = 100 - def _get_file_url(self, post_id, filetype): - extension = "jpg" if filetype == "image" else "mp4" - return "{}/posts/{}/{}/{}.{}".format(self.cdn_root, - post_id // 1000, - post_id, - post_id, - extension) + def _file_url(self, post): + post_id = post["id"] + extension = "jpg" if post["type"] == 0 else "mp4" + return "{}/posts/{}/{}/{}.{}".format( + self.root_cdn, post_id // 1000, post_id, post_id, extension) - def _parse_post(self, post_id): + def _prepare(self, post): + post.pop("files", None) + post["date"] = text.parse_datetime( + post["created"], "%Y-%m-%dT%H:%M:%S.%fZ") + if "tags" in post: + post["tags"] = [t["value"] for t in post["tags"]] + + def _tags(self, post, _): + if "tags" not in post: + post.update(self._fetch_post(post["id"])) + + def _fetch_post(self, post_id): url = "{}/api/v2/post/{}".format(self.root, post_id) - data = self.request(url).json() + return self.request(url).json() - tags = " ".join(t["value"].replace(" ", "_") for t in data["tags"]) - post = { - "id": post_id, - "tags": tags, - "uploader": data["uploader"]["userName"], - "score": data.get("likes") or 0, - "width": data["width"], - "height": data["height"], - } - post["file_url"] = self._get_file_url(post_id, - "image" if data['type'] == 0 - else "video") + def _pagination(self, endpoint, params=None): + url = "{}/api{}".format(self.root, endpoint) - return post + if params is None: + params = {} + params["CountTotal"] = True + params["Skip"] = self.page_start * self.per_page + params["take"] = self.per_page + + while True: + data = self.request(url, method="POST", json=params).json() + + yield from data["items"] + + if params["Skip"] + params["take"] > data["totalCount"]: + return + if "cursor" in data: + params["cursor"] = data["cursor"] + params["Skip"] += params["take"] class Rule34vaultPostExtractor(Rule34vaultExtractor): subcategory = "post" archive_fmt = "{id}" - pattern = r"(?:https?://)?rule34vault\.com/post/(\d+)" + pattern = BASE_PATTERN + r"/post/(\d+)" example = "https://rule34vault.com/post/399437" - def __init__(self, match): - Rule34vaultExtractor.__init__(self, match) - self.post_id = int(match.group(1)) - def posts(self): - return (self._parse_post(self.post_id),) + return (self._fetch_post(self.groups[0]),) class Rule34vaultPlaylistExtractor(Rule34vaultExtractor): subcategory = "playlist" directory_fmt = ("{category}", "{playlist_id}") - archive_fmt = "t_{playlist_id}_{id}" - pattern = r"(?:https?://)?rule34vault\.com/playlists/view/(\d+)" + archive_fmt = "p_{playlist_id}_{id}" + pattern = BASE_PATTERN + r"/playlists/view/(\d+)" example = "https://rule34vault.com/playlists/view/2" - def __init__(self, match): - Rule34vaultExtractor.__init__(self, match) - self.playlist_id = match.group(1) - def metadata(self): - return {"playlist_id": self.playlist_id} + return {"playlist_id": self.groups[0]} def posts(self): - url = "{}/api/v2/post/search/playlist/{}".format(self.root, - self.playlist_id) - current_page = self.page_start - - while True: - payload = { - "CountTotal": True, - "Skip": current_page * self.per_page, - "take": self.per_page, - } - data = self.request(url, method="POST", json=payload).json() - - for post in data["items"]: - yield self._parse_post(post["id"]) - - if current_page * self.per_page > data["totalCount"]: - return - current_page += 1 + endpoint = "/v2/post/search/playlist/" + self.groups[0] + return self._pagination(endpoint) class Rule34vaultTagExtractor(Rule34vaultExtractor): subcategory = "tag" directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" - pattern = r"(?:https?://)?rule34vault\.com/([^/?#]+)$" - example = "https://rule34vault.com/not_porn" - - def __init__(self, match): - Rule34vaultExtractor.__init__(self, match) - self.tags_ = text.unquote(match.group(1)).split("%7C") - self.tags = [t.replace("_", " ") for t in self.tags_] + pattern = BASE_PATTERN + r"/([^/?#]+)$" + example = "https://rule34vault.com/TAG" def metadata(self): - return {"search_tags": " ".join(self.tags_)} + self.tags = text.unquote(self.groups[0]).split("%7C") + return {"search_tags": " ".join(self.tags)} def posts(self): - url = '{}/api/v2/post/search/root'.format(self.root) - current_page = self.page_start - - while True: - payload = { - "CountTotal": True, - "Skip": current_page * self.per_page, - "take": self.per_page, - "includeTags": self.tags, - } - data = self.request(url, method="POST", json=payload).json() - - for post in data["items"]: - yield self._parse_post(post["id"]) - - if current_page * self.per_page > data['totalCount']: - return - current_page += 1 + endpoint = "/v2/post/search/root" + params = {"includeTags": [t.replace("_", " ") for t in self.tags]} + return self._pagination(endpoint, params) diff --git a/test/results/rule34vault.py b/test/results/rule34vault.py index 81980dac..2fe31a1f 100644 --- a/test/results/rule34vault.py +++ b/test/results/rule34vault.py @@ -8,31 +8,34 @@ from gallery_dl.extractor import rule34vault __tests__ = ( - { - "#url" : "https://rule34vault.com/sfw", - "#category": ("booru", "rule34vault", "tag"), - "#class" : rule34vault.Rule34vaultTagExtractor, - "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", - "#count" : 10, - }, - { - "#url" : "https://rule34vault.com/post/486545", - "#category": ("booru", "rule34vault", "post"), - "#class" : rule34vault.Rule34vaultPostExtractor, - "#pattern" : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg", - "#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07", - }, - { - "#url" : "https://rule34vault.com/post/382937", - "#category": ("booru", "rule34vault", "post"), - "#class" : rule34vault.Rule34vaultPostExtractor, - "#pattern" : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4", - "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af", - }, - { - "#url" : "https://rule34vault.com/playlists/view/20164", - "#category": ("booru", "rule34vault", "playlist"), - "#class" : rule34vault.Rule34vaultPlaylistExtractor, - "#pattern" : r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", - }, +{ + "#url" : "https://rule34vault.com/sfw", + "#class": rule34vault.Rule34vaultTagExtractor, + "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + "#range" : "1-10", + "#count" : 10, +}, + +{ + "#url" : "https://rule34vault.com/post/486545", + "#class": rule34vault.Rule34vaultPostExtractor, + "#pattern" : r"https://r34xyz\.b-cdn.net/posts/486/486545/486545\.jpg", + "#sha1_content": "8f53c4c9d049842d23b51fb3cf8ce11bcbe21f07", +}, + +{ + "#url" : "https://rule34vault.com/post/382937", + "#comment": "video", + "#class" : rule34vault.Rule34vaultPostExtractor, + "#pattern" : r"https://r34xyz\.b-cdn.net/posts/382/382937/382937\.mp4", + "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af", +}, + +{ + "#url" : "https://rule34vault.com/playlists/view/20164", + "#class": rule34vault.Rule34vaultPlaylistExtractor, + "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + "#count" : 55, +}, + )