diff --git a/docs/configuration.rst b/docs/configuration.rst index 00fe66b5..e4ab4582 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -9260,6 +9260,7 @@ Special Values { "coomer" : "coomerparty", "kemono" : "kemonoparty", + "turbo" : "saint", "schalenetwork": "koharu", "naver-chzzk" : "chzzk", "naver-blog" : "naver", @@ -9287,7 +9288,8 @@ Default "chzzk" : "naver-chzzk", "naver" : "naver-blog", "naverwebtoon" : "naver-webtoon", - "pixiv" : "pixiv-novel" + "pixiv" : "pixiv-novel", + "saint" : "turbo" } Description Duplicate the configuration settings of extractor `categories` diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 4254a5aa..cdb877f3 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -102,7 +102,8 @@ "chzzk" : "naver-chzzk", "naver" : "naver-blog", "naverwebtoon" : "naver-webtoon", - "pixiv" : "pixiv-novel" + "pixiv" : "pixiv-novel", + "saint" : "turbo" }, diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 09062b71..308c9cc7 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -949,12 +949,6 @@ Consider all listed sites to potentially be NSFW. Posts, Search Results, User Profiles - - Saint - https://saint2.su/ - Albums, Media Files - - Sankaku Channel https://sankaku.app/ @@ -1135,9 +1129,9 @@ Consider all listed sites to potentially be NSFW. Models, Posts, User Profiles - - turbovid.cr - https://turbovid.cr/ + + turbo.cr + https://turbo.cr/ Albums, Media Files diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index b28b5896..7c8c0f02 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -321,6 +321,7 @@ def main(): catmap = { "coomer" : "coomerparty", "kemono" : "kemonoparty", + "turbo" : "saint", "schalenetwork": "koharu", "naver-blog" : "naver", "naver-chzzk" : "chzzk", diff --git a/gallery_dl/config.py b/gallery_dl/config.py index 33a3b958..282587ec 100644 --- a/gallery_dl/config.py +++ b/gallery_dl/config.py @@ -175,6 +175,7 @@ def remap_categories(): ("chzzk" , "naver-chzzk"), ("naverwebtoon", "naver-webtoon"), ("pixiv" , "pixiv-novel"), + ("saint" , "turbo"), ) elif not cmap: return diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index e747c5d9..97a5089f 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -181,7 +181,6 @@ modules = [ "rule34vault", "rule34xyz", "s3ndpics", - "saint", "sankaku", "sankakucomplex", "schalenetwork", @@ -215,6 +214,7 @@ modules = [ "tumblr", "tumblrgallery", "tungsten", + "turbo", "twibooru", "twitter", "urlgalleries", diff --git a/gallery_dl/extractor/saint.py b/gallery_dl/extractor/saint.py deleted file mode 100644 index 3693f122..00000000 --- a/gallery_dl/extractor/saint.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2024-2026 Mike Fährmann -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -"""Extractors for https://saint2.su/ and https://turbovid.cr/""" - -from .lolisafe import LolisafeAlbumExtractor -from .. import text - -BASE_PATTERN = r"(?:https?://)?(?:turbo(?:vid)?\.cr|saint\d*\.(?:su|pk|cr|to))" - - -class SaintAlbumExtractor(LolisafeAlbumExtractor): - """Extractor for saint albums""" - category = "saint" - root = "https://saint2.su" - pattern = BASE_PATTERN + r"/a/([^/?#]+)" - example = "https://saint2.su/a/ID" - - def fetch_album(self, album_id): - # album metadata - response = self.request(self.root + "/a/" + album_id) - extr = text.extract_from(response.text) - - title = extr("", "</title") - descr = extr('name="description" content="', '"/>') - files = [] - - while True: - id2 = extr("/thumbs/", '"') - if not id2: - break - - id2, sep, ts = id2.rpartition(".")[0].rpartition("-") - if sep: - date = self.parse_timestamp(ts) - else: - date = None - id2 = ts - - files.append({ - "id" : extr("/embed/", '"'), - "id2" : id2, - "date" : date, - # "extension": extr("<td>", "</"), - "size" : text.parse_int(extr('data="', '"')), - "file" : text.unescape(extr( - "onclick=\"play(", ")").strip("\"'")), - "id_dl": extr("/d/", ")").rstrip("\"'"), - }) - - return files, { - "album_id" : album_id, - "album_name" : text.unescape(title.rpartition(" - ")[0]), - "album_size" : sum(file["size"] for file in files), - "description" : text.unescape(descr), - "count" : len(files), - "_http_headers": {"Referer": response.url} - } - - -class SaintMediaExtractor(SaintAlbumExtractor): - """Extractor for saint media links""" - subcategory = "media" - directory_fmt = ("{category}",) - pattern = BASE_PATTERN + r"(/(embe)?d/([^/?#]+))" - example = "https://saint2.su/embed/ID" - - def fetch_album(self, album_id): - try: - path, embed, _ = self.groups - - url = self.root + path - response = self.request(url) - extr = text.extract_from(response.text) - - if embed: - id2, sep, ts = extr( - "/thumbs/", '"').rpartition(".")[0].rpartition("-") - if sep: - date = self.parse_timestamp(ts) - else: - date = None - id2 = ts - - file = { - "id" : album_id, - "id2" : id2, - "date" : date, - "file" : text.unescape(extr('<source src="', '"')), - "id_dl": extr("/d/", "'"), - } - - else: # /d/ - file = { - "file" : text.unescape(extr('<a href="', '"')), - "id" : album_id, - "id_dl" : album_id, - "name" : album_id, - "filename" : album_id, - "extension": "mp4", - } - - file["_http_headers"] = {"Referer": response.url} - except Exception as exc: - self.log.error("%s: %s", exc.__class__.__name__, exc) - return (), {} - - return (file,), { - "album_id" : "", - "album_name" : "", - "album_size" : -1, - "description": "", - "count" : 1, - } diff --git a/gallery_dl/extractor/turbo.py b/gallery_dl/extractor/turbo.py new file mode 100644 index 00000000..a20076b8 --- /dev/null +++ b/gallery_dl/extractor/turbo.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024-2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Extractors for https://turbo.cr/""" + +from .lolisafe import LolisafeAlbumExtractor +from .. import text + +BASE_PATTERN = (r"(?:https?://)?(?:" + r"(?:www\.)?turbo(?:vid)?\.cr|" + r"saint\d*\.(?:su|pk|cr|to))") + + +class TurboAlbumExtractor(LolisafeAlbumExtractor): + """Extractor for turbo.cr albums""" + category = "turbo" + root = "https://turbo.cr" + pattern = BASE_PATTERN + r"/a/([^/?#]+)" + example = "https://turbo.cr/a/ID" + + def fetch_album(self, album_id): + url = f"{self.root}/a/{album_id}" + extr = text.extract_from(self.request(url).text) + title = extr("<h1 ", "<") + descr = extr("<p ", "<") + tbody = extr('id="fileTbody"', '</tbody>') + headers = {"Referer": url} + + return self._extract_files(tbody, headers), { + "album_id" : album_id, + "album_name" : text.unescape(title[title.find(">")+1:]), + "description" : text.unescape(descr[descr.find(">")+1:]), + "album_size" : sum(map(text.parse_int, text.extract_iter( + tbody, 'data-size="', '"'))), + "count" : tbody.count("data-id="), + "_http_headers": headers, + } + + def _extract_files(self, body, headers): + for file in text.extract_iter(body, "<tr", "</tr>"): + data_id = text.extr(file, 'data-id="', '"') + url = f"{self.root}/api/sign?v={data_id}" + data = self.request_json(url, headers=headers) + name = data.get("original_filename") or data.get("filename") + yield text.nameext_from_name(name, { + "id" : data_id, + "file": data.get("url"), + "size": text.parse_int(text.extr(file, 'data-size="', '"')), + "_http_headers": headers, + }) + + +class TurboMediaExtractor(TurboAlbumExtractor): + """Extractor for turbo.cr media links""" + subcategory = "media" + directory_fmt = ("{category}",) + pattern = BASE_PATTERN + r"/(?:embe)?[dv]/([^/?#]+)" + example = "https://turbo.cr/embed/ID" + + def fetch_album(self, album_id): + try: + return (self._extract_file(album_id),), { + "album_id" : "", + "album_name" : "", + "album_size" : -1, + "description": "", + "count" : 1, + } + except Exception as exc: + self.log.error("%s: %s", exc.__class__.__name__, exc) + return (), {} + + def _extract_file(self, data_id): + url = f"{self.root}/d/{data_id}" + headers = {"Referer": url} + page = self.request(url).text + size = text.extr(page, 'id="fileSizeBytes">', '<') + date = text.extract(page, "<span>", "<", page.find("File ID:"))[0] + + url = f"{self.root}/api/sign?v={data_id}" + data = self.request_json(url, headers=headers) + name = data.get("original_filename") or data.get("filename") + return text.nameext_from_name(name, { + "id" : data_id, + "file": data.get("url"), + "size": int(text.parse_float(size.replace("+", "+"))), + "date": self.parse_datetime_iso(date), + "_http_headers": headers, + }) diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 8bc8ed7f..d52afb57 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -200,7 +200,7 @@ CATEGORY_MAP = { "tmohentai" : "TMOHentai", "tumblrgallery" : "TumblrGallery", "turboimagehost" : "TurboImageHost.com", - "turbovid" : "turbovid.cr", + "turbo" : "turbo.cr", "vanillarock" : "もえぴりあ", "vidyart2" : "/v/idyart2", "vidyapics" : "Vidya Booru", @@ -709,11 +709,6 @@ def build_extractor_list(): default["wikifeetx"] = default["wikifeet"] domains["wikifeetx"] = "https://www.wikifeetx.com/" - # turbovid - default["turbovid"] = default["saint"] - domains["turbovid"] = "https://turbovid.cr/" - domains["saint"] = "https://saint2.su/" - # imgdrive / imgtaxi / imgwallet base = categories["imagehost"] base["imgtaxi"] = base["imgdrive"] diff --git a/test/results/saint.py b/test/results/saint.py deleted file mode 100644 index 88c669cd..00000000 --- a/test/results/saint.py +++ /dev/null @@ -1,145 +0,0 @@ -# -*- coding: utf-8 -*- - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. - -from gallery_dl.extractor import saint - - -__tests__ = ( -{ - "#url" : "https://saint2.su/a/2c5iuWHTumH", - "#class": saint.SaintAlbumExtractor, - "#results": ( - "https://data.saint2.cr/data/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", - "https://data.saint2.cr/data/3b125e3fb4b98693f17d85cb53590215.mp4", - ), - - "album_id" : "2c5iuWHTumH", - "album_name" : "animations", - "album_size" : 37083862, - "count" : 2, - "date" : "type:datetime", - "description": "Descriptions can contain only alphanumeric ASCII characters", - "extension" : "mp4", - "file" : r"re:https://...", - "filename" : {"3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", - "3b125e3fb4b98693f17d85cb53590215-ze10Ohbpoy5"}, - "id" : {"6lC7mKrJst8", - "ze10Ohbpoy5"}, - "id2" : {"6712834015d67", - "671284a627e0e"}, - "id_dl" : {"M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", - "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0"}, - "name" : {"3b1ccebf3576f8d5aac3ee0e5a12da95", - "3b125e3fb4b98693f17d85cb53590215"}, - "num" : {1, 2}, -}, - -{ - "#url" : "https://turbovid.cr/a/FiphGijfJoR", - "#comment" : "'turbovid' album (#8851)", - "#category": ("lolisafe", "saint", "album"), - "#class" : saint.SaintAlbumExtractor, - "#results" : ( - "https://data.saint2.cr/data/jZqe1xxqw9bX7.mp4", - "https://data.saint2.cr/data/eJ9fLurGdaHqS.mp4", - "https://data.saint2.cr/data/WkD7hRaHdBpBI.mp4", - ), - - "album_id" : "FiphGijfJoR", - "album_name" : """test-???-"&> album""", - "album_size" : 37165256, - "count" : 3, - "num" : range(1, 3), - "date" : None, - "description": """test-???-"&> description""", - "extension" : "mp4", - "file" : r"re:https://data.saint2.cr/data/\w+.mp4", - "filename" : str, - "id" : str, - "id2" : str, - "id_dl" : str, - "name" : str, - "size" : int, -}, - -{ - "#url" : "https://turbo.cr/a/FiphGijfJoR", - "#comment" : "'turbo' album (#8888)", - "#class" : saint.SaintAlbumExtractor, -}, - -{ - "#url" : "https://saint2.su/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, - "#results" : "https://data.saint2.cr/data/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", - "#sha1_content": "39037a029b3fe96f838b4545316caaa545c84075", - - "count" : 1, - "date" : "dt:2024-10-18 15:48:16", - "extension": "mp4", - "file" : str, - "filename" : "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", - "id" : "6lC7mKrJst8", - "id2" : "6712834015d67", - "id_dl" : "M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", - "name" : "3b1ccebf3576f8d5aac3ee0e5a12da95", - "num" : 1, -}, - -{ - "#url" : "https://saint2.su/d/M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "#class": saint.SaintMediaExtractor, - "#results": "https://data.saint2.cr/data/3b125e3fb4b98693f17d85cb53590215.mp4", - - "count" : 1, - "extension": "mp4", - "file" : str, - "filename" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "id" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "id_dl" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "name" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "num" : 1, -}, - -{ - "#url" : "https://saint2.pk/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, -}, - -{ - "#url" : "https://saint2.cr/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, -}, - -{ - "#url" : "https://saint.to/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, -}, - -{ - "#url" : "https://turbovid.cr/embed/WkD7hRaHdBpBI", - "#comment" : "'turbovid' URL/video", - "#category": ("lolisafe", "saint", "media"), - "#class" : saint.SaintMediaExtractor, - "#results" : "https://data.saint2.cr/data/WkD7hRaHdBpBI.mp4", - - "date" : None, - "extension" : "mp4", - "file" : "https://data.saint2.cr/data/WkD7hRaHdBpBI.mp4", - "filename" : "WkD7hRaHdBpBI", - "id" : "WkD7hRaHdBpBI", - "id2" : "WkD7hRaHdBpBI", - "id_dl" : "V2tEN2hSYUhkQnBCSS5tcDQ=", - "name" : "WkD7hRaHdBpBI", -}, - -{ - "#url" : "https://turbo.cr/embed/WkD7hRaHdBpBI", - "#comment" : "'turbo' URL/video", - "#class" : saint.SaintMediaExtractor, -}, - -) diff --git a/test/results/turbo.py b/test/results/turbo.py new file mode 100644 index 00000000..d27b8c8b --- /dev/null +++ b/test/results/turbo.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +from gallery_dl.extractor import turbo + + +__tests__ = ( +{ + "#url" : "https://turbo.cr/a/2c5iuWHTumH", + "#class": turbo.TurboAlbumExtractor, + "#pattern": ( + r"https://dl\d+.turbocdn.st/data/3b125e3fb4b98693f17d85cb53590215.mp4\?exp=\d+&token=\w+&fn=3b125e3fb4b98693f17d85cb53590215.mp4", + r"https://dl\d+.turbocdn.st/data/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4\?exp=\d+&token=\w+&fn=3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", + ), + + "album_id" : "2c5iuWHTumH", + "album_name" : "animations", + "album_size" : 37083862, + "count" : 2, + "description": "Descriptions can contain only alphanumeric ASCII characters", + "extension" : "mp4", + "file" : r"re:https://...", + "filename" : {"3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", + "3b125e3fb4b98693f17d85cb53590215-ze10Ohbpoy5"}, + "id" : {"6lC7mKrJst8", + "ze10Ohbpoy5"}, + "name" : {"3b1ccebf3576f8d5aac3ee0e5a12da95", + "3b125e3fb4b98693f17d85cb53590215"}, + "num" : {1, 2}, +}, + +{ + "#url" : "https://turbovid.cr/a/FiphGijfJoR", + "#comment" : "'turbovid' album (#8851)", + "#category": ("lolisafe", "turbo", "album"), + "#class" : turbo.TurboAlbumExtractor, + "#pattern" : ( + r"https://dl\d+.turbocdn.st/data/WkD7hRaHdBpBI.mp4\?exp=\d+&token=\w+&fn=3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8.mp4", + r"https://dl\d+.turbocdn.st/data/eJ9fLurGdaHqS.mp4\?exp=\d+&token=\w+&fn=3b125e3fb4b98693f17d85cb53590215-ze10Ohbpoy5.mp4", + r"https://dl\d+.turbocdn.st/data/jZqe1xxqw9bX7.mp4\?exp=\d+&token=\w+&fn=test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26%3E.mp4", + ), + + "album_id" : "FiphGijfJoR", + "album_name" : """test-テスト-"&> album""", + "album_size" : 37165256, + "count" : 3, + "num" : range(1, 3), + "description": """test-テスト-"&> description""", + "extension" : "mp4", + "file" : r"re:https://dl\d+.turbocdn.st/data/.+", + "filename" : str, + "id" : str, + "name" : str, + "size" : int, +}, + +{ + "#url" : "https://saint2.su/a/FiphGijfJoR", + "#comment" : "'saint' album (#8888)", + "#class" : turbo.TurboAlbumExtractor, +}, + +{ + "#url" : "https://turbo.cr/embed/6lC7mKrJst8", + "#class": turbo.TurboMediaExtractor, + "#pattern" : r"https://dl\d+.turbocdn.st/data/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", + "#sha1_content": "39037a029b3fe96f838b4545316caaa545c84075", + + "count" : 1, + "date" : "dt:2024-10-18 00:00:00", + "extension": "mp4", + "file" : str, + "filename" : "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", + "id" : "6lC7mKrJst8", + "name" : "3b1ccebf3576f8d5aac3ee0e5a12da95", + "num" : 1, +}, + +{ + "#url" : "https://turbo.cr/d/M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "#comment": "'Page not found'", + "#class" : turbo.TurboMediaExtractor, + "#count" : 0, +}, + +{ + "#url" : "https://saint2.pk/embed/6lC7mKrJst8", + "#class": turbo.TurboMediaExtractor, +}, + +{ + "#url" : "https://saint2.cr/embed/6lC7mKrJst8", + "#class": turbo.TurboMediaExtractor, +}, + +{ + "#url" : "https://saint.to/embed/6lC7mKrJst8", + "#class": turbo.TurboMediaExtractor, +}, + +{ + "#url" : "https://turbovid.cr/embed/WkD7hRaHdBpBI", + "#comment" : "'turbovid' URL/video", + "#category": ("lolisafe", "turbo", "media"), + "#class" : turbo.TurboMediaExtractor, + "#pattern" : r"https://dl\d+.turbocdn.st/data/\w+.mp4", + + "extension" : "mp4", + "file" : str, + "filename" : "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8-WkD7hRaHdBpBI", + "id" : "WkD7hRaHdBpBI", + "name" : "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", +}, + +{ + "#url" : "https://saint2.su/embed/WkD7hRaHdBpBI", + "#comment" : "'saint' URL/video", + "#class" : turbo.TurboMediaExtractor, +}, + +)