From d26da3b9e54347964474bc70f1c6bf99345feb85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 9 May 2022 22:20:09 +0200 Subject: [PATCH] add pre-generated 'pattern' for supported BaseExtractor sites --- gallery_dl/extractor/foolfuuka.py | 8 ++++++- gallery_dl/extractor/foolslide.py | 3 ++- gallery_dl/extractor/gelbooru_v01.py | 34 +++++++++++++++++++++------- gallery_dl/extractor/gelbooru_v02.py | 26 +++++++++++++++------ gallery_dl/extractor/mastodon.py | 5 +++- gallery_dl/extractor/moebooru.py | 5 +++- gallery_dl/extractor/philomena.py | 25 +++++++++++++------- gallery_dl/extractor/reactor.py | 3 ++- gallery_dl/extractor/shopify.py | 4 +++- 9 files changed, 84 insertions(+), 29 deletions(-) diff --git a/gallery_dl/extractor/foolfuuka.py b/gallery_dl/extractor/foolfuuka.py index 04e5926b..b352250c 100644 --- a/gallery_dl/extractor/foolfuuka.py +++ b/gallery_dl/extractor/foolfuuka.py @@ -6,7 +6,7 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extractors for 4chan archives based on FoolFuuka""" +"""Extractors for FoolFuuka 4chan archives""" from .common import BaseExtractor, Message from .. import text @@ -66,6 +66,7 @@ BASE_PATTERN = FoolfuukaExtractor.update({ }, "archivedmoe": { "root": "https://archived.moe", + "pattern": r"archived\.moe", }, "archiveofsins": { "root": "https://archiveofsins.com", @@ -73,12 +74,15 @@ BASE_PATTERN = FoolfuukaExtractor.update({ }, "b4k": { "root": "https://arch.b4k.co", + "pattern": r"arch\.b4k\.co", }, "desuarchive": { "root": "https://desuarchive.org", + "pattern": r"desuarchive\.org", }, "fireden": { "root": "https://boards.fireden.net", + "pattern": r"boards\.fireden\.net", }, "nyafuu": { "root": "https://archive.nyafuu.org", @@ -90,9 +94,11 @@ BASE_PATTERN = FoolfuukaExtractor.update({ }, "thebarchive": { "root": "https://thebarchive.com", + "pattern": r"thebarchive\.com", }, "wakarimasen": { "root": "https://archive.wakarimasen.moe", + "pattern": r"archive\.wakarimasen\.moe", }, }) diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py index c09eb969..382cc259 100644 --- a/gallery_dl/extractor/foolslide.py +++ b/gallery_dl/extractor/foolslide.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2021 Mike Fährmann +# Copyright 2016-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -41,6 +41,7 @@ class FoolslideExtractor(BaseExtractor): BASE_PATTERN = FoolslideExtractor.update({ "kireicake": { "root": "https://reader.kireicake.com", + "pattern": r"reader\.kireicake\.com", }, "powermanga": { "root": "https://read.powermanga.org", diff --git a/gallery_dl/extractor/gelbooru_v01.py b/gallery_dl/extractor/gelbooru_v01.py index f0f50cce..6afd246f 100644 --- a/gallery_dl/extractor/gelbooru_v01.py +++ b/gallery_dl/extractor/gelbooru_v01.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extractors for Gelbooru v0.1 sites""" +"""Extractors for Gelbooru Beta 0.1.11 sites""" from . import booru from .. import text @@ -59,12 +59,30 @@ class GelbooruV01Extractor(booru.BooruExtractor): BASE_PATTERN = GelbooruV01Extractor.update({ - "thecollection" : {"root": "https://the-collection.booru.org"}, - "illusioncardsbooru": {"root": "https://illusioncards.booru.org"}, - "allgirlbooru" : {"root": "https://allgirl.booru.org"}, - "drawfriends" : {"root": "https://drawfriends.booru.org"}, - "vidyart" : {"root": "https://vidyart.booru.org"}, - "theloudbooru" : {"root": "https://tlb.booru.org"}, + "thecollection": { + "root": "https://the-collection.booru.org", + "pattern": r"the-collection\.booru\.org", + }, + "illusioncardsbooru": { + "root": "https://illusioncards.booru.org", + "pattern": r"illusioncards\.booru\.org", + }, + "allgirlbooru": { + "root": "https://allgirl.booru.org", + "pattern": r"allgirl\.booru\.org", + }, + "drawfriends": { + "root": "https://drawfriends.booru.org", + "pattern": r"drawfriends\.booru\.org", + }, + "vidyart": { + "root": "https://vidyart.booru.org", + "pattern": r"vidyart\.booru\.org", + }, + "theloudbooru": { + "root": "https://tlb.booru.org", + "pattern": r"tlb\.booru\.org", + }, }) diff --git a/gallery_dl/extractor/gelbooru_v02.py b/gallery_dl/extractor/gelbooru_v02.py index a651c5d5..2dd0c0c8 100644 --- a/gallery_dl/extractor/gelbooru_v02.py +++ b/gallery_dl/extractor/gelbooru_v02.py @@ -6,7 +6,7 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extractors for Gelbooru v0.2 sites""" +"""Extractors for Gelbooru Beta 0.2 sites""" from . import booru from .. import text, util, exception @@ -116,11 +116,23 @@ class GelbooruV02Extractor(booru.BooruExtractor): INSTANCES = { - "realbooru": {"root": "https://realbooru.com"}, - "rule34" : {"root": "https://rule34.xxx", - "api_root": " https://api.rule34.xxx"}, - "safebooru": {"root": "https://safebooru.org"}, - "tbib" : {"root": "https://tbib.org"}, + "realbooru": { + "root": "https://realbooru.com", + "pattern": r"realbooru\.com", + }, + "rule34": { + "root": "https://rule34.xxx", + "pattern": r"rule34\.xxx", + "api_root": "https://api.rule34.xxx", + }, + "safebooru": { + "root": "https://safebooru.org", + "pattern": r"safebooru\.org", + }, + "tbib": { + "root": "https://tbib.org", + "pattern": r"tbib\.org", + }, } BASE_PATTERN = GelbooruV02Extractor.update(INSTANCES) @@ -158,7 +170,7 @@ class GelbooruV02TagExtractor(GelbooruV02Extractor): return {"search_tags": self.tags} def posts(self): - return self._pagination({"tags" : self.tags}) + return self._pagination({"tags": self.tags}) class GelbooruV02PoolExtractor(GelbooruV02Extractor): diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py index cd7cabb9..6e780e8b 100644 --- a/gallery_dl/extractor/mastodon.py +++ b/gallery_dl/extractor/mastodon.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2021 Mike Fährmann +# Copyright 2019-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -60,12 +60,14 @@ class MastodonExtractor(BaseExtractor): INSTANCES = { "mastodon.social": { "root" : "https://mastodon.social", + "pattern" : r"mastodon\.social", "access-token" : "Y06R36SMvuXXN5_wiPKFAEFiQaMSQg0o_hGgc86Jj48", "client-id" : "dBSHdpsnOUZgxOnjKSQrWEPakO3ctM7HmsyoOd4FcRo", "client-secret": "DdrODTHs_XoeOsNVXnILTMabtdpWrWOAtrmw91wU1zI", }, "pawoo": { "root" : "https://pawoo.net", + "pattern" : r"pawoo\.net", "access-token" : "c12c9d275050bce0dc92169a28db09d7" "0d62d0a75a8525953098c167eacd3668", "client-id" : "978a25f843ec01e53d09be2c290cd75c" @@ -75,6 +77,7 @@ INSTANCES = { }, "baraag": { "root" : "https://baraag.net", + "pattern" : r"baraag\.net", "access-token" : "53P1Mdigf4EJMH-RmeFOOSM9gdSDztmrAYFgabOKKE0", "client-id" : "czxx2qilLElYHQ_sm-lO8yXuGwOHxLX9RYYaD0-nq1o", "client-secret": "haMaFdMBgK_-BIxufakmI2gFgkYjqmgXGEO2tB-R2xY", diff --git a/gallery_dl/extractor/moebooru.py b/gallery_dl/extractor/moebooru.py index 604d65cc..65b9a837 100644 --- a/gallery_dl/extractor/moebooru.py +++ b/gallery_dl/extractor/moebooru.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020-2021 Mike Fährmann +# Copyright 2020-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -54,6 +54,7 @@ class MoebooruExtractor(BooruExtractor): BASE_PATTERN = MoebooruExtractor.update({ "yandere": { "root": "https://yande.re", + "pattern": r"yande\.re", }, "konachan": { "root": "https://konachan.com", @@ -61,6 +62,7 @@ BASE_PATTERN = MoebooruExtractor.update({ }, "hypnohub": { "root": "https://hypnohub.net", + "pattern": r"hypnohub\.net", }, "sakugabooru": { "root": "https://www.sakugabooru.com", @@ -68,6 +70,7 @@ BASE_PATTERN = MoebooruExtractor.update({ }, "lolibooru": { "root": "https://lolibooru.moe", + "pattern": r"lolibooru\.moe", }, }) diff --git a/gallery_dl/extractor/philomena.py b/gallery_dl/extractor/philomena.py index 92b81131..951b34df 100644 --- a/gallery_dl/extractor/philomena.py +++ b/gallery_dl/extractor/philomena.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -58,12 +58,21 @@ class PhilomenaExtractor(BooruExtractor): INSTANCES = { - "derpibooru": {"root": "https://derpibooru.org", - "filter_id": "56027"}, - "ponybooru" : {"root": "https://ponybooru.org", - "filter_id": "2"}, - "furbooru" : {"root": "https://furbooru.org", - "filter_id": "2"}, + "derpibooru": { + "root": "https://derpibooru.org", + "pattern": r"derpibooru\.org", + "filter_id": "56027", + }, + "ponybooru": { + "root": "https://ponybooru.org", + "pattern": r"ponybooru\.org", + "filter_id": "2", + }, + "furbooru": { + "root": "https://furbooru.org", + "pattern": r"furbooru\.org", + "filter_id": "2", + }, } BASE_PATTERN = PhilomenaExtractor.update(INSTANCES) @@ -239,5 +248,5 @@ class PhilomenaGalleryExtractor(PhilomenaExtractor): def posts(self): gallery_id = "gallery_id:" + self.gallery_id url = self.root + "/api/v1/json/search/images" - params = {"sd": "desc", "sf": gallery_id, "q" : gallery_id} + params = {"sd": "desc", "sf": gallery_id, "q": gallery_id} return self._pagination(url, params) diff --git a/gallery_dl/extractor/reactor.py b/gallery_dl/extractor/reactor.py index b3a620a6..db8d7004 100644 --- a/gallery_dl/extractor/reactor.py +++ b/gallery_dl/extractor/reactor.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2021 Mike Fährmann +# Copyright 2019-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -160,6 +160,7 @@ BASE_PATTERN = ReactorExtractor.update({ }, "thatpervert": { "root": "http://thatpervert.com", + "pattern": r"thatpervert\.com", }, }) diff --git a/gallery_dl/extractor/shopify.py b/gallery_dl/extractor/shopify.py index f276e845..2fd76c58 100644 --- a/gallery_dl/extractor/shopify.py +++ b/gallery_dl/extractor/shopify.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2021 Mike Fährmann +# Copyright 2019-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -49,9 +49,11 @@ BASE_PATTERN = ShopifyExtractor.update({ }, "omgmiamiswimwear": { "root": "https://www.omgmiamiswimwear.com", + "pattern": r"(?:www\.)?omgmiamiswimwear\.com", }, "windsorstore": { "root": "https://www.windsorstore.com", + "pattern": r"(?:www\.)?windsorstore\.com", }, "loungeunderwear": { "root": "https://loungeunderwear.com",