From e53cdfd6a8a84add935aff401b9f7717ad36632b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 9 Jan 2019 14:21:19 +0100 Subject: [PATCH] update build_supportedsites.py --- docs/supportedsites.rst | 4 ++-- gallery_dl/extractor/imagehosts.py | 4 ++-- gallery_dl/extractor/reactor.py | 14 +++++++------- scripts/build_supportedsites.py | 10 ++++++++-- test/test_results.py | 2 ++ 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index c80df68d..787349dd 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -14,7 +14,7 @@ Archived.Moe https://archived.moe/ Threads ArtStation https://www.artstation.com/ |Capabilities-0| Behance https://www.behance.net/ Images from Users, Galleries BobX http://www.bobx.com/dark/ Galleries, Idols -Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag-Searches +Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag-Searches Optional Desuarchive https://desuarchive.org/ Threads DeviantArt https://www.deviantart.com/ |Capabilities-1| Optional (OAuth) Doki Reader https://kobato.hologfx.com/ Chapters, Manga @@ -96,7 +96,7 @@ Imgspice https://imgspice.com/ individual Images Imxto https://imx.to/ individual Images Pixhost https://pixhost.to/ individual Images Postimg https://postimg.org/ individual Images -Turboimagehost https://turboimagehost.com/ individual Images +Turboimagehost https://www.turboimagehost.com/ individual Images ==================== =================================== ================================================== ================ .. |Site-0| replace:: yuki.la 4chan archive diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py index 982db3e9..9a66c892 100644 --- a/gallery_dl/extractor/imagehosts.py +++ b/gallery_dl/extractor/imagehosts.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2018 Mike Fährmann +# Copyright 2016-2019 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -148,7 +148,7 @@ class ImagetwistImageExtractor(ImagehostImageExtractor): """Extractor for single images from imagetwist.com""" category = "imagetwist" pattern = [r"(?:https?://)?((?:www\.)?imagetwist\.com/([a-z0-9]{12}))"] - test = [("http://imagetwist.com/4e46hv31tu0q/test.jpg", { + test = [("https://imagetwist.com/4e46hv31tu0q/test.jpg", { "url": "c999dc1a5dec0525ac9eb8c092f173dfe6dba0b0", "keyword": "30dd34dcb06b5b51c6cfff199c610b24edb7b9bc", "content": "96b1fd099b06faad5879fce23a7e4eb8290d8810", diff --git a/gallery_dl/extractor/reactor.py b/gallery_dl/extractor/reactor.py index 6a3abbeb..3c0a77fe 100644 --- a/gallery_dl/extractor/reactor.py +++ b/gallery_dl/extractor/reactor.py @@ -202,13 +202,13 @@ class JoyreactorTagExtractor(ReactorTagExtractor): category = "joyreactor" pattern = [JR_BASE_PATTERN + r"/tag/([^/?&#]+)"] test = [ + ("http://joyreactor.cc/tag/Advent+Cirno", { + "count": ">= 17", + }), ("http://joyreactor.com/tag/Cirno", { "url": "a81382a3146da50b647c475f87427a6ca1d737df", "keyword": "dcd3b101cae0a93fbb91281235de1410faf88455", }), - ("http://joyreactor.cc/tag/Advent+Cirno", { - "count": ">= 17", - }), ] @@ -217,13 +217,13 @@ class JoyreactorSearchExtractor(ReactorSearchExtractor): category = "joyreactor" pattern = [JR_BASE_PATTERN + r"/search(?:/|\?q=)([^/?&#]+)"] test = [ - ("http://joyreactor.com/search?q=Cirno+Gifs", { - "count": 0, # no search results on joyreactor.com - }), ("http://joyreactor.cc/search/Cirno+Gifs", { "range": "1-25", "count": ">= 20", }), + ("http://joyreactor.com/search?q=Cirno+Gifs", { + "count": 0, # no search results on joyreactor.com + }), ] @@ -232,11 +232,11 @@ class JoyreactorUserExtractor(ReactorUserExtractor): category = "joyreactor" pattern = [JR_BASE_PATTERN + r"/user/([^/?&#]+)"] test = [ + ("http://joyreactor.cc/user/hemantic", None), ("http://joyreactor.com/user/Tacoman123", { "url": "0444158f17c22f08515ad4e7abf69ad2f3a63b35", "keyword": "1571a81fa5b8bab81528c93065d2460a72e77102", }), - ("http://joyreactor.cc/user/hemantic", None), ] diff --git a/scripts/build_supportedsites.py b/scripts/build_supportedsites.py index ce5e87f9..6fd51528 100755 --- a/scripts/build_supportedsites.py +++ b/scripts/build_supportedsites.py @@ -87,7 +87,7 @@ SUBCATEGORY_MAP = { } AUTH_MAP = { - "batoto" : "Optional", + "danbooru" : "Optional", "deviantart" : "Optional (OAuth)", "exhentai" : "Optional", "flickr" : "Optional (OAuth)", @@ -176,7 +176,7 @@ def build_list(): last = None for extr in gallery_dl.extractor.extractors(): - if extr.category in IGNORE_LIST: + if not extr.category or extr.category in IGNORE_LIST: continue if extr.category == last or not last: classes.append(extr) @@ -200,9 +200,15 @@ def build_list(): def get_domain(classes): try: cls = classes[0] + url = sys.modules[cls.__module__].__doc__.split()[-1] if url.startswith("http"): return url + + if hasattr(cls, "test") and cls.test: + url = cls.test[0][0] + return url[:url.find("/", 8)+1] + scheme = "https" if hasattr(cls, "https") and cls.https else "http" host = cls.__doc__.split()[-1] return scheme + "://" + host + "/" diff --git a/test/test_results.py b/test/test_results.py index 0dccc299..25962a98 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -23,6 +23,7 @@ TRAVIS_SKIP = { # temporary issues, etc. BROKEN = { + "pinterest", } @@ -38,6 +39,7 @@ def setup_test_config(): config.set(("extractor", "password"), name) config.set(("extractor", "nijie", "username"), email) config.set(("extractor", "seiga", "username"), email) + config.set(("extractor", "danbooru", "username"), None) config.set(("extractor", "deviantart", "client-id"), "7777") config.set(("extractor", "deviantart", "client-secret"),