diff --git a/gallery_dl/extractor/booru.py b/gallery_dl/extractor/booru.py index 0176d769..378e1440 100644 --- a/gallery_dl/extractor/booru.py +++ b/gallery_dl/extractor/booru.py @@ -8,7 +8,7 @@ """Base classes for extractors for danbooru and co""" -from .common import Extractor, Message, SharedConfigMixin +from .common import Extractor, Message from .. import text, exception from xml.etree import ElementTree import collections @@ -17,7 +17,7 @@ import operator import re -class BooruExtractor(SharedConfigMixin, Extractor): +class BooruExtractor(Extractor): """Base class for all booru extractors""" basecategory = "booru" filename_fmt = "{category}_{id}_{md5}.{extension}" diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 5efea4a2..5480827e 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -24,6 +24,7 @@ class Extractor(): category = "" subcategory = "" + basecategory = "" categorytransfer = False directory_fmt = ("{category}",) filename_fmt = "{filename}.{extension}" @@ -54,6 +55,10 @@ class Extractor(): if self._retries < 0: self._retries = float("inf") + if self.basecategory: + self.config = self._config_shared + self.config_accumulate = self._config_shared_accumulate + self._init_headers() self._init_cookies() self._init_proxies() @@ -80,6 +85,19 @@ class Extractor(): def config_accumulate(self, key): return config.accumulate(self._cfgpath, key) + def _config_shared(self, key, default=None): + return config.interpolate_common(("extractor",), ( + (self.category, self.subcategory), + (self.basecategory, self.subcategory), + ), key, default) + + def _config_shared_accumulate(self, key): + values = config.accumulate(self._cfgpath, key) + conf = config.get(("extractor",), self.basecategory) + if conf: + values[:0] = config.accumulate((self.subcategory,), key, conf=conf) + return values + def request(self, url, *, method="GET", session=None, retries=None, encoding=None, fatal=True, notfound=None, **kwargs): tries = 1 @@ -506,28 +524,6 @@ class AsynchronousMixin(): messages.put(None) -class SharedConfigMixin(): - """Enable sharing of config settings based on 'basecategory'""" - basecategory = "" - - def config(self, key, default=None): - return config.interpolate_common( - ("extractor",), ( - (self.category, self.subcategory), - (self.basecategory, self.subcategory), - ), key, default, - ) - - def config_accumulate(self, key): - values = config.accumulate(self._cfgpath, key) - - conf = config.get(("extractor",), self.basecategory) - if conf: - values[:0] = config.accumulate((self.subcategory,), key, conf=conf) - - return values - - def generate_extractors(extractor_data, symtable, classes): """Dynamically generate Extractor classes""" extractors = config.get(("extractor",), classes[0].basecategory) diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index 1ebaf5bf..ca37cb49 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -8,7 +8,7 @@ """Extractors for https://danbooru.donmai.us/""" -from .common import Extractor, Message, SharedConfigMixin +from .common import Extractor, Message from .. import text import datetime @@ -20,7 +20,7 @@ BASE_PATTERN = ( ) -class DanbooruExtractor(SharedConfigMixin, Extractor): +class DanbooruExtractor(Extractor): """Base class for danbooru extractors""" basecategory = "booru" category = "danbooru" diff --git a/gallery_dl/extractor/foolfuuka.py b/gallery_dl/extractor/foolfuuka.py index f2019ca1..8a03dc99 100644 --- a/gallery_dl/extractor/foolfuuka.py +++ b/gallery_dl/extractor/foolfuuka.py @@ -8,13 +8,13 @@ """Extractors for 4chan archives based on FoolFuuka""" -from .common import Extractor, Message, SharedConfigMixin, generate_extractors +from .common import Extractor, Message, generate_extractors from .. import text import itertools import operator -class FoolfuukaThreadExtractor(SharedConfigMixin, Extractor): +class FoolfuukaThreadExtractor(Extractor): """Base extractor for FoolFuuka based boards/archives""" basecategory = "foolfuuka" subcategory = "thread" diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py index 42456177..db5e250b 100644 --- a/gallery_dl/extractor/foolslide.py +++ b/gallery_dl/extractor/foolslide.py @@ -12,7 +12,6 @@ from .common import ( Extractor, ChapterExtractor, MangaExtractor, - SharedConfigMixin, Message, generate_extractors, ) @@ -20,7 +19,7 @@ from .. import text, util import json -class FoolslideBase(SharedConfigMixin): +class FoolslideBase(): """Base class for FoOlSlide extractors""" basecategory = "foolslide" diff --git a/gallery_dl/extractor/imagehosts.py b/gallery_dl/extractor/imagehosts.py index ad5a5085..28af1791 100644 --- a/gallery_dl/extractor/imagehosts.py +++ b/gallery_dl/extractor/imagehosts.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2019 Mike Fährmann +# Copyright 2016-2020 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -8,13 +8,13 @@ """Collection of extractors for various imagehosts""" -from .common import Extractor, Message, SharedConfigMixin +from .common import Extractor, Message from .. import text, exception from ..cache import memcache from os.path import splitext -class ImagehostImageExtractor(SharedConfigMixin, Extractor): +class ImagehostImageExtractor(Extractor): """Base class for single-image extractors for various imagehosts""" basecategory = "imagehost" subcategory = "image" diff --git a/gallery_dl/extractor/paheal.py b/gallery_dl/extractor/paheal.py index e0b04966..88f7f65a 100644 --- a/gallery_dl/extractor/paheal.py +++ b/gallery_dl/extractor/paheal.py @@ -6,13 +6,13 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract images from https://rule34.paheal.net/""" +"""Extractors for https://rule34.paheal.net/""" -from .common import Extractor, Message, SharedConfigMixin +from .common import Extractor, Message from .. import text -class PahealExtractor(SharedConfigMixin, Extractor): +class PahealExtractor(Extractor): """Base class for paheal extractors""" basecategory = "booru" category = "paheal" diff --git a/gallery_dl/extractor/reactor.py b/gallery_dl/extractor/reactor.py index a20312f7..cfbab1d0 100644 --- a/gallery_dl/extractor/reactor.py +++ b/gallery_dl/extractor/reactor.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2020 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -8,7 +8,7 @@ """Generic extractors for *reactor sites""" -from .common import Extractor, Message, SharedConfigMixin +from .common import Extractor, Message from .. import text import urllib.parse import random @@ -19,7 +19,7 @@ import json BASE_PATTERN = r"(?:https?://)?((?:[^/.]+\.)?reactor\.cc)" -class ReactorExtractor(SharedConfigMixin, Extractor): +class ReactorExtractor(Extractor): """Base class for *reactor.cc extractors""" basecategory = "reactor" filename_fmt = "{post_id}_{num:>02}{title[:100]:?_//}.{extension}" diff --git a/gallery_dl/extractor/sankaku.py b/gallery_dl/extractor/sankaku.py index a9252f50..3beeb488 100644 --- a/gallery_dl/extractor/sankaku.py +++ b/gallery_dl/extractor/sankaku.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2019 Mike Fährmann +# Copyright 2014-2020 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -8,7 +8,7 @@ """Extractors for https://chan.sankakucomplex.com/""" -from .common import Extractor, Message, SharedConfigMixin +from .common import Extractor, Message from .. import text, util, exception from ..cache import cache import collections @@ -17,7 +17,7 @@ import time import re -class SankakuExtractor(SharedConfigMixin, Extractor): +class SankakuExtractor(Extractor): """Base class for sankaku extractors""" basecategory = "booru" category = "sankaku" diff --git a/gallery_dl/extractor/shopify.py b/gallery_dl/extractor/shopify.py index 9d1df18c..d65f3344 100644 --- a/gallery_dl/extractor/shopify.py +++ b/gallery_dl/extractor/shopify.py @@ -8,12 +8,12 @@ """Extractors for Shopify instances""" -from .common import Extractor, Message, SharedConfigMixin, generate_extractors +from .common import Extractor, Message, generate_extractors from .. import text import re -class ShopifyExtractor(SharedConfigMixin, Extractor): +class ShopifyExtractor(Extractor): """Base class for Shopify extractors""" basecategory = "shopify" filename_fmt = "{product[title]}_{num:>02}_{id}.{extension}" diff --git a/test/test_results.py b/test/test_results.py index 239b5add..6c4b25f3 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -368,7 +368,7 @@ def generate_tests(): # filter available extractor classes extractors = [ extr for extr in extractor.extractors() - if fltr(extr.category, getattr(extr, "basecategory", None)) + if fltr(extr.category, extr.basecategory) ] # add 'test_...' methods