github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update interface of config functions

Browse Source
This commit is contained in:
Mike Fährmann
2019-11-23 23:50:16 +01:00
parent 4ca883c66f
commit f5604492c3
15 changed files with 172 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
gallery_dl/__init__.py
View File

@@ -93,7 +93,8 @@ def parse_inputfile(file, log):
log.warning("input file: unable to parse '%s': %s", value, exc)
continue
conf.append((key.strip().split("."), value))
key = key.strip().split(".")
conf.append((key[:-1], key[-1], value))
else:
# url
@@ -122,11 +123,11 @@ def main():
if args.yamlfiles:
config.load(args.yamlfiles, strict=True, fmt="yaml")
if args.postprocessors:
config.set(("postprocessors",), args.postprocessors)
config.set((), "postprocessors", args.postprocessors)
if args.abort:
config.set(("skip",), "abort:" + str(args.abort))
for key, value in args.options:
config.set(key, value)
config.set((), "skip", "abort:" + str(args.abort))
for opts in args.options:
config.set(*opts)
# stream logging handler
output.configure_logging_handler(
@@ -140,7 +141,7 @@ def main():
# loglevels
if args.loglevel >= logging.ERROR:
config.set(("output", "mode"), "null")
config.set(("output",), "mode", "null")
elif args.loglevel <= logging.DEBUG:
import platform
import subprocess
@@ -230,7 +231,7 @@ def main():
ulog.propagate = False
job.Job.ulog = ulog
pformat = config.get(("output", "progress"), True)
pformat = config.get(("output",), "progress", True)
if pformat and len(urls) > 1 and args.loglevel < logging.ERROR:
urls = progress(urls, pformat)
@@ -239,8 +240,8 @@ def main():
try:
log.debug("Starting %s for '%s'", jobtype.__name__, url)
if isinstance(url, util.ExtendedUrl):
for key, value in url.gconfig:
config.set(key, value)
for opts in url.gconfig:
config.set(*opts)
with config.apply(url.lconfig):
retval |= jobtype(url.value).run()
else:

2
gallery_dl/cache.py
View File

@@ -188,7 +188,7 @@ def clear():
def _path():
path = config.get(("cache", "file"), -1)
path = config.get(("cache",), "file", -1)
if path != -1:
return util.expand_path(path)

77
gallery_dl/config.py
View File

@@ -75,62 +75,57 @@ def clear():
_config.clear()
def get(keys, default=None, conf=_config):
def get(path, key, default=None, *, conf=_config):
"""Get the value of property 'key' or a default value"""
try:
for k in keys:
conf = conf[k]
return conf
except (KeyError, AttributeError):
for p in path:
conf = conf[p]
return conf[key]
except KeyError:
return default
def interpolate(keys, default=None, conf=_config):
def interpolate(path, key, default=None, *, conf=_config):
"""Interpolate the value of 'key'"""
if key in conf:
return conf[key]
try:
lkey = keys[-1]
if lkey in conf:
return conf[lkey]
for k in keys:
if lkey in conf:
default = conf[lkey]
conf = conf[k]
return conf
except (KeyError, AttributeError):
return default
for p in path:
conf = conf[p]
if key in conf:
default = conf[key]
except KeyError:
pass
return default
def set(keys, value, conf=_config):
def set(path, key, value, *, conf=_config):
"""Set the value of property 'key' for this session"""
for k in keys[:-1]:
for p in path:
try:
conf = conf[k]
conf = conf[p]
except KeyError:
temp = {}
conf[k] = temp
conf = temp
conf[keys[-1]] = value
conf[p] = conf = {}
conf[key] = value
def setdefault(keys, value, conf=_config):
def setdefault(path, key, value, *, conf=_config):
"""Set the value of property 'key' if it doesn't exist"""
for k in keys[:-1]:
for p in path:
try:
conf = conf[k]
conf = conf[p]
except KeyError:
temp = {}
conf[k] = temp
conf = temp
return conf.setdefault(keys[-1], value)
conf[p] = conf = {}
return conf.setdefault(key, value)
def unset(keys, conf=_config):
def unset(path, key, *, conf=_config):
"""Unset the value of property 'key'"""
try:
for k in keys[:-1]:
conf = conf[k]
del conf[keys[-1]]
except (KeyError, AttributeError):
for p in path:
conf = conf[p]
del conf[key]
except KeyError:
pass
@@ -143,13 +138,13 @@ class apply():
self.kvlist = kvlist
def __enter__(self):
for key, value in self.kvlist:
self.original.append((key, get(key, self._sentinel)))
set(key, value)
for path, key, value in self.kvlist:
self.original.append((path, key, get(path, key, self._sentinel)))
set(path, key, value)
def __exit__(self, etype, value, traceback):
for key, value in self.original:
for path, key, value in self.original:
if value is self._sentinel:
unset(key)
unset(path, key)
else:
set(key, value)
set(path, key, value)

2
gallery_dl/downloader/common.py
View File

@@ -30,7 +30,7 @@ class DownloaderBase():
def config(self, key, default=None):
"""Interpolate downloader config value for 'key'"""
return config.interpolate(("downloader", self.scheme, key), default)
return config.interpolate(("downloader", self.scheme), key, default)
def download(self, url, pathfmt):
"""Write data from 'url' into the file specified by 'pathfmt'"""

6
gallery_dl/extractor/common.py
View File

@@ -69,7 +69,7 @@ class Extractor():
def config(self, key, default=None):
return config.interpolate(
("extractor", self.category, self.subcategory, key), default)
("extractor", self.category, self.subcategory), key, default)
def request(self, url, *, method="GET", session=None, retries=None,
encoding=None, fatal=True, notfound=None, **kwargs):
@@ -431,7 +431,7 @@ class SharedConfigMixin():
def generate_extractors(extractor_data, symtable, classes):
"""Dynamically generate Extractor classes"""
extractors = config.get(("extractor", classes[0].basecategory))
extractors = config.get(("extractor",), classes[0].basecategory)
ckey = extractor_data.get("_ckey")
prev = None
@@ -477,7 +477,7 @@ http.cookiejar.MozillaCookieJar.magic_re = re.compile(
"#( Netscape)? HTTP Cookie File", re.IGNORECASE)
# Replace default cipher list of urllib3 to avoid Cloudflare CAPTCHAs
ciphers = config.get(("ciphers",), True)
ciphers = config.get((), "ciphers", True)
if ciphers:
logging.getLogger("gallery-dl").debug("Updating urllib3 ciphers")

8
gallery_dl/extractor/mastodon.py
View File

@@ -31,8 +31,8 @@ class MastodonExtractor(Extractor):
if value is not sentinel:
return value
return config.interpolate(
("extractor", "mastodon", self.instance, self.subcategory, key),
default,
("extractor", "mastodon", self.instance, self.subcategory),
key, default,
)
def items(self):
@@ -145,10 +145,10 @@ def generate_extractors():
"""Dynamically generate Extractor classes for Mastodon instances"""
symtable = globals()
extractors = config.get(("extractor", "mastodon"))
extractors = config.get(("extractor",), "mastodon")
if extractors:
EXTRACTORS.update(extractors)
config.set(("extractor", "mastodon"), EXTRACTORS)
config.set(("extractor",), "mastodon", EXTRACTORS)
for instance, info in EXTRACTORS.items():

2
gallery_dl/extractor/oauth.py
View File

@@ -27,7 +27,7 @@ class OAuthBase(Extractor):
def oauth_config(self, key, default=None):
return config.interpolate(
("extractor", self.subcategory, key), default)
("extractor", self.subcategory), key, default)
def recv(self):
"""Open local HTTP server and recv callback parameters"""

14
gallery_dl/job.py
View File

@@ -286,14 +286,14 @@ class DownloadJob(Job):
except KeyError:
pass
klass = downloader.find(scheme)
if klass and config.get(("downloader", klass.scheme, "enabled"), True):
instance = klass(self.extractor, self.out)
cls = downloader.find(scheme)
if cls and config.get(("downloader", cls.scheme), "enabled", True):
instance = cls(self.extractor, self.out)
else:
instance = None
self.log.error("'%s:' URLs are not supported/enabled", scheme)
if klass and klass.scheme == "http":
if cls and cls.scheme == "http":
self.downloaders["http"] = self.downloaders["https"] = instance
else:
self.downloaders[scheme] = instance
@@ -472,9 +472,9 @@ class DataJob(Job):
Job.__init__(self, url, parent)
self.file = file
self.data = []
self.ascii = config.get(("output", "ascii"), ensure_ascii)
self.ascii = config.get(("output",), "ascii", ensure_ascii)
private = config.get(("output", "private"))
private = config.get(("output",), "private")
self.filter = (lambda x: x) if private else util.filter_dict
def run(self):
@@ -490,7 +490,7 @@ class DataJob(Job):
pass
# convert numbers to string
if config.get(("output", "num-to-str"), False):
if config.get(("output",), "num-to-str", False):
for msg in self.data:
util.transform_dict(msg[-1], util.number_to_string)

10
gallery_dl/option.py
View File

@@ -18,13 +18,13 @@ from . import job, version
class ConfigAction(argparse.Action):
"""Set argparse results as config values"""
def __call__(self, parser, namespace, values, option_string=None):
namespace.options.append(((self.dest,), values))
namespace.options.append(((), self.dest, values))
class ConfigConstAction(argparse.Action):
"""Set argparse const values as config values"""
def __call__(self, parser, namespace, values, option_string=None):
namespace.options.append(((self.dest,), self.const))
namespace.options.append(((), self.dest, self.const))
class AppendCommandAction(argparse.Action):
@@ -41,7 +41,7 @@ class DeprecatedConfigConstAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
print("warning: {} is deprecated. Use {} instead.".format(
"/".join(self.option_strings), self.choices), file=sys.stderr)
namespace.options.append(((self.dest,), self.const))
namespace.options.append(((), self.dest, self.const))
class ParseAction(argparse.Action):
@@ -52,8 +52,8 @@ class ParseAction(argparse.Action):
value = json.loads(value)
except ValueError:
pass
key = key.split(".")
namespace.options.append((key, value))
key = key.split(".") # splitting an empty string becomes [""]
namespace.options.append((key[:-1], key[-1], value))
class Formatter(argparse.HelpFormatter):

8
gallery_dl/output.py
View File

@@ -83,7 +83,7 @@ def initialize_logging(loglevel):
def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL):
"""Setup a new logging handler"""
opts = config.interpolate(("output", key))
opts = config.interpolate(("output",), key)
if not opts:
return None
if not isinstance(opts, dict):
@@ -114,7 +114,7 @@ def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL):
def configure_logging_handler(key, handler):
"""Configure a logging handler"""
opts = config.interpolate(("output", key))
opts = config.interpolate(("output",), key)
if not opts:
return
if isinstance(opts, str):
@@ -156,7 +156,7 @@ def select():
"color": ColorOutput,
"null": NullOutput,
}
omode = config.get(("output", "mode"), "auto").lower()
omode = config.get(("output",), "mode", "auto").lower()
if omode in pdict:
return pdict[omode]()
elif omode == "auto":
@@ -192,7 +192,7 @@ class PipeOutput(NullOutput):
class TerminalOutput(NullOutput):
def __init__(self):
self.short = config.get(("output", "shorten"), True)
self.short = config.get(("output",), "shorten", True)
if self.short:
self.width = shutil.get_terminal_size().columns - OFFSET

106
test/test_config.py
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2015-2017 Mike Fährmann
# Copyright 2015-2019 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -16,65 +16,93 @@ import tempfile
class TestConfig(unittest.TestCase):
def setUp(self):
fd, self._configfile = tempfile.mkstemp()
with os.fdopen(fd, "w") as file:
file.write('{"a": "1", "b": {"a": 2, "c": "text"}}')
config.load((self._configfile,))
config.set((), "a", "1")
config.set((), "b", {
"a": 2,
"c": "text",
})
def tearDown(self):
config.clear()
os.remove(self._configfile)
def test_get(self):
self.assertEqual(config.get(["a"]), "1")
self.assertEqual(config.get(["b", "c"]), "text")
self.assertEqual(config.get(["d"]), None)
self.assertEqual(config.get(["e", "f", "g"], 123), 123)
self.assertEqual(config.get(() , "a") , "1")
self.assertEqual(config.get(("b",) , "c") , "text")
self.assertEqual(config.get(() , "d") , None)
self.assertEqual(config.get(("e", "f"), "g", 123), 123)
def test_interpolate(self):
self.assertEqual(config.interpolate(["a"]), "1")
self.assertEqual(config.interpolate(["b", "a"]), "1")
self.assertEqual(config.interpolate(["b", "c"], "2"), "text")
self.assertEqual(config.interpolate(["b", "d"], "2"), "2")
config.set(["d"], 123)
self.assertEqual(config.interpolate(["b", "d"], "2"), 123)
self.assertEqual(config.interpolate(["d", "d"], "2"), 123)
self.assertEqual(config.interpolate(() , "a") , "1")
self.assertEqual(config.interpolate(("b",), "a") , "1")
self.assertEqual(config.interpolate(("b",), "c", "2"), "text")
self.assertEqual(config.interpolate(("b",), "d", "2"), "2")
config.set((), "d", 123)
self.assertEqual(config.interpolate(("b",), "d", "2"), 123)
self.assertEqual(config.interpolate(("d",), "d", "2"), 123)
def test_set(self):
config.set(["b", "c"], [1, 2, 3])
config.set(["e", "f", "g"], value=234)
self.assertEqual(config.get(["b", "c"]), [1, 2, 3])
self.assertEqual(config.get(["e", "f", "g"]), 234)
config.set(("b",) , "c", [1, 2, 3])
config.set(("e", "f"), "g", value=234)
self.assertEqual(config.get(("b",) , "c"), [1, 2, 3])
self.assertEqual(config.get(("e", "f"), "g"), 234)
def test_setdefault(self):
config.setdefault(["b", "c"], [1, 2, 3])
config.setdefault(["e", "f", "g"], value=234)
self.assertEqual(config.get(["b", "c"]), "text")
self.assertEqual(config.get(["e", "f", "g"]), 234)
config.setdefault(("b",) , "c", [1, 2, 3])
config.setdefault(("e", "f"), "g", value=234)
self.assertEqual(config.get(("b",) , "c"), "text")
self.assertEqual(config.get(("e", "f"), "g"), 234)
def test_unset(self):
config.unset(["a"])
config.unset(["b", "c"])
config.unset(["c", "d"])
self.assertEqual(config.get(["a"]), None)
self.assertEqual(config.get(["b", "a"]), 2)
self.assertEqual(config.get(["b", "c"]), None)
config.unset(() , "a")
config.unset(("b",), "c")
config.unset(("c",), "d")
self.assertEqual(config.get(() , "a"), None)
self.assertEqual(config.get(("b",), "a"), 2)
self.assertEqual(config.get(("b",), "c"), None)
def test_apply(self):
options = (
(["b", "c"], [1, 2, 3]),
(["e", "f", "g"], 234),
(("b",) , "c", [1, 2, 3]),
(("e", "f"), "g", 234),
)
self.assertEqual(config.get(["b", "c"]), "text")
self.assertEqual(config.get(["e", "f", "g"]), None)
self.assertEqual(config.get(("b",) , "c"), "text")
self.assertEqual(config.get(("e", "f"), "g"), None)
with config.apply(options):
self.assertEqual(config.get(["b", "c"]), [1, 2, 3])
self.assertEqual(config.get(["e", "f", "g"]), 234)
self.assertEqual(config.get(("b",) , "c"), [1, 2, 3])
self.assertEqual(config.get(("e", "f"), "g"), 234)
self.assertEqual(config.get(["b", "c"]), "text")
self.assertEqual(config.get(["e", "f", "g"]), None)
self.assertEqual(config.get(("b",) , "c"), "text")
self.assertEqual(config.get(("e", "f"), "g"), None)
def test_load(self):
with tempfile.TemporaryDirectory() as base:
path1 = os.path.join(base, "cfg1")
with open(path1, "w") as file:
file.write('{"a": "1", "b": {"a": 2, "c": "text"}}')
path2 = os.path.join(base, "cfg2")
with open(path2, "w") as file:
file.write('{"a": "7", "b": {"a": 8, "e": "foo"}}')
config.load((path1,))
self.assertEqual(config.get(() , "a"), "1")
self.assertEqual(config.get(("b",), "a"), 2)
self.assertEqual(config.get(("b",), "c"), "text")
config.load((path2,))
self.assertEqual(config.get(() , "a"), "7")
self.assertEqual(config.get(("b",), "a"), 8)
self.assertEqual(config.get(("b",), "c"), "text")
self.assertEqual(config.get(("b",), "e"), "foo")
config.clear()
config.load((path1, path2))
self.assertEqual(config.get(() , "a"), "7")
self.assertEqual(config.get(("b",), "a"), 8)
self.assertEqual(config.get(("b",), "c"), "text")
self.assertEqual(config.get(("b",), "e"), "foo")
if __name__ == '__main__':

12
test/test_cookies.py
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2017 Mike Fährmann
# Copyright 2017-2019 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -18,8 +18,6 @@ from os.path import join
import gallery_dl.config as config
import gallery_dl.extractor as extractor
CKEY = ("cookies",)
class TestCookiejar(unittest.TestCase):
@@ -45,7 +43,7 @@ class TestCookiejar(unittest.TestCase):
config.clear()
def test_cookiefile(self):
config.set(CKEY, self.cookiefile)
config.set((), "cookies", self.cookiefile)
cookies = extractor.find("test:").session.cookies
self.assertEqual(len(cookies), 1)
@@ -63,7 +61,7 @@ class TestCookiejar(unittest.TestCase):
self._test_warning(join(self.path.name, "nothing"), FileNotFoundError)
def _test_warning(self, filename, exc):
config.set(CKEY, filename)
config.set((), "cookies", filename)
log = logging.getLogger("test")
with mock.patch.object(log, "warning") as mock_warning:
cookies = extractor.find("test:").session.cookies
@@ -77,7 +75,7 @@ class TestCookiedict(unittest.TestCase):
def setUp(self):
self.cdict = {"NAME1": "VALUE1", "NAME2": "VALUE2"}
config.set(CKEY, self.cdict)
config.set((), "cookies", self.cdict)
def tearDown(self):
config.clear()
@@ -112,7 +110,7 @@ class TestCookieLogin(unittest.TestCase):
}
for category, cookienames in extr_cookies.items():
cookies = {name: "value" for name in cookienames}
config.set(CKEY, cookies)
config.set((), "cookies", cookies)
extr = _get_extractor(category)
with mock.patch.object(extr, "_login_impl") as mock_login:
extr.login()

2
test/test_downloader.py
View File

@@ -100,7 +100,7 @@ class TestDownloaderBase(unittest.TestCase):
cls.extractor = extractor.find("test:")
cls.dir = tempfile.TemporaryDirectory()
cls.fnum = 0
config.set(("base-directory",), cls.dir.name)
config.set((), "base-directory", cls.dir.name)
@classmethod
def tearDownClass(cls):

2
test/test_postprocessor.py
View File

@@ -58,7 +58,7 @@ class BasePostprocessorTest(unittest.TestCase):
def setUpClass(cls):
cls.extractor = extractor.find("test:")
cls.dir = tempfile.TemporaryDirectory()
config.set(("base-directory",), cls.dir.name)
config.set((), "base-directory", cls.dir.name)
@classmethod
def tearDownClass(cls):

50
test/test_results.py
View File

@@ -55,10 +55,11 @@ class TestExtractorResults(unittest.TestCase):
if result:
if "options" in result:
for key, value in result["options"]:
config.set(key.split("."), value)
key = key.split(".")
config.set(key[:-1], key[-1], value)
if "range" in result:
config.set(("image-range",), result["range"])
config.set(("chapter-range",), result["range"])
config.set((), "image-range" , result["range"])
config.set((), "chapter-range", result["range"])
content = "content" in result
else:
content = False
@@ -285,35 +286,36 @@ def setup_test_config():
email = "gallerydl@openaliasbox.org"
config.clear()
config.set(("cache", "file"), ":memory:")
config.set(("downloader", "part"), False)
config.set(("downloader", "adjust-extensions"), False)
config.set(("extractor", "timeout"), 60)
config.set(("extractor", "username"), name)
config.set(("extractor", "password"), name)
config.set(("extractor", "nijie" , "username"), email)
config.set(("extractor", "seiga" , "username"), email)
config.set(("cache",), "file", None)
config.set(("downloader",), "part", False)
config.set(("downloader",), "adjust-extensions", False)
config.set(("extractor" ,), "timeout" , 60)
config.set(("extractor" ,), "username", name)
config.set(("extractor" ,), "password", name)
config.set(("extractor", "danbooru" , "username"), None)
config.set(("extractor", "instagram" , "username"), None)
config.set(("extractor", "imgur" , "username"), None)
config.set(("extractor", "newgrounds", "username"), None)
config.set(("extractor", "twitter" , "username"), None)
config.set(("extractor", "nijie") , "username", email)
config.set(("extractor", "seiga") , "username", email)
config.set(("extractor", "danbooru") , "username", None)
config.set(("extractor", "instagram") , "username", None)
config.set(("extractor", "twitter") , "username", None)
config.set(("extractor", "mangoxo" , "username"), "LiQiang3")
config.set(("extractor", "mangoxo" , "password"), "5zbQF10_5u25259Ma")
config.set(("extractor", "newgrounds"), "username", "d1618111")
config.set(("extractor", "newgrounds"), "password", "d1618111")
config.set(("extractor", "deviantart", "client-id"), "7777")
config.set(("extractor", "deviantart", "client-secret"),
config.set(("extractor", "mangoxo") , "username", "LiQiang3")
config.set(("extractor", "mangoxo") , "password", "5zbQF10_5u25259Ma")
config.set(("extractor", "deviantart"), "client-id", "7777")
config.set(("extractor", "deviantart"), "client-secret",
"ff14994c744d9208e5caeec7aab4a026")
config.set(("extractor", "tumblr", "api-key"),
config.set(("extractor", "tumblr"), "api-key",
"0cXoHfIqVzMQcc3HESZSNsVlulGxEXGDTTZCDrRrjaa0jmuTc6")
config.set(("extractor", "tumblr", "api-secret"),
config.set(("extractor", "tumblr"), "api-secret",
"6wxAK2HwrXdedn7VIoZWxGqVhZ8JdYKDLjiQjL46MLqGuEtyVj")
config.set(("extractor", "tumblr", "access-token"),
config.set(("extractor", "tumblr"), "access-token",
"N613fPV6tOZQnyn0ERTuoEZn0mEqG8m2K8M3ClSJdEHZJuqFdG")
config.set(("extractor", "tumblr", "access-token-secret"),
config.set(("extractor", "tumblr"), "access-token-secret",
"sgOA7ZTT4FBXdOGGVV331sSp0jHYp4yMDRslbhaQf7CaS71i4O")