diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index e93e3579..923d04e8 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -20,6 +20,7 @@ if sys.hexversion < 0x3030000: print("Python 3.3+ required", file=sys.stderr) sys.exit(1) +import json import logging from . import version, config, option, extractor, job, util, exception @@ -51,14 +52,6 @@ def progress(urls, pformat): yield pinfo["url"] -def sanatize_input(file): - """Filter and strip strings from an input file""" - for line in file: - line = line.strip() - if line: - yield line - - def prepare_range(rangespec, target): if rangespec: range = util.optimize_range(util.parse_range(rangespec)) @@ -78,6 +71,56 @@ def prepare_filter(filterexpr, target): log.warning(exc) +def parse_inputfile(file): + """Filter and strip strings from an input file + + Lines starting with '#' and empty lines will be ignored. + Lines starting with '{' will be interpreted as JSON-object and + its values, while processing the next URL, are going to be + applied to the global config. + Everything else will be used as potential URL. + + Example input file: + + # this is a comment + {"base-directory": "/tmp/", "skip": false} + {"more": "multiple objects before an URL will be merged together"} + https://example.org/ + + # config is back to its initial values + https://example.com/index.htm + """ + confdict = None + + for line in file: + line = line.strip() + + if not line or line[0] == "#": + # empty line or comment + continue + + elif line[0] == "{": + # url-specfic config spec + try: + cfd = json.loads(line) + except ValueError as exc: + log.warning("input file: unable to parse config line: %s",exc) + continue + + if confdict: + util.combine_dict(confdict, cfd) + else: + confdict = cfd + + else: + # url + if confdict: + yield util.ExtendedUrl(line, confdict) + else: + yield line + confdict = None + + def main(): try: parser = option.build_parser() @@ -162,7 +205,7 @@ def main(): file = sys.stdin else: file = open(args.inputfile) - urls += sanatize_input(file) + urls += parse_inputfile(file) file.close() except OSError as exc: log.warning("input file: %s", exc) @@ -187,7 +230,11 @@ def main(): for url in urls: try: log.debug("Starting %s for '%s'", jobtype.__name__, url) - jobtype(url).run() + if isinstance(url, util.ExtendedUrl): + with config.apply(url.config): + jobtype(url.value).run() + else: + jobtype(url).run() except exception.NoExtractorError: log.error("No suitable extractor found for '%s'", url) diff --git a/gallery_dl/config.py b/gallery_dl/config.py index 6066078c..5e9d62e0 100644 --- a/gallery_dl/config.py +++ b/gallery_dl/config.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2017 Mike Fährmann +# Copyright 2015-2018 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -73,8 +73,8 @@ def load(*files, format="json", strict=False): def clear(): - """Reset configuration to en empty state""" - globals()["_config"].clear() + """Reset configuration to an empty state""" + _config.clear() def get(keys, default=None, conf=_config): @@ -124,3 +124,24 @@ def setdefault(keys, value, conf=_config): conf[k] = temp conf = temp return conf.setdefault(keys[-1], value) + + +class apply(): + """Context Manager to apply a dict to global config""" + _sentinel = object() + + def __init__(self, config_dict): + self.original_values = {} + self.config_dict = config_dict + for key, value in config_dict.items(): + self.original_values[key] = _config.get(key, self._sentinel) + + def __enter__(self): + _config.update(self.config_dict) + + def __exit__(self, etype, value, traceback): + for key, value in self.original_values.items(): + if value is self._sentinel: + del _config[key] + else: + _config[key] = value diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 0e42b7f5..182a3df5 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -259,6 +259,16 @@ class ChainPredicate(): return True +class ExtendedUrl(): + """URL with attached config dict""" + def __init__(self, url, confdict): + self.value = url + self.config = confdict + + def __str__(self): + return self.value + + class Formatter(): """Custom, trimmed-down version of string.Formatter