extend input-file format with per-URL config and comments

- see docstring of parse_inputfile() for details
- TODO: unittests, recursion (currently setting for example
  {"extractor": {"key": "value"}} will override the whole "extractor"
  branch instead of merging {"key": "value"} into the already existing
  dictionary)

gallery_dl/__init__.py

@@ -20,6 +20,7 @@ if sys.hexversion < 0x3030000:
     print("Python 3.3+ required", file=sys.stderr)
     sys.exit(1)
 
+import json
 import logging
 from . import version, config, option, extractor, job, util, exception
 
@@ -51,14 +52,6 @@ def progress(urls, pformat):
         yield pinfo["url"]
 
 
-def sanatize_input(file):
-    """Filter and strip strings from an input file"""
-    for line in file:
-        line = line.strip()
-        if line:
-            yield line
-
-
 def prepare_range(rangespec, target):
     if rangespec:
         range = util.optimize_range(util.parse_range(rangespec))
@@ -78,6 +71,56 @@ def prepare_filter(filterexpr, target):
             log.warning(exc)
 
 
+def parse_inputfile(file):
+    """Filter and strip strings from an input file
+
+    Lines starting with '#' and empty lines will be ignored.
+    Lines starting with '{' will be interpreted as JSON-object and
+      its values, while processing the next URL, are going to be
+      applied to the global config.
+    Everything else will be used as potential URL.
+
+    Example input file:
+
+    # this is a comment
+    {"base-directory": "/tmp/", "skip": false}
+    {"more": "multiple objects before an URL will be merged together"}
+    https://example.org/
+
+    # config is back to its initial values
+    https://example.com/index.htm
+    """
+    confdict = None
+
+    for line in file:
+        line = line.strip()
+
+        if not line or line[0] == "#":
+            # empty line or comment
+            continue
+
+        elif line[0] == "{":
+            # url-specfic config spec
+            try:
+                cfd = json.loads(line)
+            except ValueError as exc:
+                log.warning("input file: unable to parse config line: %s",exc)
+                continue
+
+            if confdict:
+                util.combine_dict(confdict, cfd)
+            else:
+                confdict = cfd
+
+        else:
+            # url
+            if confdict:
+                yield util.ExtendedUrl(line, confdict)
+            else:
+                yield line
+            confdict = None
+
+
 def main():
     try:
         parser = option.build_parser()
@@ -162,7 +205,7 @@ def main():
                         file = sys.stdin
                     else:
                         file = open(args.inputfile)
-                    urls += sanatize_input(file)
+                    urls += parse_inputfile(file)
                     file.close()
                 except OSError as exc:
                     log.warning("input file: %s", exc)
@@ -187,7 +230,11 @@ def main():
             for url in urls:
                 try:
                     log.debug("Starting %s for '%s'", jobtype.__name__, url)
-                    jobtype(url).run()
+                    if isinstance(url, util.ExtendedUrl):
+                        with config.apply(url.config):
+                            jobtype(url.value).run()
+                    else:
+                        jobtype(url).run()
                 except exception.NoExtractorError:
                     log.error("No suitable extractor found for '%s'", url)
 

gallery_dl/config.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2017 Mike Fährmann
+# Copyright 2015-2018 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -73,8 +73,8 @@ def load(*files, format="json", strict=False):
 
 
 def clear():
-    """Reset configuration to en empty state"""
-    globals()["_config"].clear()
+    """Reset configuration to an empty state"""
+    _config.clear()
 
 
 def get(keys, default=None, conf=_config):
@@ -124,3 +124,24 @@ def setdefault(keys, value, conf=_config):
             conf[k] = temp
             conf = temp
     return conf.setdefault(keys[-1], value)
+
+
+class apply():
+    """Context Manager to apply a dict to global config"""
+    _sentinel = object()
+
+    def __init__(self, config_dict):
+        self.original_values = {}
+        self.config_dict = config_dict
+        for key, value in config_dict.items():
+            self.original_values[key] = _config.get(key, self._sentinel)
+
+    def __enter__(self):
+        _config.update(self.config_dict)
+
+    def __exit__(self, etype, value, traceback):
+        for key, value in self.original_values.items():
+            if value is self._sentinel:
+                del _config[key]
+            else:
+                _config[key] = value

gallery_dl/util.py

@@ -259,6 +259,16 @@ class ChainPredicate():
         return True
 
 
+class ExtendedUrl():
+    """URL with attached config dict"""
+    def __init__(self, url, confdict):
+        self.value = url
+        self.config = confdict
+
+    def __str__(self):
+        return self.value
+
+
 class Formatter():
     """Custom, trimmed-down version of string.Formatter