provide "extractor" and "job" keys for logging output

This allows for stuff like "{extractor.url}" and "{extractor.category}"
in logging format strings.
Accessing 'extractor' and 'job' in any way will return "None" if those
fields aren't defined, i.e. in general logging messages.

docs/configuration.rst

@@ -1432,7 +1432,14 @@ Description Extended logging output configuration.
 
             * format
                 * Format string for logging messages
-                  (see `LogRecord attributes <https://docs.python.org/3/library/logging.html#logrecord-attributes>`__)
+
+                  In addition to the default
+                  `LogRecord attributes <https://docs.python.org/3/library/logging.html#logrecord-attributes>`__,
+                  it is also possible to access the current
+                  `extractor <https://github.com/mikf/gallery-dl/blob/2e516a1e3e09cb8a9e36a8f6f7e41ce8d4402f5a/gallery_dl/extractor/common.py#L24>`__
+                  and `job <https://github.com/mikf/gallery-dl/blob/2e516a1e3e09cb8a9e36a8f6f7e41ce8d4402f5a/gallery_dl/job.py#L19>`__
+                  objects as well as their attributes
+                  (e.g. ``"{extractor.url}"``)
                 * Default: ``"[{name}][{levelname}] {message}"``
             * format-date
                 * Format string for ``{asctime}`` fields in logging messages

gallery_dl/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2014-2018 Mike Fährmann
+# Copyright 2014-2019 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -21,88 +21,9 @@ if sys.hexversion < 0x3040000:
 
 import json
 import logging
-from . import version, config, option, extractor, job, util, exception
+from . import version, config, option, output, extractor, job, util, exception
 
 __version__ = version.__version__
-log = logging.getLogger("gallery-dl")
-
-LOG_FORMAT = "[{name}][{levelname}] {message}"
-LOG_FORMAT_DATE = "%Y-%m-%d %H:%M:%S"
-LOG_LEVEL = logging.INFO
-
-def initialize_logging(loglevel):
-    """Setup basic logging functionality before configfiles have been loaded"""
-    # convert levelnames to lowercase
-    for level in (10, 20, 30, 40, 50):
-        name = logging.getLevelName(level)
-        logging.addLevelName(level, name.lower())
-    # setup basic logging to stderr
-    formatter = logging.Formatter(LOG_FORMAT, LOG_FORMAT_DATE, "{")
-    handler = logging.StreamHandler()
-    handler.setFormatter(formatter)
-    handler.setLevel(loglevel)
-    root = logging.getLogger()
-    root.setLevel(logging.NOTSET)
-    root.addHandler(handler)
-
-
-def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL):
-    """Setup a new logging handler"""
-    opts = config.interpolate(("output", key))
-    if not opts:
-        return None
-    if not isinstance(opts, dict):
-        opts = {"path": opts}
-
-    path = opts.get("path")
-    mode = opts.get("mode", "w")
-    encoding = opts.get("encoding", "utf-8")
-    try:
-        path = util.expand_path(path)
-        handler = logging.FileHandler(path, mode, encoding)
-    except (OSError, ValueError) as exc:
-        log.warning("%s: %s", key, exc)
-        return None
-    except TypeError as exc:
-        log.warning("%s: missing or invalid path (%s)", key, exc)
-        return None
-
-    level = opts.get("level", lvl)
-    logfmt = opts.get("format", fmt)
-    datefmt = opts.get("format-date", LOG_FORMAT_DATE)
-    formatter = logging.Formatter(logfmt, datefmt, "{")
-    handler.setFormatter(formatter)
-    handler.setLevel(level)
-
-    return handler
-
-
-def configure_logging_handler(key, handler):
-    """Configure a logging handler"""
-    opts = config.interpolate(("output", key))
-    if not opts:
-        return
-    if isinstance(opts, str):
-        opts = {"format": opts}
-    if handler.level == LOG_LEVEL and "level" in opts:
-        handler.setLevel(opts["level"])
-    if "format" in opts or "format-date" in opts:
-        logfmt = opts.get("format", LOG_FORMAT)
-        datefmt = opts.get("format-date", LOG_FORMAT_DATE)
-        formatter = logging.Formatter(logfmt, datefmt, "{")
-        handler.setFormatter(formatter)
-
-
-def replace_std_streams(errors="replace"):
-    """Replace standard streams and set their error handlers to 'errors'"""
-    for name in ("stdout", "stdin", "stderr"):
-        stream = getattr(sys, name)
-        setattr(sys, name, stream.__class__(
-            stream.buffer,
-            errors=errors,
-            newline=stream.newlines,
-            line_buffering=stream.line_buffering,
-        ))
 
 
 def progress(urls, pformat):
@@ -115,7 +36,7 @@ def progress(urls, pformat):
         yield pinfo["url"]
 
 
-def parse_inputfile(file):
+def parse_inputfile(file, log):
     """Filter and process strings from an input file.
 
     Lines starting with '#' and empty lines will be ignored.
@@ -187,12 +108,11 @@ def parse_inputfile(file):
 def main():
     try:
         if sys.stdout.encoding.lower() != "utf-8":
-            replace_std_streams()
+            output.replace_std_streams()
 
         parser = option.build_parser()
         args = parser.parse_args()
-
+        log = output.initialize_logging(args.loglevel)
-        initialize_logging(args.loglevel)
 
         # configuration
         if args.load_config:
@@ -205,10 +125,12 @@ def main():
             config.set(key, value)
 
         # stream logging handler
-        configure_logging_handler("log", logging.getLogger().handlers[0])
+        output.configure_logging_handler(
+            "log", logging.getLogger().handlers[0])
 
         # file logging handler
-        handler = setup_logging_handler("logfile", lvl=args.loglevel)
+        handler = output.setup_logging_handler(
+            "logfile", lvl=args.loglevel)
         if handler:
             logging.getLogger().addHandler(handler)
 
@@ -284,13 +206,14 @@ def main():
                         file = sys.stdin
                     else:
                         file = open(args.inputfile, encoding="utf-8")
-                    urls += parse_inputfile(file)
+                    urls += parse_inputfile(file, log)
                     file.close()
                 except OSError as exc:
                     log.warning("input file: %s", exc)
 
             # unsupported file logging handler
-            handler = setup_logging_handler("unsupportedfile", fmt="{message}")
+            handler = output.setup_logging_handler(
+                "unsupportedfile", fmt="{message}")
             if handler:
                 ulog = logging.getLogger("unsupported")
                 ulog.addHandler(handler)

gallery_dl/job.py

@@ -25,9 +25,11 @@ class Job():
             extr = extractor.find(extr)
         if not extr:
             raise exception.NoExtractorError()
+
         self.extractor = extr
-        self.extractor.log.debug(
+        extr.log.extractor = extr
-            "Using %s for '%s'", extr.__class__.__name__, extr.url)
+        extr.log.job = self
+        extr.log.debug("Using %s for '%s'", extr.__class__.__name__, extr.url)
 
         # url predicates
         self.pred_url = self._prepare_predicates(

gallery_dl/output.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2018 Mike Fährmann
+# Copyright 2015-2019 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -9,9 +9,121 @@
 import os
 import sys
 import shutil
-from . import config
+import logging
+from . import config, util
 
 
+# --------------------------------------------------------------------
+# Logging
+
+LOG_FORMAT = "[{name}][{levelname}] {message}"
+LOG_FORMAT_DATE = "%Y-%m-%d %H:%M:%S"
+LOG_LEVEL = logging.INFO
+
+
+class Logger(logging.Logger):
+    """Custom logger that includes extractor and job info in log records"""
+    extractor = util.NONE
+    job = util.NONE
+
+    def makeRecord(self, name, level, fn, lno, msg, args, exc_info,
+                   func=None, extra=None, sinfo=None,
+                   factory=logging._logRecordFactory):
+        rv = factory(name, level, fn, lno, msg, args, exc_info, func, sinfo)
+        rv.extractor = self.extractor
+        rv.job = self.job
+        return rv
+
+
+def initialize_logging(loglevel):
+    """Setup basic logging functionality before configfiles have been loaded"""
+    # convert levelnames to lowercase
+    for level in (10, 20, 30, 40, 50):
+        name = logging.getLevelName(level)
+        logging.addLevelName(level, name.lower())
+
+    # register custom Logging class
+    logging.Logger.manager.setLoggerClass(Logger)
+
+    # setup basic logging to stderr
+    formatter = logging.Formatter(LOG_FORMAT, LOG_FORMAT_DATE, "{")
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    handler.setLevel(loglevel)
+    root = logging.getLogger()
+    root.setLevel(logging.NOTSET)
+    root.addHandler(handler)
+
+    return logging.getLogger("gallery-dl")
+
+
+def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL):
+    """Setup a new logging handler"""
+    opts = config.interpolate(("output", key))
+    if not opts:
+        return None
+    if not isinstance(opts, dict):
+        opts = {"path": opts}
+
+    path = opts.get("path")
+    mode = opts.get("mode", "w")
+    encoding = opts.get("encoding", "utf-8")
+    try:
+        path = util.expand_path(path)
+        handler = logging.FileHandler(path, mode, encoding)
+    except (OSError, ValueError) as exc:
+        logging.getLogger("gallery-dl").warning(
+            "%s: %s", key, exc)
+        return None
+    except TypeError as exc:
+        logging.getLogger("gallery-dl").warning(
+            "%s: missing or invalid path (%s)", key, exc)
+        return None
+
+    level = opts.get("level", lvl)
+    logfmt = opts.get("format", fmt)
+    datefmt = opts.get("format-date", LOG_FORMAT_DATE)
+    formatter = logging.Formatter(logfmt, datefmt, "{")
+    handler.setFormatter(formatter)
+    handler.setLevel(level)
+
+    return handler
+
+
+def configure_logging_handler(key, handler):
+    """Configure a logging handler"""
+    opts = config.interpolate(("output", key))
+    if not opts:
+        return
+    if isinstance(opts, str):
+        opts = {"format": opts}
+    if handler.level == LOG_LEVEL and "level" in opts:
+        handler.setLevel(opts["level"])
+    if "format" in opts or "format-date" in opts:
+        logfmt = opts.get("format", LOG_FORMAT)
+        datefmt = opts.get("format-date", LOG_FORMAT_DATE)
+        formatter = logging.Formatter(logfmt, datefmt, "{")
+        handler.setFormatter(formatter)
+
+
+# --------------------------------------------------------------------
+# Utility functions
+
+def replace_std_streams(errors="replace"):
+    """Replace standard streams and set their error handlers to 'errors'"""
+    for name in ("stdout", "stdin", "stderr"):
+        stream = getattr(sys, name)
+        setattr(sys, name, stream.__class__(
+            stream.buffer,
+            errors=errors,
+            newline=stream.newlines,
+            line_buffering=stream.line_buffering,
+        ))
+
+
+# --------------------------------------------------------------------
+# Downloader output
+
 def select():
     """Automatically select a suitable output class"""
     pdict = {

gallery_dl/util.py

@@ -140,6 +140,28 @@ CODES = {
 SPECIAL_EXTRACTORS = {"oauth", "recursive", "test"}
 
 
+class UniversalNone():
+    """None-style object that also supports __getattr__ and __getitem__"""
+    def __getattr__(self, _):
+        return self
+
+    def __getitem__(self, _):
+        return self
+
+    @staticmethod
+    def __bool__():
+        return False
+
+    @staticmethod
+    def __str__():
+        return "None"
+
+    __repr__ = __str__
+
+
+NONE = UniversalNone()
+
+
 def build_predicate(predicates):
     if not predicates:
         return lambda url, kwds: True