implement logging options

Standard logging to stderr, logfiles, and unsupported URL files (which
are now handled through the logging module) can now be configured by
setting their respective option keys (log, logfile, unsupportedfile)
to a dict and specifying the following options;

- format:
    format string for logging messages
    available keys: see [1]
    default: "[{name}][{levelname}] {message}"
- format-date:
    format string for {asctime} fields in logging messages
    available keys: see [2]
    default: "%Y-%m-%d %H:%M:%S"
- level:
    the lowercase levelname until which the logger should activate;
    available levels are debug, info, warning, error, exception
    default: "info"
- path:
    path of the file to be written to
- mode:
    'mode' argument when opening the specified file
    can be either "w" to truncate the file or "a" to append to it (see [3])

If 'output.log', '.logfile', or '.unsupportedfile' is a string, it will
be interpreted, as it has been, as the filepath
(or as format string for .log)

[1] https://docs.python.org/3/library/logging.html#logrecord-attributes
[2] https://docs.python.org/3/library/time.html#time.strftime
[3] https://docs.python.org/3/library/functions.html#open

docs/gallery-dl-example.conf

@@ -100,7 +100,21 @@
     "output":
     {
         "mode": "terminal",
-        "logfile": "~/gallery-dl/log.txt"
+        "log": {
+            "format": "{name}: {message}",
+            "level": "info"
+        },
+        "logfile": {
+            "path": "~/gallery-dl/log.txt",
+            "mode": "w",
+            "level": "debug"
+        },
+        "unsupportedfile": {
+            "path": "~/gallery-dl/unsupported.txt",
+            "mode": "a",
+            "format": "{asctime} {message}",
+            "format-date": "%Y-%m-%d-%H-%M-%S"
+        }
     },
 
     "cache": {

gallery_dl/__init__.py

@@ -27,20 +27,72 @@ from . import version, config, option, extractor, job, util, exception
 __version__ = version.__version__
 log = logging.getLogger("gallery-dl")
 
-def initialize_logging(loglevel, formatter):
+LOG_FORMAT = "[{name}][{levelname}] {message}"
+LOG_FORMAT_DATE = "%Y-%m-%d %H:%M:%S"
+LOG_LEVEL = logging.INFO
+
+def initialize_logging(loglevel):
     """Setup basic logging functionality before configfiles have been loaded"""
     # convert levelnames to lowercase
     for level in (10, 20, 30, 40, 50):
         name = logging.getLevelName(level)
         logging.addLevelName(level, name.lower())
     # setup basic logging to stderr
+    formatter = logging.Formatter(LOG_FORMAT, LOG_FORMAT_DATE, "{")
     handler = logging.StreamHandler()
     handler.setFormatter(formatter)
+    handler.setLevel(loglevel)
     root = logging.getLogger()
-    root.setLevel(loglevel)
+    root.setLevel(logging.NOTSET)
     root.addHandler(handler)
 
 
+def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL):
+    """Setup a new logging handler"""
+    opts = config.interpolate(("output", key))
+    if not opts:
+        return None
+    if isinstance(opts, str):
+        opts = {"path": opts}
+
+    path = opts.get("path")
+    mode = opts.get("mode", "w")
+    try:
+        path = util.expand_path(path)
+        handler = logging.FileHandler(path, mode)
+    except (OSError, ValueError) as exc:
+        log.warning("%s: %s", key, exc)
+        return None
+    except TypeError as exc:
+        log.warning("%s: missing or invalid path (%s)", key, exc)
+        return None
+
+    level = opts.get("level", lvl)
+    logfmt = opts.get("format", fmt)
+    datefmt = opts.get("format-date", LOG_FORMAT_DATE)
+    formatter = logging.Formatter(logfmt, datefmt, "{")
+    handler.setFormatter(formatter)
+    handler.setLevel(level)
+
+    return handler
+
+
+def configure_logging_handler(key, handler):
+    """Configure a logging handler"""
+    opts = config.interpolate(("output", key))
+    if not opts:
+        return
+    if isinstance(opts, str):
+        opts = {"format": opts}
+    if handler.level == LOG_LEVEL and "level" in opts:
+        handler.setLevel(opts["level"])
+    if "format" in opts or "format-date" in opts:
+        logfmt = opts.get("format", LOG_FORMAT)
+        datefmt = opts.get("format-date", LOG_FORMAT_DATE)
+        formatter = logging.Formatter(logfmt, datefmt, "{")
+        handler.setFormatter(formatter)
+
+
 def replace_std_streams(errors="replace"):
     """Replace standard streams and set their error handlers to 'errors'"""
     for name in ("stdout", "stdin", "stderr"):
@@ -159,8 +211,7 @@ def main():
         parser = option.build_parser()
         args = parser.parse_args()
 
-        formatter = logging.Formatter("[%(name)s][%(levelname)s] %(message)s")
+        initialize_logging(args.loglevel)
-        initialize_logging(args.loglevel, formatter)
 
         # configuration
         if args.load_config:
@@ -173,17 +224,13 @@ def main():
             config.set(key, value)
         config.set(("_",), {})
 
-        # logfile
+        # stream logging handler
-        logfile = config.interpolate(("output", "logfile"))
+        configure_logging_handler("log", logging.getLogger().handlers[0])
-        if logfile:
+
-            try:
+        # file logging handler
-                path = util.expand_path(logfile)
+        handler = setup_logging_handler("logfile", lvl=args.loglevel)
-                handler = logging.FileHandler(path, "w")
+        if handler:
-            except OSError as exc:
+            logging.getLogger().addHandler(handler)
-                log.warning("log file: %s", exc)
-            else:
-                handler.setFormatter(formatter)
-                logging.getLogger().addHandler(handler)
 
         # loglevels
         if args.loglevel >= logging.ERROR:
@@ -243,13 +290,13 @@ def main():
                 except OSError as exc:
                     log.warning("input file: %s", exc)
 
-            unsupportedfile = config.interpolate(("output", "unsupportedfile"))
+            # unsupported file logging handler
-            if unsupportedfile:
+            handler = setup_logging_handler("unsupportedfile", fmt="{message}")
-                try:
+            if handler:
-                    path = util.expand_path(unsupportedfile)
+                ulog = logging.getLogger("unsupported")
-                    job.Job.ufile = open(path, "w")
+                ulog.addHandler(handler)
-                except OSError as exc:
+                ulog.propagate = False
-                    log.warning("unsupported-URL file: %s", exc)
+                job.Job.ulog = ulog
 
             prepare_range(args.image_range, "image")
             prepare_range(args.chapter_range, "chapter")

gallery_dl/job.py

@@ -17,7 +17,7 @@ from .extractor.message import Message
 
 class Job():
     """Base class for Job-types"""
-    ufile = None
+    ulog = None
 
     def __init__(self, url, parent=None):
         self.url = url
@@ -144,8 +144,8 @@ class Job():
             kwdict.update(self.userkwds)
 
     def _write_unsupported(self, url):
-        if self.ufile:
+        if self.ulog:
-            print(url, file=self.ufile, flush=True)
+            self.ulog.info(url)
 
 
 class DownloadJob(Job):