diff --git a/docs/gallery-dl-example.conf b/docs/gallery-dl-example.conf index 16445e2e..a3ca7f63 100644 --- a/docs/gallery-dl-example.conf +++ b/docs/gallery-dl-example.conf @@ -100,7 +100,21 @@ "output": { "mode": "terminal", - "logfile": "~/gallery-dl/log.txt" + "log": { + "format": "{name}: {message}", + "level": "info" + }, + "logfile": { + "path": "~/gallery-dl/log.txt", + "mode": "w", + "level": "debug" + }, + "unsupportedfile": { + "path": "~/gallery-dl/unsupported.txt", + "mode": "a", + "format": "{asctime} {message}", + "format-date": "%Y-%m-%d-%H-%M-%S" + } }, "cache": { diff --git a/gallery_dl/__init__.py b/gallery_dl/__init__.py index 8e62f9c7..e698de62 100644 --- a/gallery_dl/__init__.py +++ b/gallery_dl/__init__.py @@ -27,20 +27,72 @@ from . import version, config, option, extractor, job, util, exception __version__ = version.__version__ log = logging.getLogger("gallery-dl") -def initialize_logging(loglevel, formatter): +LOG_FORMAT = "[{name}][{levelname}] {message}" +LOG_FORMAT_DATE = "%Y-%m-%d %H:%M:%S" +LOG_LEVEL = logging.INFO + +def initialize_logging(loglevel): """Setup basic logging functionality before configfiles have been loaded""" # convert levelnames to lowercase for level in (10, 20, 30, 40, 50): name = logging.getLevelName(level) logging.addLevelName(level, name.lower()) # setup basic logging to stderr + formatter = logging.Formatter(LOG_FORMAT, LOG_FORMAT_DATE, "{") handler = logging.StreamHandler() handler.setFormatter(formatter) + handler.setLevel(loglevel) root = logging.getLogger() - root.setLevel(loglevel) + root.setLevel(logging.NOTSET) root.addHandler(handler) +def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL): + """Setup a new logging handler""" + opts = config.interpolate(("output", key)) + if not opts: + return None + if isinstance(opts, str): + opts = {"path": opts} + + path = opts.get("path") + mode = opts.get("mode", "w") + try: + path = util.expand_path(path) + handler = logging.FileHandler(path, mode) + except (OSError, ValueError) as exc: + log.warning("%s: %s", key, exc) + return None + except TypeError as exc: + log.warning("%s: missing or invalid path (%s)", key, exc) + return None + + level = opts.get("level", lvl) + logfmt = opts.get("format", fmt) + datefmt = opts.get("format-date", LOG_FORMAT_DATE) + formatter = logging.Formatter(logfmt, datefmt, "{") + handler.setFormatter(formatter) + handler.setLevel(level) + + return handler + + +def configure_logging_handler(key, handler): + """Configure a logging handler""" + opts = config.interpolate(("output", key)) + if not opts: + return + if isinstance(opts, str): + opts = {"format": opts} + if handler.level == LOG_LEVEL and "level" in opts: + handler.setLevel(opts["level"]) + if "format" in opts or "format-date" in opts: + logfmt = opts.get("format", LOG_FORMAT) + datefmt = opts.get("format-date", LOG_FORMAT_DATE) + formatter = logging.Formatter(logfmt, datefmt, "{") + handler.setFormatter(formatter) + + def replace_std_streams(errors="replace"): """Replace standard streams and set their error handlers to 'errors'""" for name in ("stdout", "stdin", "stderr"): @@ -159,8 +211,7 @@ def main(): parser = option.build_parser() args = parser.parse_args() - formatter = logging.Formatter("[%(name)s][%(levelname)s] %(message)s") - initialize_logging(args.loglevel, formatter) + initialize_logging(args.loglevel) # configuration if args.load_config: @@ -173,17 +224,13 @@ def main(): config.set(key, value) config.set(("_",), {}) - # logfile - logfile = config.interpolate(("output", "logfile")) - if logfile: - try: - path = util.expand_path(logfile) - handler = logging.FileHandler(path, "w") - except OSError as exc: - log.warning("log file: %s", exc) - else: - handler.setFormatter(formatter) - logging.getLogger().addHandler(handler) + # stream logging handler + configure_logging_handler("log", logging.getLogger().handlers[0]) + + # file logging handler + handler = setup_logging_handler("logfile", lvl=args.loglevel) + if handler: + logging.getLogger().addHandler(handler) # loglevels if args.loglevel >= logging.ERROR: @@ -243,13 +290,13 @@ def main(): except OSError as exc: log.warning("input file: %s", exc) - unsupportedfile = config.interpolate(("output", "unsupportedfile")) - if unsupportedfile: - try: - path = util.expand_path(unsupportedfile) - job.Job.ufile = open(path, "w") - except OSError as exc: - log.warning("unsupported-URL file: %s", exc) + # unsupported file logging handler + handler = setup_logging_handler("unsupportedfile", fmt="{message}") + if handler: + ulog = logging.getLogger("unsupported") + ulog.addHandler(handler) + ulog.propagate = False + job.Job.ulog = ulog prepare_range(args.image_range, "image") prepare_range(args.chapter_range, "chapter") diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 3e31e4cd..61755f3a 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -17,7 +17,7 @@ from .extractor.message import Message class Job(): """Base class for Job-types""" - ufile = None + ulog = None def __init__(self, url, parent=None): self.url = url @@ -144,8 +144,8 @@ class Job(): kwdict.update(self.userkwds) def _write_unsupported(self, url): - if self.ufile: - print(url, file=self.ufile, flush=True) + if self.ulog: + self.ulog.info(url) class DownloadJob(Job):