From 4235d412c4c19a0be3b8f03582f7ab0fe58654a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 10 Mar 2023 22:08:10 +0100 Subject: [PATCH] implement 'actions' continuation of d37e7f48 but more versatile and extendable Example: "actions": [ # change debug messages to info ["debug", "level ~info"], # change exit status to a non-zero value ["info:^No results for", "status |= 1"], # exit with status 2 on 429 ["warning:429", "exit 2"], # restart extractor when no cookies found ["warning:^[Nn]o .*cookies", "restart"] ] --- gallery_dl/actions.py | 112 ++++++++++++++++++++++++++++++++++++++++++ gallery_dl/job.py | 22 +++------ gallery_dl/output.py | 39 +++++++-------- 3 files changed, 138 insertions(+), 35 deletions(-) create mode 100644 gallery_dl/actions.py diff --git a/gallery_dl/actions.py b/gallery_dl/actions.py new file mode 100644 index 00000000..15ca31ec --- /dev/null +++ b/gallery_dl/actions.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- + +# Copyright 2023 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +""" """ + +import re +import sys +import logging +import operator +from . import util, exception + + +def parse(actionspec): + if isinstance(actionspec, dict): + actionspec = actionspec.items() + + actions = {} + actions[logging.DEBUG] = actions_d = [] + actions[logging.INFO] = actions_i = [] + actions[logging.WARNING] = actions_w = [] + actions[logging.ERROR] = actions_e = [] + + for event, spec in actionspec: + level, _, pattern = event.partition(":") + type, _, args = spec.partition(" ") + action = (re.compile(pattern).search, ACTIONS[type](args)) + + level = level.strip() + if not level or level == "*": + actions_d.append(action) + actions_i.append(action) + actions_w.append(action) + actions_e.append(action) + else: + + actions[_level_to_int(level)].append(action) + + return actions + + +def _level_to_int(level): + try: + return logging._nameToLevel[level] + except KeyError: + return int(level) + + +def action_print(opts): + def _print(_): + print(opts) + return _print + + +def action_status(opts): + op, value = re.match(r"\s*([&|^=])=?\s*(\d+)", opts).groups() + + op = { + "&": operator.and_, + "|": operator.or_, + "^": operator.xor, + "=": lambda x, y: y, + }[op] + + value = int(value) + + def _status(args): + args["job"].status = op(args["job"].status, value) + return _status + + +def action_level(opts): + level = _level_to_int(opts.lstrip(" ~=")) + + def _level(args): + args["level"] = level + return _level + + +def action_wait(opts): + def _wait(args): + input("Press Enter to continue") + return _wait + + +def action_restart(opts): + return util.raises(exception.RestartExtraction) + + +def action_exit(opts): + try: + opts = int(opts) + except ValueError: + pass + + def _exit(args): + sys.exit(opts) + return _exit + + +ACTIONS = { + "print" : action_print, + "status" : action_status, + "level" : action_level, + "restart": action_restart, + "wait" : action_wait, + "exit" : action_exit, +} diff --git a/gallery_dl/job.py b/gallery_dl/job.py index 4c4e9259..a64c040f 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -6,7 +6,6 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import re import sys import errno import logging @@ -33,15 +32,11 @@ class Job(): self.kwdict = {} self.status = 0 - hooks = extr.config("hooks") - if hooks: - if isinstance(hooks, dict): - hooks = hooks.items() - self._wrap_logger = self._wrap_logger_hooks - self._logger_hooks = [ - (re.compile(pattern).search, hook) - for pattern, hook in hooks - ] + actions = extr.config("actions") + if actions: + from .actions import parse + self._logger_actions = parse(actions) + self._wrap_logger = self._wrap_logger_actions path_proxy = output.PathfmtProxy(self) self._logger_extra = { @@ -211,11 +206,10 @@ class Job(): return self._wrap_logger(logging.getLogger(name)) def _wrap_logger(self, logger): - return output.LoggerAdapter(logger, self._logger_extra) + return output.LoggerAdapter(logger, self) - def _wrap_logger_hooks(self, logger): - return output.LoggerAdapterEx( - logger, self._logger_extra, self) + def _wrap_logger_actions(self, logger): + return output.LoggerAdapterActions(logger, self) def _write_unsupported(self, url): if self.ulog: diff --git a/gallery_dl/output.py b/gallery_dl/output.py index 7d74b699..1d53851d 100644 --- a/gallery_dl/output.py +++ b/gallery_dl/output.py @@ -12,7 +12,7 @@ import shutil import logging import functools import unicodedata -from . import config, util, formatter, exception +from . import config, util, formatter # -------------------------------------------------------------------- @@ -39,9 +39,9 @@ class LoggerAdapter(): """Trimmed-down version of logging.LoggingAdapter""" __slots__ = ("logger", "extra") - def __init__(self, logger, extra): + def __init__(self, logger, job): self.logger = logger - self.extra = extra + self.extra = job._logger_extra def debug(self, msg, *args, **kwargs): if self.logger.isEnabledFor(logging.DEBUG): @@ -64,12 +64,12 @@ class LoggerAdapter(): self.logger._log(logging.ERROR, msg, args, **kwargs) -class LoggerAdapterEx(): +class LoggerAdapterActions(): - def __init__(self, logger, extra, job): + def __init__(self, logger, job): self.logger = logger - self.extra = extra - self.job = job + self.extra = job._logger_extra + self.actions = job._logger_actions self.debug = functools.partial(self.log, logging.DEBUG) self.info = functools.partial(self.log, logging.INFO) @@ -79,24 +79,21 @@ class LoggerAdapterEx(): def log(self, level, msg, *args, **kwargs): if args: msg = msg % args - args = None - for search, action in self.job._logger_hooks: - match = search(msg) - if match: - if action == "wait+restart": - kwargs["extra"] = self.extra - self.logger._log(level, msg, args, **kwargs) - input("Press Enter to continue") - raise exception.RestartExtraction() - elif action.startswith("~"): - level = logging._nameToLevel[action[1:]] - elif action.startswith("|"): - self.job.status |= int(action[1:]) + actions = self.actions[level] + if actions: + args = self.extra.copy() + args["level"] = level + + for cond, action in actions: + if cond(msg): + action(args) + + level = args["level"] if self.logger.isEnabledFor(level): kwargs["extra"] = self.extra - self.logger._log(level, msg, args, **kwargs) + self.logger._log(level, msg, (), **kwargs) class PathfmtProxy():