update post processor config capabilities

This change makes it possible to specify just the name of a post processor
in the "postprocessors" list instead of a dict with all of its options.
The options for it will then be taken from inside the "postprocessor"
block similar to "extractor", "downloader", or "output" blocks.

This makes it possible to for example override the default settings for
--write-metadata by specifying a custom "metadata" block, or to set a
custom post processor block ("cbz") and then use it by referencing just
its name in "postprocessors" lists.

{
    "postprocessor":
    {
        "metadata": {
            "name": "metadata",
            "event": "post",
            "filename": "{tweet_id|post_id|id}.json"
        },
        "cbz": {
            "name"       : "zip",
            "compression": "store",
            "extension"  : "cbz"
        }
    }
}
This commit is contained in:
Mike Fährmann
2021-06-05 01:37:47 +02:00
parent 4a747a31a3
commit 83fc4c1098
2 changed files with 11 additions and 8 deletions

View File

@@ -376,17 +376,17 @@ class DownloadJob(Job):
def initialize(self, kwdict=None):
"""Delayed initialization of PathFormat, etc."""
config = self.extractor.config
cfg = self.extractor.config
pathfmt = self.pathfmt = util.PathFormat(self.extractor)
if kwdict:
pathfmt.set_directory(kwdict)
self.sleep = config("sleep")
if not config("download", True):
self.sleep = cfg("sleep")
if not cfg("download", True):
# monkey-patch method to do nothing and always return True
self.download = pathfmt.fix_extension
archive = config("archive")
archive = cfg("archive")
if archive:
path = util.expand_path(archive)
try:
@@ -400,7 +400,7 @@ class DownloadJob(Job):
else:
self.extractor.log.debug("Using download archive '%s'", path)
skip = config("skip", True)
skip = cfg("skip", True)
if skip:
self._skipexc = None
if skip == "enumerate":
@@ -428,7 +428,10 @@ class DownloadJob(Job):
category = self.extractor.category
basecategory = self.extractor.basecategory
pp_conf = config.get((), "postprocessor") or {}
for pp_dict in postprocessors:
if isinstance(pp_dict, str):
pp_dict = pp_conf.get(pp_dict) or {"name": pp_dict}
whitelist = pp_dict.get("whitelist")
if whitelist and category not in whitelist and \

View File

@@ -335,7 +335,7 @@ def build_parser():
postprocessor.add_argument(
"--zip",
dest="postprocessors",
action="append_const", const={"name": "zip"},
action="append_const", const="zip",
help="Store downloaded files in a ZIP archive",
)
postprocessor.add_argument(
@@ -362,7 +362,7 @@ def build_parser():
postprocessor.add_argument(
"--write-metadata",
dest="postprocessors",
action="append_const", const={"name": "metadata"},
action="append_const", const="metadata",
help="Write metadata to separate JSON files",
)
postprocessor.add_argument(
@@ -374,7 +374,7 @@ def build_parser():
postprocessor.add_argument(
"--mtime-from-date",
dest="postprocessors",
action="append_const", const={"name": "mtime"},
action="append_const", const="mtime",
help="Set file modification times according to 'date' metadata",
)
postprocessor.add_argument(