From eee78f8148b1ee6c69e811115b2f07d2f64a4dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 26 Sep 2025 21:55:37 +0200 Subject: [PATCH] [pp:python] restore archive functionality fixes regression introduced in 09f0ba8e9cee151bbd03bc3b2c31b97e3fe68ca1 --- gallery_dl/postprocessor/common.py | 8 +++-- gallery_dl/postprocessor/exec.py | 3 +- gallery_dl/postprocessor/metadata.py | 4 ++- gallery_dl/postprocessor/python.py | 7 ++-- test/test_postprocessor.py | 54 ++++++++++++++++++++++++++-- 5 files changed, 67 insertions(+), 9 deletions(-) diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py index 9992c567..82ad3888 100644 --- a/gallery_dl/postprocessor/common.py +++ b/gallery_dl/postprocessor/common.py @@ -21,7 +21,7 @@ class PostProcessor(): def __repr__(self): return self.__class__.__name__ - def _init_archive(self, job, options, prefix=None): + def _archive_init(self, job, options, prefix=None): if archive_path := options.get("archive"): extr = job.extractor @@ -54,11 +54,13 @@ class PostProcessor(): else: self.log.debug( "Using %s archive '%s'", self.name, archive_path) - job.register_hooks({"finalize": self._close_archive}) return True self.archive = None return False - def _close_archive(self, _): + def _archive_register(self, job): + job.register_hooks({"finalize": self._archive_close}) + + def _archive_close(self, _): self.archive.close() diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py index 0bfe1a2f..ef11bff7 100644 --- a/gallery_dl/postprocessor/exec.py +++ b/gallery_dl/postprocessor/exec.py @@ -50,7 +50,8 @@ class ExecPP(PostProcessor): events = events.split(",") job.register_hooks({event: execute for event in events}, options) - self._init_archive(job, options) + if self._archive_init(job, options): + self._archive_register(job) def _prepare_cmd(self, cmd): if isinstance(cmd, str): diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py index a6d2b7f8..90e6e3dc 100644 --- a/gallery_dl/postprocessor/metadata.py +++ b/gallery_dl/postprocessor/metadata.py @@ -110,7 +110,9 @@ class MetadataPP(PostProcessor): events = events.split(",") job.register_hooks({event: self.run for event in events}, options) - self._init_archive(job, options, "_MD_") + if self._archive_init(job, options, "_MD_"): + self._archive_register(job) + self.filter = self._make_filter(options) self.mtime = options.get("mtime") self.omode = options.get("open", omode) diff --git a/gallery_dl/postprocessor/python.py b/gallery_dl/postprocessor/python.py index 66d93438..5011ce8a 100644 --- a/gallery_dl/postprocessor/python.py +++ b/gallery_dl/postprocessor/python.py @@ -26,6 +26,9 @@ class PythonPP(PostProcessor): module = util.import_file(module_name) self.function = getattr(module, function_name) + if archive := self._archive_init(job, options): + self.run = self.run_archive + events = options.get("event") if events is None: events = ("file",) @@ -33,8 +36,8 @@ class PythonPP(PostProcessor): events = events.split(",") job.register_hooks({event: self.run for event in events}, options) - if self._init_archive(job, options): - self.run = self.run_archive + if archive: + self._archive_register(job) def run(self, pathfmt): self.function(pathfmt.kwdict) diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 2902feab..17b36b62 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -21,7 +21,7 @@ from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import extractor, output, path, util, exception # noqa E402 -from gallery_dl import postprocessor, config # noqa E402 +from gallery_dl import postprocessor, config, archive # noqa E402 from gallery_dl.postprocessor.common import PostProcessor # noqa E402 @@ -39,7 +39,7 @@ class FakeJob(): self.get_logger = logging.getLogger self.hooks = collections.defaultdict(list) - def register_hooks(self, hooks, options): + def register_hooks(self, hooks, options=None): for hook, callback in hooks.items(): self.hooks[hook].append(callback) @@ -353,6 +353,23 @@ class ExecTest(BasePostprocessorTest): ) i.wait.assert_called_once_with() + def test_archive(self): + pp = self._create({ + "command": ["echo", "foobar"], + "archive": ":memory:", + "event" : "finalize", + }) + + self.assertIsInstance(pp.archive, archive.DownloadArchive) + + with patch.object(pp.archive, "add") as m_aa, \ + patch.object(pp.archive, "close") as m_ac: + self._trigger(("finalize",)) + pp.archive.close() + + m_aa.assert_called_once_with(self.pathfmt.kwdict) + m_ac.assert_called_once() + class HashTest(BasePostprocessorTest): @@ -811,6 +828,22 @@ class MetadataTest(BasePostprocessorTest): } """) + def test_archive(self): + pp = self._create({ + "archive": ":memory:", + "event" : "finalize", + }) + + self.assertIsInstance(pp.archive, archive.DownloadArchive) + + with patch.object(pp.archive, "add") as m_aa, \ + patch.object(pp.archive, "close") as m_ac: + self._trigger(("finalize",)) + pp.archive.close() + + m_aa.assert_called_once_with(self.pathfmt.kwdict) + m_ac.assert_called_once() + def _output(self, mock): return "".join( call[1][0] @@ -890,6 +923,23 @@ class PythonTest(BasePostprocessorTest): with self.assertRaises(exception.StopExtraction): self._trigger() + def test_archive(self): + pp = self._create({ + "expression": "True", + "archive" : ":memory:", + "event" : "finalize", + }) + + self.assertIsInstance(pp.archive, archive.DownloadArchive) + + with patch.object(pp.archive, "add") as m_aa, \ + patch.object(pp.archive, "close") as m_ac: + self._trigger(("finalize",)) + pp.archive.close() + + m_aa.assert_called_once_with(self.pathfmt.kwdict) + m_ac.assert_called_once() + def _write_module(self, path): with open(path, "w") as fp: fp.write("""