[pp:python] restore archive functionality
fixes regression introduced in 09f0ba8e9c
This commit is contained in:
@@ -21,7 +21,7 @@ class PostProcessor():
|
||||
def __repr__(self):
|
||||
return self.__class__.__name__
|
||||
|
||||
def _init_archive(self, job, options, prefix=None):
|
||||
def _archive_init(self, job, options, prefix=None):
|
||||
if archive_path := options.get("archive"):
|
||||
extr = job.extractor
|
||||
|
||||
@@ -54,11 +54,13 @@ class PostProcessor():
|
||||
else:
|
||||
self.log.debug(
|
||||
"Using %s archive '%s'", self.name, archive_path)
|
||||
job.register_hooks({"finalize": self._close_archive})
|
||||
return True
|
||||
|
||||
self.archive = None
|
||||
return False
|
||||
|
||||
def _close_archive(self, _):
|
||||
def _archive_register(self, job):
|
||||
job.register_hooks({"finalize": self._archive_close})
|
||||
|
||||
def _archive_close(self, _):
|
||||
self.archive.close()
|
||||
|
||||
@@ -50,7 +50,8 @@ class ExecPP(PostProcessor):
|
||||
events = events.split(",")
|
||||
job.register_hooks({event: execute for event in events}, options)
|
||||
|
||||
self._init_archive(job, options)
|
||||
if self._archive_init(job, options):
|
||||
self._archive_register(job)
|
||||
|
||||
def _prepare_cmd(self, cmd):
|
||||
if isinstance(cmd, str):
|
||||
|
||||
@@ -110,7 +110,9 @@ class MetadataPP(PostProcessor):
|
||||
events = events.split(",")
|
||||
job.register_hooks({event: self.run for event in events}, options)
|
||||
|
||||
self._init_archive(job, options, "_MD_")
|
||||
if self._archive_init(job, options, "_MD_"):
|
||||
self._archive_register(job)
|
||||
|
||||
self.filter = self._make_filter(options)
|
||||
self.mtime = options.get("mtime")
|
||||
self.omode = options.get("open", omode)
|
||||
|
||||
@@ -26,6 +26,9 @@ class PythonPP(PostProcessor):
|
||||
module = util.import_file(module_name)
|
||||
self.function = getattr(module, function_name)
|
||||
|
||||
if archive := self._archive_init(job, options):
|
||||
self.run = self.run_archive
|
||||
|
||||
events = options.get("event")
|
||||
if events is None:
|
||||
events = ("file",)
|
||||
@@ -33,8 +36,8 @@ class PythonPP(PostProcessor):
|
||||
events = events.split(",")
|
||||
job.register_hooks({event: self.run for event in events}, options)
|
||||
|
||||
if self._init_archive(job, options):
|
||||
self.run = self.run_archive
|
||||
if archive:
|
||||
self._archive_register(job)
|
||||
|
||||
def run(self, pathfmt):
|
||||
self.function(pathfmt.kwdict)
|
||||
|
||||
@@ -21,7 +21,7 @@ from datetime import datetime
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from gallery_dl import extractor, output, path, util, exception # noqa E402
|
||||
from gallery_dl import postprocessor, config # noqa E402
|
||||
from gallery_dl import postprocessor, config, archive # noqa E402
|
||||
from gallery_dl.postprocessor.common import PostProcessor # noqa E402
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ class FakeJob():
|
||||
self.get_logger = logging.getLogger
|
||||
self.hooks = collections.defaultdict(list)
|
||||
|
||||
def register_hooks(self, hooks, options):
|
||||
def register_hooks(self, hooks, options=None):
|
||||
for hook, callback in hooks.items():
|
||||
self.hooks[hook].append(callback)
|
||||
|
||||
@@ -353,6 +353,23 @@ class ExecTest(BasePostprocessorTest):
|
||||
)
|
||||
i.wait.assert_called_once_with()
|
||||
|
||||
def test_archive(self):
|
||||
pp = self._create({
|
||||
"command": ["echo", "foobar"],
|
||||
"archive": ":memory:",
|
||||
"event" : "finalize",
|
||||
})
|
||||
|
||||
self.assertIsInstance(pp.archive, archive.DownloadArchive)
|
||||
|
||||
with patch.object(pp.archive, "add") as m_aa, \
|
||||
patch.object(pp.archive, "close") as m_ac:
|
||||
self._trigger(("finalize",))
|
||||
pp.archive.close()
|
||||
|
||||
m_aa.assert_called_once_with(self.pathfmt.kwdict)
|
||||
m_ac.assert_called_once()
|
||||
|
||||
|
||||
class HashTest(BasePostprocessorTest):
|
||||
|
||||
@@ -811,6 +828,22 @@ class MetadataTest(BasePostprocessorTest):
|
||||
}
|
||||
""")
|
||||
|
||||
def test_archive(self):
|
||||
pp = self._create({
|
||||
"archive": ":memory:",
|
||||
"event" : "finalize",
|
||||
})
|
||||
|
||||
self.assertIsInstance(pp.archive, archive.DownloadArchive)
|
||||
|
||||
with patch.object(pp.archive, "add") as m_aa, \
|
||||
patch.object(pp.archive, "close") as m_ac:
|
||||
self._trigger(("finalize",))
|
||||
pp.archive.close()
|
||||
|
||||
m_aa.assert_called_once_with(self.pathfmt.kwdict)
|
||||
m_ac.assert_called_once()
|
||||
|
||||
def _output(self, mock):
|
||||
return "".join(
|
||||
call[1][0]
|
||||
@@ -890,6 +923,23 @@ class PythonTest(BasePostprocessorTest):
|
||||
with self.assertRaises(exception.StopExtraction):
|
||||
self._trigger()
|
||||
|
||||
def test_archive(self):
|
||||
pp = self._create({
|
||||
"expression": "True",
|
||||
"archive" : ":memory:",
|
||||
"event" : "finalize",
|
||||
})
|
||||
|
||||
self.assertIsInstance(pp.archive, archive.DownloadArchive)
|
||||
|
||||
with patch.object(pp.archive, "add") as m_aa, \
|
||||
patch.object(pp.archive, "close") as m_ac:
|
||||
self._trigger(("finalize",))
|
||||
pp.archive.close()
|
||||
|
||||
m_aa.assert_called_once_with(self.pathfmt.kwdict)
|
||||
m_ac.assert_called_once()
|
||||
|
||||
def _write_module(self, path):
|
||||
with open(path, "w") as fp:
|
||||
fp.write("""
|
||||
|
||||
Reference in New Issue
Block a user