github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rework post processor callbacks

Browse Source
This commit is contained in:
Mike Fährmann
2020-11-18 17:11:55 +01:00
parent d6986be8b0
commit 9fffa9c343
10 changed files with 109 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
test/test_postprocessor.py
View File

@@ -15,11 +15,12 @@ from unittest.mock import Mock, mock_open, patch
import logging
import zipfile
import tempfile
import collections
from datetime import datetime, timezone as tz
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from gallery_dl import extractor, output, util # noqa E402
from gallery_dl import postprocessor, util, config # noqa E402
from gallery_dl import postprocessor, config # noqa E402
from gallery_dl.postprocessor.common import PostProcessor # noqa E402
@@ -34,6 +35,7 @@ class FakeJob():
self.pathfmt = util.PathFormat(self.extractor)
self.out = output.NullOutput()
self.get_logger = logging.getLogger
self.hooks = collections.defaultdict(list)
class TestPostprocessorModule(unittest.TestCase):
@@ -78,6 +80,9 @@ class BasePostprocessorTest(unittest.TestCase):
cls.dir.cleanup()
config.clear()
def tearDown(self):
self.job.hooks.clear()
def _create(self, options=None, data=None):
kwdict = {"category": "test", "filename": "file", "extension": "ext"}
if options is None:
@@ -92,6 +97,11 @@ class BasePostprocessorTest(unittest.TestCase):
pp = postprocessor.find(self.__class__.__name__[:-4].lower())
return pp(self.job, options)
def _trigger(self, events=None, *args):
for event in (events or ("prepare", "file")):
for callback in self.job.hooks[event]:
callback(self.pathfmt, *args)
class ClassifyTest(BasePostprocessorTest):
@@ -111,7 +121,7 @@ class ClassifyTest(BasePostprocessorTest):
self.assertEqual(self.pathfmt.realpath, path + "/file.jpg")
with patch("os.makedirs") as mkdirs:
pp.run(self.pathfmt)
self._trigger()
mkdirs.assert_called_once_with(path, exist_ok=True)
def test_classify_noop(self):
@@ -123,7 +133,7 @@ class ClassifyTest(BasePostprocessorTest):
self.assertEqual(self.pathfmt.realpath, rp)
with patch("os.makedirs") as mkdirs:
pp.run(self.pathfmt)
self._trigger()
self.assertEqual(mkdirs.call_count, 0)
def test_classify_custom(self):
@@ -143,7 +153,7 @@ class ClassifyTest(BasePostprocessorTest):
self.assertEqual(self.pathfmt.realpath, path + "/file.foo")
with patch("os.makedirs") as mkdirs:
pp.run(self.pathfmt)
self._trigger()
mkdirs.assert_called_once_with(path, exist_ok=True)
@@ -175,8 +185,7 @@ class MetadataTest(BasePostprocessorTest):
self.assertEqual(pp.extension, "JSON")
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
path = self.pathfmt.realpath + ".JSON"
m.assert_called_once_with(path, "w", encoding="utf-8")
@@ -197,41 +206,37 @@ class MetadataTest(BasePostprocessorTest):
self.assertEqual(pp.extension, "txt")
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
path = self.pathfmt.realpath + ".txt"
m.assert_called_once_with(path, "w", encoding="utf-8")
self.assertEqual(self._output(m), "foo\nbar\nbaz\n")
def test_metadata_tags_split_1(self):
pp = self._create(
self._create(
{"mode": "tags"},
{"tags": "foo, bar, baz"},
)
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
self.assertEqual(self._output(m), "foo\nbar\nbaz\n")
def test_metadata_tags_split_2(self):
pp = self._create(
self._create(
{"mode": "tags"},
{"tags": "foobar1 foobar2 foobarbaz"},
)
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
self.assertEqual(self._output(m), "foobar1\nfoobar2\nfoobarbaz\n")
def test_metadata_tags_tagstring(self):
pp = self._create(
self._create(
{"mode": "tags"},
{"tag_string": "foo, bar, baz"},
)
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
self.assertEqual(self._output(m), "foo\nbar\nbaz\n")
def test_metadata_custom(self):
@@ -242,9 +247,9 @@ class MetadataTest(BasePostprocessorTest):
self.assertTrue(pp.contentfmt)
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
self.assertEqual(self._output(m), "bar\nNone\n")
self.job.hooks.clear()
test({"mode": "custom", "content-format": "{foo}\n{missing}\n"})
test({"mode": "custom", "content-format": ["{foo}", "{missing}"]})
@@ -259,46 +264,42 @@ class MetadataTest(BasePostprocessorTest):
self.assertEqual(pp._filename, pp._filename_custom)
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
path = self.pathfmt.realdirectory + "file.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
def test_metadata_extfmt_2(self):
pp = self._create({
self._create({
"extension-format": "{extension!u}-data:{category:Res/ES/}",
})
self.pathfmt.prefix = "2."
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
path = self.pathfmt.realdirectory + "file.2.EXT-data:tESt"
m.assert_called_once_with(path, "w", encoding="utf-8")
def test_metadata_directory(self):
pp = self._create({
self._create({
"directory": "metadata",
})
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
path = self.pathfmt.realdirectory + "metadata/file.ext.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
def test_metadata_directory_2(self):
pp = self._create({
self._create({
"directory" : "metadata////",
"extension-format": "json",
})
with patch("builtins.open", mock_open()) as m:
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
path = self.pathfmt.realdirectory + "metadata/file.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
@@ -319,21 +320,18 @@ class MtimeTest(BasePostprocessorTest):
self.assertEqual(pp.key, "date")
def test_mtime_datetime(self):
pp = self._create(None, {"date": datetime(1980, 1, 1, tzinfo=tz.utc)})
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._create(None, {"date": datetime(1980, 1, 1, tzinfo=tz.utc)})
self._trigger()
self.assertEqual(self.pathfmt.kwdict["_mtime"], 315532800)
def test_mtime_timestamp(self):
pp = self._create(None, {"date": 315532800})
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._create(None, {"date": 315532800})
self._trigger()
self.assertEqual(self.pathfmt.kwdict["_mtime"], 315532800)
def test_mtime_custom(self):
pp = self._create({"key": "foo"}, {"foo": 315532800})
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._create({"key": "foo"}, {"foo": 315532800})
self._trigger()
self.assertEqual(self.pathfmt.kwdict["_mtime"], 315532800)
@@ -341,8 +339,8 @@ class ZipTest(BasePostprocessorTest):
def test_zip_default(self):
pp = self._create()
self.assertEqual(self.job.hooks["file"][0], pp.write)
self.assertEqual(pp.path, self.pathfmt.realdirectory)
self.assertEqual(pp.run, pp._write)
self.assertEqual(pp.delete, True)
self.assertEqual(pp.args, (
pp.path[:-1] + ".zip", "a", zipfile.ZIP_STORED, True,
@@ -351,8 +349,8 @@ class ZipTest(BasePostprocessorTest):
def test_zip_safe(self):
pp = self._create({"mode": "safe"})
self.assertEqual(self.job.hooks["file"][0], pp.write_safe)
self.assertEqual(pp.path, self.pathfmt.realdirectory)
self.assertEqual(pp.run, pp._write_safe)
self.assertEqual(pp.delete, True)
self.assertEqual(pp.args, (
pp.path[:-1] + ".zip", "a", zipfile.ZIP_STORED, True,
@@ -383,8 +381,7 @@ class ZipTest(BasePostprocessorTest):
self.pathfmt.temppath = file.name
self.pathfmt.filename = name
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
nti = pp.zfile.NameToInfo
self.assertEqual(len(nti), i+1)
@@ -397,12 +394,11 @@ class ZipTest(BasePostprocessorTest):
self.assertIn("file2.ext", nti)
# write the last file a second time (will be skipped)
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
self.assertEqual(len(pp.zfile.NameToInfo), 3)
# close file
pp.run_final(self.pathfmt, 0)
self._trigger(("finalize",), 0)
# reopen to check persistence
with zipfile.ZipFile(pp.zfile.filename) as file:
@@ -428,14 +424,13 @@ class ZipTest(BasePostprocessorTest):
for i in range(3):
self.pathfmt.temppath = self.pathfmt.realdirectory + "file.ext"
self.pathfmt.filename = "file{}.ext".format(i)
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
self._trigger()
# write the last file a second time (will be skipped)
pp.prepare(self.pathfmt)
pp.run(self.pathfmt)
# write the last file a second time (should be skipped)
self._trigger()
pp.run_final(self.pathfmt, 0)
# close file
self._trigger(("finalize",), 0)
self.assertEqual(pp.zfile.write.call_count, 3)
for call in pp.zfile.write.call_args_list: