From ca59bd691c2a2c2925c9b1ee33c445ee49e969a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 20 Nov 2020 22:28:01 +0100 Subject: [PATCH] [postprocessor:metadata] add 'event' and 'filename' options --- gallery_dl/postprocessor/metadata.py | 60 ++++++++++++++++++---------- test/test_postprocessor.py | 16 +++++++- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py index 1f786e7a..7f258c75 100644 --- a/gallery_dl/postprocessor/metadata.py +++ b/gallery_dl/postprocessor/metadata.py @@ -6,7 +6,7 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Write metadata to JSON files""" +"""Write metadata to external files""" from .common import PostProcessor from .. import util @@ -24,7 +24,7 @@ class MetadataPP(PostProcessor): cfmt = options.get("content-format") or options.get("format") if isinstance(cfmt, list): cfmt = "\n".join(cfmt) + "\n" - self.contentfmt = util.Formatter(cfmt).format_map + self._content_fmt = util.Formatter(cfmt).format_map ext = "txt" elif mode == "tags": self.write = self._write_tags @@ -39,47 +39,64 @@ class MetadataPP(PostProcessor): if directory: self._directory = self._directory_custom sep = os.sep + (os.altsep or "") - self.metadir = directory.rstrip(sep) + os.sep + self._metadir = directory.rstrip(sep) + os.sep + filename = options.get("filename") extfmt = options.get("extension-format") - if extfmt: + if filename: self._filename = self._filename_custom - self.extfmt = util.Formatter(extfmt).format_map + self._filename_fmt = util.Formatter(filename).format_map + elif extfmt: + self._filename = self._filename_extfmt + self._extension_fmt = util.Formatter(extfmt).format_map else: self.extension = options.get("extension", ext) - event = "metadata" if options.get("bypost") else "file" - job.hooks[event].append(self.run) + events = options.get("event") + if events is None: + events = ("file",) + elif isinstance(events, str): + events = events.split(",") + for event in events: + job.hooks[event].append(self.run) def run(self, pathfmt): - path = self._directory(pathfmt) + self._filename(pathfmt) - with open(path, "w", encoding="utf-8") as file: - self.write(file, pathfmt.kwdict) + directory = self._directory(pathfmt) + path = directory + self._filename(pathfmt) + + try: + with open(path, "w", encoding="utf-8") as fp: + self.write(fp, pathfmt.kwdict) + except FileNotFoundError: + os.makedirs(directory, exist_ok=True) + with open(path, "w", encoding="utf-8") as fp: + self.write(fp, pathfmt.kwdict) def _directory(self, pathfmt): return pathfmt.realdirectory def _directory_custom(self, pathfmt): - directory = os.path.join(pathfmt.realdirectory, self.metadir) - os.makedirs(directory, exist_ok=True) - return directory + return os.path.join(pathfmt.realdirectory, self._metadir) def _filename(self, pathfmt): - return pathfmt.filename + "." + self.extension + return (pathfmt.filename or "metadata") + "." + self.extension def _filename_custom(self, pathfmt): + return self._filename_fmt(pathfmt.kwdict) + + def _filename_extfmt(self, pathfmt): kwdict = pathfmt.kwdict ext = kwdict["extension"] kwdict["extension"] = pathfmt.extension - kwdict["extension"] = pathfmt.prefix + self.extfmt(kwdict) + kwdict["extension"] = pathfmt.prefix + self._extension_fmt(kwdict) filename = pathfmt.build_filename() kwdict["extension"] = ext return filename - def _write_custom(self, file, kwdict): - file.write(self.contentfmt(kwdict)) + def _write_custom(self, fp, kwdict): + fp.write(self._content_fmt(kwdict)) - def _write_tags(self, file, kwdict): + def _write_tags(self, fp, kwdict): tags = kwdict.get("tags") or kwdict.get("tag_string") if not tags: @@ -91,11 +108,10 @@ class MetadataPP(PostProcessor): taglist = tags.split(" ") tags = taglist - file.write("\n".join(tags)) - file.write("\n") + fp.write("\n".join(tags) + "\n") - def _write_json(self, file, kwdict): - util.dump_json(util.filter_dict(kwdict), file, self.ascii, self.indent) + def _write_json(self, fp, kwdict): + util.dump_json(util.filter_dict(kwdict), fp, self.ascii, self.indent) __postprocessor__ = MetadataPP diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 03ee36ba..74e87427 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -244,7 +244,7 @@ class MetadataTest(BasePostprocessorTest): pp = self._create(pp_info, {"foo": "bar"}) self.assertEqual(pp.write, pp._write_custom) self.assertEqual(pp.extension, "txt") - self.assertTrue(pp.contentfmt) + self.assertTrue(pp._content_fmt) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -261,7 +261,7 @@ class MetadataTest(BasePostprocessorTest): "extension-format": "json", }) - self.assertEqual(pp._filename, pp._filename_custom) + self.assertEqual(pp._filename, pp._filename_extfmt) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -304,6 +304,18 @@ class MetadataTest(BasePostprocessorTest): path = self.pathfmt.realdirectory + "metadata/file.json" m.assert_called_once_with(path, "w", encoding="utf-8") + def test_metadata_filename(self): + self._create({ + "filename" : "{category}_{filename}_meta.data", + "extension-format": "json", + }) + + with patch("builtins.open", mock_open()) as m: + self._trigger() + + path = self.pathfmt.realdirectory + "test_file_meta.data" + m.assert_called_once_with(path, "w", encoding="utf-8") + @staticmethod def _output(mock): return "".join(