diff --git a/docs/configuration.rst b/docs/configuration.rst index f05cb487..e4d13892 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -8225,7 +8225,7 @@ metadata.open ------------- Type ``string`` -Defsult +Default ``"w"`` Description The ``mode`` in which metadata files get opened. @@ -8241,7 +8241,7 @@ metadata.encoding ----------------- Type ``string`` -Defsult +Default ``"utf-8"`` Description Name of the encoding used to encode a file's content. @@ -8249,6 +8249,29 @@ Description See the ``encoding`` argument of |open()|_ for further details. +metadata.newline +----------------- +Type + ``string`` +Default + ``null`` +Description + The newline sequence used in metadata files. + + If ``null``, any ``\n`` characters + written are translated to the system default line separator. + + See the ``newline`` argument of |open()|_ for further details. +Supported Values + ``null`` + Any ``\n`` characters + written are translated to the system default line separator. + ``""`` | ``"\n"`` + Don't replace newline characters. + ``"\r"`` | ``"\r\n"`` + Replace newline characters with the given sequence. + + metadata.private ---------------- Type diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py index 90e6e3dc..0017b5b5 100644 --- a/gallery_dl/postprocessor/metadata.py +++ b/gallery_dl/postprocessor/metadata.py @@ -117,9 +117,15 @@ class MetadataPP(PostProcessor): self.mtime = options.get("mtime") self.omode = options.get("open", omode) self.encoding = options.get("encoding", "utf-8") + self.newline = options.get("newline") self.skip = options.get("skip", False) self.meta_path = options.get("metadata-path") + def open(self, path): + return open(path, self.omode, + encoding=self.encoding, + newline=self.newline) + def run(self, pathfmt): archive = self.archive if archive and archive.check(pathfmt.kwdict): @@ -138,11 +144,11 @@ class MetadataPP(PostProcessor): return try: - with open(path, self.omode, encoding=self.encoding) as fp: + with self.open(path) as fp: self.write(fp, pathfmt.kwdict) except FileNotFoundError: os.makedirs(directory, exist_ok=True) - with open(path, self.omode, encoding=self.encoding) as fp: + with self.open(path) as fp: self.write(fp, pathfmt.kwdict) if archive: diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 5d52e1db..ac107f71 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -453,7 +453,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realpath}.JSON" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) self.assertEqual(self._output(m), """{ "category": "test", @@ -473,6 +473,7 @@ class MetadataTest(BasePostprocessorTest): "indent" : None, "open" : "a", "encoding" : "UTF-8", + "newline" : "\r\n", "extension" : "JSON", }, { "public" : "hello ワールド", @@ -487,7 +488,9 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realpath}.JSON" - m.assert_called_once_with(path, "a", encoding="UTF-8") + m.assert_called_once_with(path, "a", encoding="UTF-8", newline='\r\n') + # Since we mocked the call to open, + # we don't actually see the effect of setting newline. self.assertEqual(self._output(m), """{\ "_private" : "foo \\u30d0\\u30fc",\ "category" : "test",\ @@ -508,7 +511,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realpath}.txt" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) self.assertEqual(self._output(m), "foo\nbar\nbaz\n") def test_metadata_tags_split_1(self): @@ -599,7 +602,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}file.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_extfmt_2(self): self._create({ @@ -611,7 +614,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}file.2.EXT-data:tESt" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_directory(self): self._create({ @@ -622,7 +625,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}metadata/file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_directory_2(self): self._create({ @@ -634,7 +637,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}metadata/file.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_directory_format(self): self._create( @@ -646,7 +649,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}../json/12500/file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_directory_empty(self): self._create( @@ -657,7 +660,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}./file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_basedirectory(self): self._create({"base-directory": True}) @@ -666,7 +669,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.basedirectory}file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_basedirectory_custom(self): self._create({ @@ -678,7 +681,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = "/home/test/meta/file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_filename(self): self._create({ @@ -690,7 +693,7 @@ class MetadataTest(BasePostprocessorTest): self._trigger() path = f"{self.pathfmt.realdirectory}test_file__meta_.data" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_meta_path(self): self._create({ @@ -790,7 +793,7 @@ class MetadataTest(BasePostprocessorTest): self.assertGreater(len(self._output(m)), 0) path = f"{self.pathfmt.realdirectory}file.ext.json" - m.assert_called_once_with(path, "w", encoding="utf-8") + m.assert_called_once_with(path, "w", encoding="utf-8", newline=None) def test_metadata_option_skip_false(self): self._create({"skip": False}) @@ -802,6 +805,28 @@ class MetadataTest(BasePostprocessorTest): self.assertTrue(not e.called) self.assertTrue(m.called) + def test_metadata_option_newline(self): + self._create({ + "newline": "\r\n", + "filename" : "data.json", + "directory" : "", + "base-directory": self.dir.name, + }) + + self._trigger() + + path = os.path.join(self.dir.name, "data.json") + with open(path, newline="") as fp: + content = fp.read() + + self.assertEqual(content, """\ +{\r\n\ + "category": "test",\r\n\ + "filename": "file",\r\n\ + "extension": "ext"\r\n\ +}\r\n\ +""") + def test_metadata_option_include(self): self._create( {"include": ["_private", "filename", "foo"], "sort": True},