github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[pp:metadata] add 'newline' option (#8439)

Browse Source 
* Add configuration setting to control newline characters in metadata.
* update docs/configuration
* introduce 'open()' method
* add 'newline' test
This commit is contained in:
featherbutt
2025-10-21 09:47:39 -07:00
committed by GitHub
parent 3deb5a413d
commit 06e3126bba
3 changed files with 71 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
docs/configuration.rst
View File

@@ -8225,7 +8225,7 @@ metadata.open
-------------
Type
``string``
Defsult
Default
``"w"``
Description
The ``mode`` in which metadata files get opened.
@@ -8241,7 +8241,7 @@ metadata.encoding
-----------------
Type
``string``
Defsult
Default
``"utf-8"``
Description
Name of the encoding used to encode a file's content.
@@ -8249,6 +8249,29 @@ Description
See the ``encoding`` argument of |open()|_ for further details.
metadata.newline
-----------------
Type
``string``
Default
``null``
Description
The newline sequence used in metadata files.
If ``null``, any ``\n`` characters
written are translated to the system default line separator.
See the ``newline`` argument of |open()|_ for further details.
Supported Values
``null``
Any ``\n`` characters
written are translated to the system default line separator.
``""`` | ``"\n"``
Don't replace newline characters.
``"\r"`` | ``"\r\n"``
Replace newline characters with the given sequence.
metadata.private
----------------
Type

10
gallery_dl/postprocessor/metadata.py
View File

@@ -117,9 +117,15 @@ class MetadataPP(PostProcessor):
self.mtime = options.get("mtime")
self.omode = options.get("open", omode)
self.encoding = options.get("encoding", "utf-8")
self.newline = options.get("newline")
self.skip = options.get("skip", False)
self.meta_path = options.get("metadata-path")
def open(self, path):
return open(path, self.omode,
encoding=self.encoding,
newline=self.newline)
def run(self, pathfmt):
archive = self.archive
if archive and archive.check(pathfmt.kwdict):
@@ -138,11 +144,11 @@ class MetadataPP(PostProcessor):
return
try:
with open(path, self.omode, encoding=self.encoding) as fp:
with self.open(path) as fp:
self.write(fp, pathfmt.kwdict)
except FileNotFoundError:
os.makedirs(directory, exist_ok=True)
with open(path, self.omode, encoding=self.encoding) as fp:
with self.open(path) as fp:
self.write(fp, pathfmt.kwdict)
if archive:

51
test/test_postprocessor.py
View File

@@ -453,7 +453,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realpath}.JSON"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
self.assertEqual(self._output(m), """{
"category": "test",
@@ -473,6 +473,7 @@ class MetadataTest(BasePostprocessorTest):
"indent" : None,
"open" : "a",
"encoding" : "UTF-8",
"newline" : "\r\n",
"extension" : "JSON",
}, {
"public" : "hello ワールド",
@@ -487,7 +488,9 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realpath}.JSON"
m.assert_called_once_with(path, "a", encoding="UTF-8")
m.assert_called_once_with(path, "a", encoding="UTF-8", newline='\r\n')
# Since we mocked the call to open,
# we don't actually see the effect of setting newline.
self.assertEqual(self._output(m), """{\
"_private" : "foo \\u30d0\\u30fc",\
"category" : "test",\
@@ -508,7 +511,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realpath}.txt"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
self.assertEqual(self._output(m), "foo\nbar\nbaz\n")
def test_metadata_tags_split_1(self):
@@ -599,7 +602,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realdirectory}file.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_extfmt_2(self):
self._create({
@@ -611,7 +614,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realdirectory}file.2.EXT-data:tESt"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_directory(self):
self._create({
@@ -622,7 +625,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realdirectory}metadata/file.ext.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_directory_2(self):
self._create({
@@ -634,7 +637,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realdirectory}metadata/file.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_directory_format(self):
self._create(
@@ -646,7 +649,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realdirectory}../json/12500/file.ext.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_directory_empty(self):
self._create(
@@ -657,7 +660,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realdirectory}./file.ext.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_basedirectory(self):
self._create({"base-directory": True})
@@ -666,7 +669,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.basedirectory}file.ext.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_basedirectory_custom(self):
self._create({
@@ -678,7 +681,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = "/home/test/meta/file.ext.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_filename(self):
self._create({
@@ -690,7 +693,7 @@ class MetadataTest(BasePostprocessorTest):
self._trigger()
path = f"{self.pathfmt.realdirectory}test_file__meta_.data"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_meta_path(self):
self._create({
@@ -790,7 +793,7 @@ class MetadataTest(BasePostprocessorTest):
self.assertGreater(len(self._output(m)), 0)
path = f"{self.pathfmt.realdirectory}file.ext.json"
m.assert_called_once_with(path, "w", encoding="utf-8")
m.assert_called_once_with(path, "w", encoding="utf-8", newline=None)
def test_metadata_option_skip_false(self):
self._create({"skip": False})
@@ -802,6 +805,28 @@ class MetadataTest(BasePostprocessorTest):
self.assertTrue(not e.called)
self.assertTrue(m.called)
def test_metadata_option_newline(self):
self._create({
"newline": "\r\n",
"filename" : "data.json",
"directory" : "",
"base-directory": self.dir.name,
})
self._trigger()
path = os.path.join(self.dir.name, "data.json")
with open(path, newline="") as fp:
content = fp.read()
self.assertEqual(content, """\
{\r\n\
"category": "test",\r\n\
"filename": "file",\r\n\
"extension": "ext"\r\n\
}\r\n\
""")
def test_metadata_option_include(self):
self._create(
{"include": ["_private", "filename", "foo"], "sort": True},