diff --git a/docs/configuration.rst b/docs/configuration.rst index 1c7611ba..36a0f4be 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -4240,6 +4240,20 @@ Description Note: Only applies for ``"mode": "custom"``. +metadata.ascii +-------------- +Type + ``bool`` +Default + ``false`` +Description + Escape all non-ASCII characters. + + See the ``ensure_ascii`` argument of |json.dump()|_ for further details. + + Note: Only applies for ``"mode": "json"`` and ``"jsonl"``. + + metadata.indent --------------- Type @@ -4255,6 +4269,35 @@ Description Note: Only applies for ``"mode": "json"``. +metadata.separators +------------------- +Type + ``list`` with two ``string`` elements +Default + ``[", ", ": "]`` +Description + ```` - ```` pair + to separate JSON keys and values with. + + See the ``separators`` argument of |json.dump()|_ for further details. + + Note: Only applies for ``"mode": "json"`` and ``"jsonl"``. + + +metadata.sort +------------- +Type + ``bool`` +Default + ``false`` +Description + Sort output by `key`. + + See the ``sort_keys`` argument of |json.dump()|_ for further details. + + Note: Only applies for ``"mode": "json"`` and ``"jsonl"``. + + metadata.open ------------- Type diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py index cac78b79..3f5d1d6f 100644 --- a/gallery_dl/postprocessor/metadata.py +++ b/gallery_dl/postprocessor/metadata.py @@ -47,20 +47,12 @@ class MetadataPP(PostProcessor): ext = "txt" elif mode == "jsonl": self.write = self._write_json - self._json_encode = json.JSONEncoder( - ensure_ascii=options.get("ascii", False), - sort_keys=True, indent=None, default=str, - ).encode + self._json_encode = self._make_encoder(options).encode omode = "a" filename = "data.jsonl" else: self.write = self._write_json - self._json_encode = json.JSONEncoder( - ensure_ascii=options.get("ascii", False), - indent=options.get("indent", 4), - sort_keys=True, - default=str, - ).encode + self._json_encode = self._make_encoder(options, 4).encode ext = "json" directory = options.get("directory") @@ -200,5 +192,15 @@ class MetadataPP(PostProcessor): kwdict = util.filter_dict(kwdict) fp.write(self._json_encode(kwdict) + "\n") + @staticmethod + def _make_encoder(options, indent=None): + return json.JSONEncoder( + ensure_ascii=options.get("ascii", False), + sort_keys=options.get("sort", False), + separators=options.get("separators"), + indent=options.get("indent", indent), + check_circular=False, default=str, + ) + __postprocessor__ = MetadataPP diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 63dcfc57..136a0e02 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -176,13 +176,11 @@ class MetadataTest(BasePostprocessorTest): def test_metadata_json(self): pp = self._create({ - "mode" : "json", - "ascii" : True, - "indent" : 2, - "extension": "JSON", + "mode" : "json", + "extension" : "JSON", }, { - "public" : "hello ワールド", - "_private" : "foo バール", + "public" : "hello ワールド", + "_private" : "foo バー", }) self.assertEqual(pp.write , pp._write_json) @@ -194,26 +192,31 @@ class MetadataTest(BasePostprocessorTest): path = self.pathfmt.realpath + ".JSON" m.assert_called_once_with(path, "w", encoding="utf-8") - self.assertEqual(self._output(m), r"""{ - "category": "test", - "extension": "ext", - "filename": "file", - "public": "hello \u30ef\u30fc\u30eb\u30c9" + + if sys.hexversion >= 0x3060000: + # python 3.4 & 3.5 have random order without 'sort: True' + self.assertEqual(self._output(m), """{ + "category": "test", + "filename": "file", + "extension": "ext", + "public": "hello ワールド" } """) def test_metadata_json_options(self): pp = self._create({ - "mode" : "json", - "ascii" : False, - "private" : True, - "indent" : None, - "open" : "a", - "encoding" : "UTF-8", - "extension": "JSON", + "mode" : "json", + "ascii" : True, + "sort" : True, + "separators": [",", " : "], + "private" : True, + "indent" : None, + "open" : "a", + "encoding" : "UTF-8", + "extension" : "JSON", }, { - "public" : "hello ワールド", - "_private" : "foo バール", + "public" : "hello ワールド", + "_private" : "foo バー", }) self.assertEqual(pp.write , pp._write_json) @@ -226,11 +229,11 @@ class MetadataTest(BasePostprocessorTest): path = self.pathfmt.realpath + ".JSON" m.assert_called_once_with(path, "a", encoding="UTF-8") self.assertEqual(self._output(m), """{\ -"_private": "foo バール", \ -"category": "test", \ -"extension": "ext", \ -"filename": "file", \ -"public": "hello ワールド"} +"_private" : "foo \\u30d0\\u30fc",\ +"category" : "test",\ +"extension" : "ext",\ +"filename" : "file",\ +"public" : "hello \\u30ef\\u30fc\\u30eb\\u30c9"} """) def test_metadata_tags(self): @@ -363,7 +366,7 @@ class MetadataTest(BasePostprocessorTest): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_stdout(self): - self._create({"filename": "-", "indent": None}) + self._create({"filename": "-", "indent": None, "sort": True}) with patch("sys.stdout", Mock()) as m: self._trigger()