diff --git a/gallery_dl/config.py b/gallery_dl/config.py
index 0f2d1f19..819347da 100644
--- a/gallery_dl/config.py
+++ b/gallery_dl/config.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2015-2021 Mike Fährmann
+# Copyright 2015-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,7 +9,6 @@
"""Global configuration module"""
import sys
-import json
import os.path
import logging
from . import util
@@ -55,18 +54,18 @@ def load(files=None, strict=False, fmt="json"):
if fmt == "yaml":
try:
import yaml
- parsefunc = yaml.safe_load
+ load = yaml.safe_load
except ImportError:
log.error("Could not import 'yaml' module")
return
else:
- parsefunc = json.load
+ load = util.json_loads
for pathfmt in files or _default_configs:
path = util.expand_path(pathfmt)
try:
with open(path, encoding="utf-8") as file:
- confdict = parsefunc(file)
+ conf = load(file.read())
except OSError as exc:
if strict:
log.error(exc)
@@ -77,9 +76,9 @@ def load(files=None, strict=False, fmt="json"):
sys.exit(2)
else:
if not _config:
- _config.update(confdict)
+ _config.update(conf)
else:
- util.combine_dict(_config, confdict)
+ util.combine_dict(_config, conf)
_files.append(pathfmt)
diff --git a/gallery_dl/cookies.py b/gallery_dl/cookies.py
index f18cc475..e2a7e484 100644
--- a/gallery_dl/cookies.py
+++ b/gallery_dl/cookies.py
@@ -12,7 +12,6 @@
import binascii
import contextlib
import ctypes
-import json
import logging
import os
import shutil
@@ -24,7 +23,7 @@ import tempfile
from datetime import datetime, timedelta, timezone
from hashlib import pbkdf2_hmac
from http.cookiejar import Cookie
-from . import aes, text
+from . import aes, text, util
SUPPORTED_BROWSERS_CHROMIUM = {
@@ -169,8 +168,8 @@ def _firefox_cookies_database(profile=None, container=None):
os.path.dirname(path), "containers.json")
try:
- with open(containers_path) as containers:
- identities = json.load(containers)["identities"]
+ with open(containers_path) as file:
+ identities = util.json_loads(file.read())["identities"]
except OSError:
logger.error("Unable to read Firefox container database at %s",
containers_path)
@@ -716,8 +715,8 @@ def _get_windows_v10_key(browser_root):
logger.error("could not find local state file")
return None
logger.debug("Found local state file at '%s'", path)
- with open(path, encoding="utf8") as f:
- data = json.load(f)
+ with open(path, encoding="utf-8") as file:
+ data = util.json_loads(file.read())
try:
base64_key = data["os_crypt"]["encrypted_key"]
except KeyError:
diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py
index fed49913..26ac8b21 100644
--- a/gallery_dl/extractor/8muses.py
+++ b/gallery_dl/extractor/8muses.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2019-2022 Mike Fährmann
+# Copyright 2019-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,7 @@
"""Extractors for https://comics.8muses.com/"""
from .common import Extractor, Message
-from .. import text
-import json
+from .. import text, util
class _8musesAlbumExtractor(Extractor):
@@ -131,7 +130,7 @@ class _8musesAlbumExtractor(Extractor):
@staticmethod
def _unobfuscate(data):
- return json.loads("".join([
+ return util.json_loads("".join([
chr(33 + (ord(c) + 14) % 94) if "!" <= c <= "~" else c
for c in text.unescape(data.strip("\t\n\r !"))
]))
diff --git a/gallery_dl/extractor/bbc.py b/gallery_dl/extractor/bbc.py
index 1b49d6a5..638fedc2 100644
--- a/gallery_dl/extractor/bbc.py
+++ b/gallery_dl/extractor/bbc.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2021 Mike Fährmann
+# Copyright 2021-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -10,7 +10,6 @@
from .common import GalleryExtractor, Extractor, Message
from .. import text, util
-import json
BASE_PATTERN = r"(?:https?://)?(?:www\.)?bbc\.co\.uk(/programmes/"
@@ -38,7 +37,7 @@ class BbcGalleryExtractor(GalleryExtractor):
)
def metadata(self, page):
- data = json.loads(text.extr(
+ data = util.json_loads(text.extr(
page, ''))
return {
"programme": self.gallery_url.split("/")[4],
diff --git a/gallery_dl/extractor/bcy.py b/gallery_dl/extractor/bcy.py
index 44d60650..d6adb4eb 100644
--- a/gallery_dl/extractor/bcy.py
+++ b/gallery_dl/extractor/bcy.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2020-2022 Mike Fährmann
+# Copyright 2020-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,7 @@
"""Extractors for https://bcy.net/"""
from .common import Extractor, Message
-from .. import text, exception
-import json
+from .. import text, util, exception
import re
@@ -100,9 +99,9 @@ class BcyExtractor(Extractor):
.replace('\\\\u002F', '/')
.replace('\\"', '"'))
try:
- return json.loads(data)["detail"]
+ return util.json_loads(data)["detail"]
except ValueError:
- return json.loads(data.replace('\\"', '"'))["detail"]
+ return util.json_loads(data.replace('\\"', '"'))["detail"]
class BcyUserExtractor(BcyExtractor):
diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py
index 6da61752..1469aad9 100644
--- a/gallery_dl/extractor/behance.py
+++ b/gallery_dl/extractor/behance.py
@@ -9,8 +9,7 @@
"""Extractors for https://www.behance.net/"""
from .common import Extractor, Message
-from .. import text
-import json
+from .. import text, util
class BehanceExtractor(Extractor):
@@ -120,7 +119,7 @@ class BehanceGalleryExtractor(BehanceExtractor):
}
page = self.request(url, cookies=cookies).text
- data = json.loads(text.extr(
+ data = util.json_loads(text.extr(
page, 'id="beconfig-store_state">', ''))
return self._update(data["project"]["project"])
diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py
index 8a1a42eb..56010c20 100644
--- a/gallery_dl/extractor/blogger.py
+++ b/gallery_dl/extractor/blogger.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2019-2022 Mike Fährmann
+# Copyright 2019-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,7 @@
"""Extractors for Blogger blogs"""
from .common import Extractor, Message
-from .. import text
-import json
+from .. import text, util
import re
BASE_PATTERN = (
@@ -61,7 +60,7 @@ class BloggerExtractor(Extractor):
page = self.request(post["url"]).text
for url in findall_video(page):
page = self.request(url).text
- video_config = json.loads(text.extr(
+ video_config = util.json_loads(text.extr(
page, 'var VIDEO_CONFIG =', '\n'))
files.append(max(
video_config["streams"],
diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py
index 1c339a9e..250b40a1 100644
--- a/gallery_dl/extractor/bunkr.py
+++ b/gallery_dl/extractor/bunkr.py
@@ -9,8 +9,7 @@
"""Extractors for https://bunkr.ru/"""
from .lolisafe import LolisafeAlbumExtractor
-from .. import text
-import json
+from .. import text, util
class BunkrAlbumExtractor(LolisafeAlbumExtractor):
@@ -49,7 +48,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor):
root = self.root
try:
- data = json.loads(text.extr(
+ data = util.json_loads(text.extr(
self.request(root + "/a/" + self.album_id).text,
'id="__NEXT_DATA__" type="application/json">', '<'))
album = data["props"]["pageProps"]["album"]
diff --git a/gallery_dl/extractor/dynastyscans.py b/gallery_dl/extractor/dynastyscans.py
index d78f25be..59e8c90d 100644
--- a/gallery_dl/extractor/dynastyscans.py
+++ b/gallery_dl/extractor/dynastyscans.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2015-2022 Mike Fährmann
+# Copyright 2015-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,7 @@
"""Extractors for https://dynasty-scans.com/"""
from .common import ChapterExtractor, MangaExtractor, Extractor, Message
-from .. import text
-import json
+from .. import text, util
import re
BASE_PATTERN = r"(?:https?://)?(?:www\.)?dynasty-scans\.com"
@@ -86,7 +85,7 @@ class DynastyscansChapterExtractor(DynastyscansBase, ChapterExtractor):
data = text.extr(page, "var pages = ", ";\n")
return [
(self.root + img["image"], None)
- for img in json.loads(data)
+ for img in util.json_loads(data)
]
diff --git a/gallery_dl/extractor/fallenangels.py b/gallery_dl/extractor/fallenangels.py
index 57587b60..0503dcfa 100644
--- a/gallery_dl/extractor/fallenangels.py
+++ b/gallery_dl/extractor/fallenangels.py
@@ -6,11 +6,10 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
-"""Extract manga-chapters from https://www.fascans.com/"""
+"""Extractors for https://www.fascans.com/"""
from .common import ChapterExtractor, MangaExtractor
from .. import text, util
-import json
class FallenangelsChapterExtractor(ChapterExtractor):
@@ -56,7 +55,7 @@ class FallenangelsChapterExtractor(ChapterExtractor):
def images(page):
return [
(img["page_image"], None)
- for img in json.loads(
+ for img in util.json_loads(
text.extr(page, "var pages = ", ";")
)
]
diff --git a/gallery_dl/extractor/fantia.py b/gallery_dl/extractor/fantia.py
index 476fdeb2..13dfeada 100644
--- a/gallery_dl/extractor/fantia.py
+++ b/gallery_dl/extractor/fantia.py
@@ -7,8 +7,7 @@
"""Extractors for https://fantia.jp/"""
from .common import Extractor, Message
-from .. import text
-import json
+from .. import text, util
class FantiaExtractor(Extractor):
@@ -117,7 +116,7 @@ class FantiaExtractor(Extractor):
yield self.root+"/"+content["download_uri"], post
if content["category"] == "blog" and "comment" in content:
- comment_json = json.loads(content["comment"])
+ comment_json = util.json_loads(content["comment"])
ops = comment_json.get("ops", ())
# collect blogpost text first
diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py
index 2290cc25..4a38fb4f 100644
--- a/gallery_dl/extractor/foolslide.py
+++ b/gallery_dl/extractor/foolslide.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2016-2022 Mike Fährmann
+# Copyright 2016-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -10,7 +10,6 @@
from .common import BaseExtractor, Message
from .. import text, util
-import json
class FoolslideExtractor(BaseExtractor):
@@ -106,7 +105,7 @@ class FoolslideChapterExtractor(FoolslideExtractor):
})
def images(self, page):
- return json.loads(text.extr(page, "var pages = ", ";"))
+ return util.json_loads(text.extr(page, "var pages = ", ";"))
class FoolslideMangaExtractor(FoolslideExtractor):
diff --git a/gallery_dl/extractor/hbrowse.py b/gallery_dl/extractor/hbrowse.py
index 43479c6f..5b561ead 100644
--- a/gallery_dl/extractor/hbrowse.py
+++ b/gallery_dl/extractor/hbrowse.py
@@ -1,16 +1,15 @@
# -*- coding: utf-8 -*-
-# Copyright 2015-2019 Mike Fährmann
+# Copyright 2015-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
-"""Extract images from https://www.hbrowse.com/"""
+"""Extractors for https://www.hbrowse.com/"""
from .common import ChapterExtractor, MangaExtractor
-from .. import text, exception
-import json
+from .. import text, util, exception
class HbrowseBase():
@@ -68,7 +67,7 @@ class HbrowseChapterExtractor(HbrowseBase, ChapterExtractor):
def images(self, page):
base = self.root + "/data" + self.path
json_data = text.extract(page, ';list = ', ',"zzz"')[0] + "]"
- return [(base + name, None) for name in json.loads(json_data)]
+ return [(base + name, None) for name in util.json_loads(json_data)]
class HbrowseMangaExtractor(HbrowseBase, MangaExtractor):
diff --git a/gallery_dl/extractor/hentai2read.py b/gallery_dl/extractor/hentai2read.py
index dc4e31d2..e771a4fc 100644
--- a/gallery_dl/extractor/hentai2read.py
+++ b/gallery_dl/extractor/hentai2read.py
@@ -9,8 +9,7 @@
"""Extractors for https://hentai2read.com/"""
from .common import ChapterExtractor, MangaExtractor
-from .. import text
-import json
+from .. import text, util
import re
@@ -78,7 +77,7 @@ class Hentai2readChapterExtractor(Hentai2readBase, ChapterExtractor):
images = text.extract(page, "'images' : ", ",\n")[0]
return [
("https://hentaicdn.com/hentai" + part, None)
- for part in json.loads(images)
+ for part in util.json_loads(images)
]
diff --git a/gallery_dl/extractor/hentaifox.py b/gallery_dl/extractor/hentaifox.py
index 0327f563..ed8576f1 100644
--- a/gallery_dl/extractor/hentaifox.py
+++ b/gallery_dl/extractor/hentaifox.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2019-2021 Mike Fährmann
+# Copyright 2019-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,7 @@
"""Extractors for https://hentaifox.com/"""
from .common import GalleryExtractor, Extractor, Message
-from .. import text
-import json
+from .. import text, util
class HentaifoxBase():
@@ -90,7 +89,7 @@ class HentaifoxGalleryExtractor(HentaifoxBase, GalleryExtractor):
server1 = "https://i.hentaifox.com"
server2 = "https://i2.hentaifox.com"
- for num, image in json.loads(data).items():
+ for num, image in util.json_loads(data).items():
ext, width, height = image.split(",")
path = urlfmt(num, extmap[ext])
append((server1 + path, {
diff --git a/gallery_dl/extractor/hentaihand.py b/gallery_dl/extractor/hentaihand.py
index bf9e4641..0617330b 100644
--- a/gallery_dl/extractor/hentaihand.py
+++ b/gallery_dl/extractor/hentaihand.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2020-2022 Mike Fährmann
+# Copyright 2020-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -10,7 +10,6 @@
from .common import GalleryExtractor, Extractor, Message
from .. import text, util
-import json
class HentaihandGalleryExtractor(GalleryExtractor):
@@ -46,7 +45,7 @@ class HentaihandGalleryExtractor(GalleryExtractor):
GalleryExtractor.__init__(self, match, url)
def metadata(self, page):
- info = json.loads(page)
+ info = util.json_loads(page)
data = {
"gallery_id" : text.parse_int(info["id"]),
"title" : info["title"],
diff --git a/gallery_dl/extractor/hentaihere.py b/gallery_dl/extractor/hentaihere.py
index 38ec77c6..2297cc06 100644
--- a/gallery_dl/extractor/hentaihere.py
+++ b/gallery_dl/extractor/hentaihere.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2016-2022 Mike Fährmann
+# Copyright 2016-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,7 @@
"""Extractors for https://hentaihere.com/"""
from .common import ChapterExtractor, MangaExtractor
-from .. import text
-import json
+from .. import text, util
import re
@@ -80,7 +79,7 @@ class HentaihereChapterExtractor(HentaihereBase, ChapterExtractor):
images = text.extr(page, "var rff_imageList = ", ";")
return [
("https://hentaicdn.com/hentai" + part, None)
- for part in json.loads(images)
+ for part in util.json_loads(images)
]
diff --git a/gallery_dl/extractor/hitomi.py b/gallery_dl/extractor/hitomi.py
index 44459ceb..4e8d1cae 100644
--- a/gallery_dl/extractor/hitomi.py
+++ b/gallery_dl/extractor/hitomi.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2015-2022 Mike Fährmann
+# Copyright 2015-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -13,7 +13,6 @@ from .nozomi import decode_nozomi
from ..cache import memcache
from .. import text, util
import string
-import json
import re
@@ -75,7 +74,7 @@ class HitomiGalleryExtractor(GalleryExtractor):
self.root, gid)
def metadata(self, page):
- self.info = info = json.loads(page.partition("=")[2])
+ self.info = info = util.json_loads(page.partition("=")[2])
iget = info.get
language = iget("language")
diff --git a/gallery_dl/extractor/imagefap.py b/gallery_dl/extractor/imagefap.py
index 378e26db..497f1efa 100644
--- a/gallery_dl/extractor/imagefap.py
+++ b/gallery_dl/extractor/imagefap.py
@@ -9,8 +9,7 @@
"""Extractors for https://www.imagefap.com/"""
from .common import Extractor, Message
-from .. import text, exception
-import json
+from .. import text, util, exception
BASE_PATTERN = r"(?:https?://)?(?:www\.|beta\.)?imagefap\.com"
@@ -173,7 +172,7 @@ class ImagefapImageExtractor(ImagefapExtractor):
page, 'id="imageid_input" value="', '"', pos)
gallery_id, pos = text.extract(
page, 'id="galleryid_input" value="', '"', pos)
- info = json.loads(info)
+ info = util.json_loads(info)
url = info["contentUrl"]
return url, text.nameext_from_url(url, {
diff --git a/gallery_dl/extractor/imgbb.py b/gallery_dl/extractor/imgbb.py
index 49082d85..a2210752 100644
--- a/gallery_dl/extractor/imgbb.py
+++ b/gallery_dl/extractor/imgbb.py
@@ -9,9 +9,8 @@
"""Extractors for https://imgbb.com/"""
from .common import Extractor, Message
-from .. import text, exception
+from .. import text, util, exception
from ..cache import cache
-import json
class ImgbbExtractor(Extractor):
@@ -98,7 +97,7 @@ class ImgbbExtractor(Extractor):
while True:
for img in text.extract_iter(page, "data-object='", "'"):
- yield json.loads(text.unquote(img))
+ yield util.json_loads(text.unquote(img))
if data:
if params["seek"] == data["seekEnd"]:
return
diff --git a/gallery_dl/extractor/issuu.py b/gallery_dl/extractor/issuu.py
index 8067f63d..c0a1de1c 100644
--- a/gallery_dl/extractor/issuu.py
+++ b/gallery_dl/extractor/issuu.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2019-2022 Mike Fährmann
+# Copyright 2019-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,7 @@
"""Extractors for https://issuu.com/"""
from .common import GalleryExtractor, Extractor, Message
-from .. import text
-import json
+from .. import text, util
class IssuuBase():
@@ -54,7 +53,7 @@ class IssuuPublicationExtractor(IssuuBase, GalleryExtractor):
})
def metadata(self, page):
- data = json.loads(text.extr(
+ data = util.json_loads(text.extr(
page, '"))["data"]
if not isinstance(data["galleries"], dict):
diff --git a/gallery_dl/option.py b/gallery_dl/option.py
index 9b915505..af70c840 100644
--- a/gallery_dl/option.py
+++ b/gallery_dl/option.py
@@ -10,9 +10,8 @@
import argparse
import logging
-import json
import sys
-from . import job, version
+from . import job, util, version
class ConfigAction(argparse.Action):
@@ -79,7 +78,7 @@ class Formatter(argparse.HelpFormatter):
def _parse_option(opt):
key, _, value = opt.partition("=")
try:
- value = json.loads(value)
+ value = util.json_loads(value)
except ValueError:
pass
return key, value
diff --git a/gallery_dl/util.py b/gallery_dl/util.py
index 30f99a18..a5e782d3 100644
--- a/gallery_dl/util.py
+++ b/gallery_dl/util.py
@@ -204,6 +204,9 @@ def datetime_to_timestamp_string(dt):
return ""
+json_loads = json._default_decoder.decode
+
+
def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4):
"""Serialize 'obj' as JSON and write it to 'fp'"""
json.dump(
@@ -513,7 +516,7 @@ def parse_inputfile(file, log):
continue
try:
- value = json.loads(value.strip())
+ value = json_loads(value.strip())
except ValueError as exc:
log.warning("input file: unable to parse '%s': %s", value, exc)
continue
diff --git a/test/test_config.py b/test/test_config.py
index 7cbb12b8..859faf53 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
-# Copyright 2015-2020 Mike Fährmann
+# Copyright 2015-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -11,12 +11,11 @@ import os
import sys
import unittest
-import json
import tempfile
ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOTDIR)
-from gallery_dl import config # noqa E402
+from gallery_dl import config, util # noqa E402
class TestConfig(unittest.TestCase):
@@ -209,8 +208,8 @@ class TestConfigFiles(unittest.TestCase):
def _load(name):
path = os.path.join(ROOTDIR, "docs", name)
try:
- with open(path) as fp:
- return json.load(fp)
+ with open(path) as file:
+ return util.json_loads(file.read())
except FileNotFoundError:
raise unittest.SkipTest(path + " not available")