diff --git a/gallery_dl/config.py b/gallery_dl/config.py index 0f2d1f19..819347da 100644 --- a/gallery_dl/config.py +++ b/gallery_dl/config.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2021 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,7 +9,6 @@ """Global configuration module""" import sys -import json import os.path import logging from . import util @@ -55,18 +54,18 @@ def load(files=None, strict=False, fmt="json"): if fmt == "yaml": try: import yaml - parsefunc = yaml.safe_load + load = yaml.safe_load except ImportError: log.error("Could not import 'yaml' module") return else: - parsefunc = json.load + load = util.json_loads for pathfmt in files or _default_configs: path = util.expand_path(pathfmt) try: with open(path, encoding="utf-8") as file: - confdict = parsefunc(file) + conf = load(file.read()) except OSError as exc: if strict: log.error(exc) @@ -77,9 +76,9 @@ def load(files=None, strict=False, fmt="json"): sys.exit(2) else: if not _config: - _config.update(confdict) + _config.update(conf) else: - util.combine_dict(_config, confdict) + util.combine_dict(_config, conf) _files.append(pathfmt) diff --git a/gallery_dl/cookies.py b/gallery_dl/cookies.py index f18cc475..e2a7e484 100644 --- a/gallery_dl/cookies.py +++ b/gallery_dl/cookies.py @@ -12,7 +12,6 @@ import binascii import contextlib import ctypes -import json import logging import os import shutil @@ -24,7 +23,7 @@ import tempfile from datetime import datetime, timedelta, timezone from hashlib import pbkdf2_hmac from http.cookiejar import Cookie -from . import aes, text +from . import aes, text, util SUPPORTED_BROWSERS_CHROMIUM = { @@ -169,8 +168,8 @@ def _firefox_cookies_database(profile=None, container=None): os.path.dirname(path), "containers.json") try: - with open(containers_path) as containers: - identities = json.load(containers)["identities"] + with open(containers_path) as file: + identities = util.json_loads(file.read())["identities"] except OSError: logger.error("Unable to read Firefox container database at %s", containers_path) @@ -716,8 +715,8 @@ def _get_windows_v10_key(browser_root): logger.error("could not find local state file") return None logger.debug("Found local state file at '%s'", path) - with open(path, encoding="utf8") as f: - data = json.load(f) + with open(path, encoding="utf-8") as file: + data = util.json_loads(file.read()) try: base64_key = data["os_crypt"]["encrypted_key"] except KeyError: diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py index fed49913..26ac8b21 100644 --- a/gallery_dl/extractor/8muses.py +++ b/gallery_dl/extractor/8muses.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,8 +9,7 @@ """Extractors for https://comics.8muses.com/""" from .common import Extractor, Message -from .. import text -import json +from .. import text, util class _8musesAlbumExtractor(Extractor): @@ -131,7 +130,7 @@ class _8musesAlbumExtractor(Extractor): @staticmethod def _unobfuscate(data): - return json.loads("".join([ + return util.json_loads("".join([ chr(33 + (ord(c) + 14) % 94) if "!" <= c <= "~" else c for c in text.unescape(data.strip("\t\n\r !")) ])) diff --git a/gallery_dl/extractor/bbc.py b/gallery_dl/extractor/bbc.py index 1b49d6a5..638fedc2 100644 --- a/gallery_dl/extractor/bbc.py +++ b/gallery_dl/extractor/bbc.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2021 Mike Fährmann +# Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -10,7 +10,6 @@ from .common import GalleryExtractor, Extractor, Message from .. import text, util -import json BASE_PATTERN = r"(?:https?://)?(?:www\.)?bbc\.co\.uk(/programmes/" @@ -38,7 +37,7 @@ class BbcGalleryExtractor(GalleryExtractor): ) def metadata(self, page): - data = json.loads(text.extr( + data = util.json_loads(text.extr( page, '')) return { "programme": self.gallery_url.split("/")[4], diff --git a/gallery_dl/extractor/bcy.py b/gallery_dl/extractor/bcy.py index 44d60650..d6adb4eb 100644 --- a/gallery_dl/extractor/bcy.py +++ b/gallery_dl/extractor/bcy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020-2022 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,8 +9,7 @@ """Extractors for https://bcy.net/""" from .common import Extractor, Message -from .. import text, exception -import json +from .. import text, util, exception import re @@ -100,9 +99,9 @@ class BcyExtractor(Extractor): .replace('\\\\u002F', '/') .replace('\\"', '"')) try: - return json.loads(data)["detail"] + return util.json_loads(data)["detail"] except ValueError: - return json.loads(data.replace('\\"', '"'))["detail"] + return util.json_loads(data.replace('\\"', '"'))["detail"] class BcyUserExtractor(BcyExtractor): diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py index 6da61752..1469aad9 100644 --- a/gallery_dl/extractor/behance.py +++ b/gallery_dl/extractor/behance.py @@ -9,8 +9,7 @@ """Extractors for https://www.behance.net/""" from .common import Extractor, Message -from .. import text -import json +from .. import text, util class BehanceExtractor(Extractor): @@ -120,7 +119,7 @@ class BehanceGalleryExtractor(BehanceExtractor): } page = self.request(url, cookies=cookies).text - data = json.loads(text.extr( + data = util.json_loads(text.extr( page, 'id="beconfig-store_state">', '')) return self._update(data["project"]["project"]) diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py index 8a1a42eb..56010c20 100644 --- a/gallery_dl/extractor/blogger.py +++ b/gallery_dl/extractor/blogger.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,8 +9,7 @@ """Extractors for Blogger blogs""" from .common import Extractor, Message -from .. import text -import json +from .. import text, util import re BASE_PATTERN = ( @@ -61,7 +60,7 @@ class BloggerExtractor(Extractor): page = self.request(post["url"]).text for url in findall_video(page): page = self.request(url).text - video_config = json.loads(text.extr( + video_config = util.json_loads(text.extr( page, 'var VIDEO_CONFIG =', '\n')) files.append(max( video_config["streams"], diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py index 1c339a9e..250b40a1 100644 --- a/gallery_dl/extractor/bunkr.py +++ b/gallery_dl/extractor/bunkr.py @@ -9,8 +9,7 @@ """Extractors for https://bunkr.ru/""" from .lolisafe import LolisafeAlbumExtractor -from .. import text -import json +from .. import text, util class BunkrAlbumExtractor(LolisafeAlbumExtractor): @@ -49,7 +48,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): root = self.root try: - data = json.loads(text.extr( + data = util.json_loads(text.extr( self.request(root + "/a/" + self.album_id).text, 'id="__NEXT_DATA__" type="application/json">', '<')) album = data["props"]["pageProps"]["album"] diff --git a/gallery_dl/extractor/dynastyscans.py b/gallery_dl/extractor/dynastyscans.py index d78f25be..59e8c90d 100644 --- a/gallery_dl/extractor/dynastyscans.py +++ b/gallery_dl/extractor/dynastyscans.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2022 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,8 +9,7 @@ """Extractors for https://dynasty-scans.com/""" from .common import ChapterExtractor, MangaExtractor, Extractor, Message -from .. import text -import json +from .. import text, util import re BASE_PATTERN = r"(?:https?://)?(?:www\.)?dynasty-scans\.com" @@ -86,7 +85,7 @@ class DynastyscansChapterExtractor(DynastyscansBase, ChapterExtractor): data = text.extr(page, "var pages = ", ";\n") return [ (self.root + img["image"], None) - for img in json.loads(data) + for img in util.json_loads(data) ] diff --git a/gallery_dl/extractor/fallenangels.py b/gallery_dl/extractor/fallenangels.py index 57587b60..0503dcfa 100644 --- a/gallery_dl/extractor/fallenangels.py +++ b/gallery_dl/extractor/fallenangels.py @@ -6,11 +6,10 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract manga-chapters from https://www.fascans.com/""" +"""Extractors for https://www.fascans.com/""" from .common import ChapterExtractor, MangaExtractor from .. import text, util -import json class FallenangelsChapterExtractor(ChapterExtractor): @@ -56,7 +55,7 @@ class FallenangelsChapterExtractor(ChapterExtractor): def images(page): return [ (img["page_image"], None) - for img in json.loads( + for img in util.json_loads( text.extr(page, "var pages = ", ";") ) ] diff --git a/gallery_dl/extractor/fantia.py b/gallery_dl/extractor/fantia.py index 476fdeb2..13dfeada 100644 --- a/gallery_dl/extractor/fantia.py +++ b/gallery_dl/extractor/fantia.py @@ -7,8 +7,7 @@ """Extractors for https://fantia.jp/""" from .common import Extractor, Message -from .. import text -import json +from .. import text, util class FantiaExtractor(Extractor): @@ -117,7 +116,7 @@ class FantiaExtractor(Extractor): yield self.root+"/"+content["download_uri"], post if content["category"] == "blog" and "comment" in content: - comment_json = json.loads(content["comment"]) + comment_json = util.json_loads(content["comment"]) ops = comment_json.get("ops", ()) # collect blogpost text first diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py index 2290cc25..4a38fb4f 100644 --- a/gallery_dl/extractor/foolslide.py +++ b/gallery_dl/extractor/foolslide.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2022 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -10,7 +10,6 @@ from .common import BaseExtractor, Message from .. import text, util -import json class FoolslideExtractor(BaseExtractor): @@ -106,7 +105,7 @@ class FoolslideChapterExtractor(FoolslideExtractor): }) def images(self, page): - return json.loads(text.extr(page, "var pages = ", ";")) + return util.json_loads(text.extr(page, "var pages = ", ";")) class FoolslideMangaExtractor(FoolslideExtractor): diff --git a/gallery_dl/extractor/hbrowse.py b/gallery_dl/extractor/hbrowse.py index 43479c6f..5b561ead 100644 --- a/gallery_dl/extractor/hbrowse.py +++ b/gallery_dl/extractor/hbrowse.py @@ -1,16 +1,15 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2019 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -"""Extract images from https://www.hbrowse.com/""" +"""Extractors for https://www.hbrowse.com/""" from .common import ChapterExtractor, MangaExtractor -from .. import text, exception -import json +from .. import text, util, exception class HbrowseBase(): @@ -68,7 +67,7 @@ class HbrowseChapterExtractor(HbrowseBase, ChapterExtractor): def images(self, page): base = self.root + "/data" + self.path json_data = text.extract(page, ';list = ', ',"zzz"')[0] + "]" - return [(base + name, None) for name in json.loads(json_data)] + return [(base + name, None) for name in util.json_loads(json_data)] class HbrowseMangaExtractor(HbrowseBase, MangaExtractor): diff --git a/gallery_dl/extractor/hentai2read.py b/gallery_dl/extractor/hentai2read.py index dc4e31d2..e771a4fc 100644 --- a/gallery_dl/extractor/hentai2read.py +++ b/gallery_dl/extractor/hentai2read.py @@ -9,8 +9,7 @@ """Extractors for https://hentai2read.com/""" from .common import ChapterExtractor, MangaExtractor -from .. import text -import json +from .. import text, util import re @@ -78,7 +77,7 @@ class Hentai2readChapterExtractor(Hentai2readBase, ChapterExtractor): images = text.extract(page, "'images' : ", ",\n")[0] return [ ("https://hentaicdn.com/hentai" + part, None) - for part in json.loads(images) + for part in util.json_loads(images) ] diff --git a/gallery_dl/extractor/hentaifox.py b/gallery_dl/extractor/hentaifox.py index 0327f563..ed8576f1 100644 --- a/gallery_dl/extractor/hentaifox.py +++ b/gallery_dl/extractor/hentaifox.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2021 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,8 +9,7 @@ """Extractors for https://hentaifox.com/""" from .common import GalleryExtractor, Extractor, Message -from .. import text -import json +from .. import text, util class HentaifoxBase(): @@ -90,7 +89,7 @@ class HentaifoxGalleryExtractor(HentaifoxBase, GalleryExtractor): server1 = "https://i.hentaifox.com" server2 = "https://i2.hentaifox.com" - for num, image in json.loads(data).items(): + for num, image in util.json_loads(data).items(): ext, width, height = image.split(",") path = urlfmt(num, extmap[ext]) append((server1 + path, { diff --git a/gallery_dl/extractor/hentaihand.py b/gallery_dl/extractor/hentaihand.py index bf9e4641..0617330b 100644 --- a/gallery_dl/extractor/hentaihand.py +++ b/gallery_dl/extractor/hentaihand.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2020-2022 Mike Fährmann +# Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -10,7 +10,6 @@ from .common import GalleryExtractor, Extractor, Message from .. import text, util -import json class HentaihandGalleryExtractor(GalleryExtractor): @@ -46,7 +45,7 @@ class HentaihandGalleryExtractor(GalleryExtractor): GalleryExtractor.__init__(self, match, url) def metadata(self, page): - info = json.loads(page) + info = util.json_loads(page) data = { "gallery_id" : text.parse_int(info["id"]), "title" : info["title"], diff --git a/gallery_dl/extractor/hentaihere.py b/gallery_dl/extractor/hentaihere.py index 38ec77c6..2297cc06 100644 --- a/gallery_dl/extractor/hentaihere.py +++ b/gallery_dl/extractor/hentaihere.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2016-2022 Mike Fährmann +# Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,8 +9,7 @@ """Extractors for https://hentaihere.com/""" from .common import ChapterExtractor, MangaExtractor -from .. import text -import json +from .. import text, util import re @@ -80,7 +79,7 @@ class HentaihereChapterExtractor(HentaihereBase, ChapterExtractor): images = text.extr(page, "var rff_imageList = ", ";") return [ ("https://hentaicdn.com/hentai" + part, None) - for part in json.loads(images) + for part in util.json_loads(images) ] diff --git a/gallery_dl/extractor/hitomi.py b/gallery_dl/extractor/hitomi.py index 44459ceb..4e8d1cae 100644 --- a/gallery_dl/extractor/hitomi.py +++ b/gallery_dl/extractor/hitomi.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2015-2022 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -13,7 +13,6 @@ from .nozomi import decode_nozomi from ..cache import memcache from .. import text, util import string -import json import re @@ -75,7 +74,7 @@ class HitomiGalleryExtractor(GalleryExtractor): self.root, gid) def metadata(self, page): - self.info = info = json.loads(page.partition("=")[2]) + self.info = info = util.json_loads(page.partition("=")[2]) iget = info.get language = iget("language") diff --git a/gallery_dl/extractor/imagefap.py b/gallery_dl/extractor/imagefap.py index 378e26db..497f1efa 100644 --- a/gallery_dl/extractor/imagefap.py +++ b/gallery_dl/extractor/imagefap.py @@ -9,8 +9,7 @@ """Extractors for https://www.imagefap.com/""" from .common import Extractor, Message -from .. import text, exception -import json +from .. import text, util, exception BASE_PATTERN = r"(?:https?://)?(?:www\.|beta\.)?imagefap\.com" @@ -173,7 +172,7 @@ class ImagefapImageExtractor(ImagefapExtractor): page, 'id="imageid_input" value="', '"', pos) gallery_id, pos = text.extract( page, 'id="galleryid_input" value="', '"', pos) - info = json.loads(info) + info = util.json_loads(info) url = info["contentUrl"] return url, text.nameext_from_url(url, { diff --git a/gallery_dl/extractor/imgbb.py b/gallery_dl/extractor/imgbb.py index 49082d85..a2210752 100644 --- a/gallery_dl/extractor/imgbb.py +++ b/gallery_dl/extractor/imgbb.py @@ -9,9 +9,8 @@ """Extractors for https://imgbb.com/""" from .common import Extractor, Message -from .. import text, exception +from .. import text, util, exception from ..cache import cache -import json class ImgbbExtractor(Extractor): @@ -98,7 +97,7 @@ class ImgbbExtractor(Extractor): while True: for img in text.extract_iter(page, "data-object='", "'"): - yield json.loads(text.unquote(img)) + yield util.json_loads(text.unquote(img)) if data: if params["seek"] == data["seekEnd"]: return diff --git a/gallery_dl/extractor/issuu.py b/gallery_dl/extractor/issuu.py index 8067f63d..c0a1de1c 100644 --- a/gallery_dl/extractor/issuu.py +++ b/gallery_dl/extractor/issuu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019-2022 Mike Fährmann +# Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,8 +9,7 @@ """Extractors for https://issuu.com/""" from .common import GalleryExtractor, Extractor, Message -from .. import text -import json +from .. import text, util class IssuuBase(): @@ -54,7 +53,7 @@ class IssuuPublicationExtractor(IssuuBase, GalleryExtractor): }) def metadata(self, page): - data = json.loads(text.extr( + data = util.json_loads(text.extr( page, '"))["data"] if not isinstance(data["galleries"], dict): diff --git a/gallery_dl/option.py b/gallery_dl/option.py index 9b915505..af70c840 100644 --- a/gallery_dl/option.py +++ b/gallery_dl/option.py @@ -10,9 +10,8 @@ import argparse import logging -import json import sys -from . import job, version +from . import job, util, version class ConfigAction(argparse.Action): @@ -79,7 +78,7 @@ class Formatter(argparse.HelpFormatter): def _parse_option(opt): key, _, value = opt.partition("=") try: - value = json.loads(value) + value = util.json_loads(value) except ValueError: pass return key, value diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 30f99a18..a5e782d3 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -204,6 +204,9 @@ def datetime_to_timestamp_string(dt): return "" +json_loads = json._default_decoder.decode + + def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4): """Serialize 'obj' as JSON and write it to 'fp'""" json.dump( @@ -513,7 +516,7 @@ def parse_inputfile(file, log): continue try: - value = json.loads(value.strip()) + value = json_loads(value.strip()) except ValueError as exc: log.warning("input file: unable to parse '%s': %s", value, exc) continue diff --git a/test/test_config.py b/test/test_config.py index 7cbb12b8..859faf53 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright 2015-2020 Mike Fährmann +# Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -11,12 +11,11 @@ import os import sys import unittest -import json import tempfile ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOTDIR) -from gallery_dl import config # noqa E402 +from gallery_dl import config, util # noqa E402 class TestConfig(unittest.TestCase): @@ -209,8 +208,8 @@ class TestConfigFiles(unittest.TestCase): def _load(name): path = os.path.join(ROOTDIR, "docs", name) try: - with open(path) as fp: - return json.load(fp) + with open(path) as file: + return util.json_loads(file.read()) except FileNotFoundError: raise unittest.SkipTest(path + " not available")