diff --git a/scripts/build_testresult_db.py b/scripts/build_testresult_db.py index 85b332e0..00b42c57 100755 --- a/scripts/build_testresult_db.py +++ b/scripts/build_testresult_db.py @@ -51,7 +51,7 @@ for idx, extr, url, result in tests: # write test data try: - with open(os.path.join(path, name), "w") as outfile: + with util.open(os.path.join(path, name), "w") as outfile: job.DataJob(url, file=outfile, ensure_ascii=False).run() except KeyboardInterrupt: sys.exit() diff --git a/scripts/docs_compare.py b/scripts/docs_compare.py index 5d370788..dd57188a 100755 --- a/scripts/docs_compare.py +++ b/scripts/docs_compare.py @@ -14,7 +14,7 @@ from gallery_dl import text, extractor def read(fname): path = util.path("docs", fname) try: - with open(path) as fp: + with util.open(path) as fp: return fp.read() except Exception as exc: sys.exit("Unable to read {} ({}: {})".format( @@ -110,7 +110,7 @@ def sleeprequest(): continue min, _, max = value.partition("-") - tup = (float(min), float(max)) + tup = (float(min), float(max)) if max else float(min) if tup != extr.request_interval: result[category] = extr.request_interval return result diff --git a/scripts/export_tests.py b/scripts/export_tests.py index 7ea62852..30a8517f 100755 --- a/scripts/export_tests.py +++ b/scripts/export_tests.py @@ -71,7 +71,7 @@ def get_test_source(extr, *, cache={}): tests = cache[extr.__module__] except KeyError: path = sys.modules[extr.__module__].__file__ - with open(path) as fp: + with util.open(path) as fp: lines = fp.readlines() tests = cache[extr.__module__] = extract_tests_from_source(lines) return tests.get(extr.url) or ("",) diff --git a/scripts/generate_test_result.py b/scripts/generate_test_result.py index 4a3f286b..e2dc414a 100755 --- a/scripts/generate_test_result.py +++ b/scripts/generate_test_result.py @@ -82,7 +82,7 @@ def insert_test_result(args, result): path = util.path("test", "results", f"{args.cat}.py") LOG.info("Adding '%s:%s' test result into '%s'", args.cat, args.sub, path) - with open(path) as fp: + with util.open(path) as fp: lines = fp.readlines() lines.insert(-2, result) diff --git a/scripts/init.py b/scripts/init.py index d5549eaa..1f660e30 100755 --- a/scripts/init.py +++ b/scripts/init.py @@ -69,7 +69,7 @@ def create_extractor_module(opts=NONE): else: generate_extractors = generate_extractors_basic - with open(path, opts["open_mode"], encoding="utf-8") as fp: + with util.open(path, opts["open_mode"]) as fp: if copyright := opts.get("copyright", ""): copyright = f"\n# Copyright {dt.date.today().year} {copyright}\n#" @@ -222,8 +222,7 @@ def create_test_results_file(opts=NONE): LOG.info("Creating '%s'", trim_path(path)) import_stmt = generate_test_result_import(opts) - with open(path, "x", encoding="utf-8") as fp: - + with util.open(path, "x") as fp: fp.write(f"""\ {ENCODING} {LICENSE} @@ -260,7 +259,7 @@ def insert_into_modules_list(opts=NONE): category) path = util.path("gallery_dl", "extractor", "__init__.py") - with open(path) as fp: + with util.open(path) as fp: lines = fp.readlines() module_name = f' "{category}",\n' @@ -289,7 +288,7 @@ def insert_into_supportedsites(opts): category) path = util.path("scripts", "supportedsites.py") - with open(path) as fp: + with util.open(path) as fp: lines = fp.readlines() compare = False diff --git a/scripts/man.py b/scripts/man.py index 2aff221d..a6e3d778 100755 --- a/scripts/man.py +++ b/scripts/man.py @@ -228,8 +228,8 @@ and https://github.com/mikf/gallery-dl/graphs/contributors def parse_docs_configuration(): - doc_path = util.path("docs", "configuration.rst") - with open(doc_path, encoding="utf-8") as fp: + path = util.path("docs", "configuration.rst") + with util.open(path) as fp: doc_lines = fp.readlines() sections = {} diff --git a/scripts/util.py b/scripts/util.py index 48b8d319..7579d25e 100644 --- a/scripts/util.py +++ b/scripts/util.py @@ -7,17 +7,22 @@ import os import io import sys +import builtins ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.realpath(ROOTDIR)) -def path(*segments, join=os.path.join): - result = join(ROOTDIR, *segments) +def path(*segments): + result = os.path.join(ROOTDIR, *segments) os.makedirs(os.path.dirname(result), exist_ok=True) return result +def open(path, mode="r"): + return builtins.open(path, mode, encoding="utf-8", newline="\n") + + class lazy(): def __init__(self, path): @@ -30,7 +35,7 @@ class lazy(): def __exit__(self, exc_type, exc_value, traceback): # get content of old file try: - with open(self.path, encoding="utf-8") as fp: + with builtins.open(self.path, encoding="utf-8", newline="") as fp: old = fp.read() except Exception: old = None @@ -40,7 +45,8 @@ class lazy(): if new != old: # rewrite entire file - with open(self.path, "w", encoding="utf-8") as fp: + with builtins.open( + self.path, "w", encoding="utf-8", newline="") as fp: fp.write(new) else: # only update atime and mtime