[scripts] ensure files use 'utf-8' encoding and '\n' newlines (#7872)

scripts/build_testresult_db.py

@@ -51,7 +51,7 @@ for idx, extr, url, result in tests:
 
     # write test data
     try:
-        with open(os.path.join(path, name), "w") as outfile:
+        with util.open(os.path.join(path, name), "w") as outfile:
             job.DataJob(url, file=outfile, ensure_ascii=False).run()
     except KeyboardInterrupt:
         sys.exit()

scripts/docs_compare.py

@@ -14,7 +14,7 @@ from gallery_dl import text, extractor
 def read(fname):
     path = util.path("docs", fname)
     try:
-        with open(path) as fp:
+        with util.open(path) as fp:
             return fp.read()
     except Exception as exc:
         sys.exit("Unable to read {} ({}: {})".format(
@@ -110,7 +110,7 @@ def sleeprequest():
                 continue
 
         min, _, max = value.partition("-")
-        tup = (float(min), float(max))
+        tup = (float(min), float(max)) if max else float(min)
         if tup != extr.request_interval:
             result[category] = extr.request_interval
     return result

scripts/export_tests.py

@@ -71,7 +71,7 @@ def get_test_source(extr, *, cache={}):
         tests = cache[extr.__module__]
     except KeyError:
         path = sys.modules[extr.__module__].__file__
-        with open(path) as fp:
+        with util.open(path) as fp:
             lines = fp.readlines()
         tests = cache[extr.__module__] = extract_tests_from_source(lines)
     return tests.get(extr.url) or ("",)

scripts/generate_test_result.py

@@ -82,7 +82,7 @@ def insert_test_result(args, result):
     path = util.path("test", "results", f"{args.cat}.py")
     LOG.info("Adding '%s:%s' test result into '%s'", args.cat, args.sub, path)
 
-    with open(path) as fp:
+    with util.open(path) as fp:
         lines = fp.readlines()
 
     lines.insert(-2, result)

scripts/init.py

@@ -69,7 +69,7 @@ def create_extractor_module(opts=NONE):
     else:
         generate_extractors = generate_extractors_basic
 
-    with open(path, opts["open_mode"], encoding="utf-8") as fp:
+    with util.open(path, opts["open_mode"]) as fp:
         if copyright := opts.get("copyright", ""):
             copyright = f"\n# Copyright {dt.date.today().year} {copyright}\n#"
 
@@ -222,8 +222,7 @@ def create_test_results_file(opts=NONE):
     LOG.info("Creating '%s'", trim_path(path))
 
     import_stmt = generate_test_result_import(opts)
-    with open(path, "x", encoding="utf-8") as fp:
+    with util.open(path, "x") as fp:
-
         fp.write(f"""\
 {ENCODING}
 {LICENSE}
@@ -260,7 +259,7 @@ def insert_into_modules_list(opts=NONE):
              category)
 
     path = util.path("gallery_dl", "extractor", "__init__.py")
-    with open(path) as fp:
+    with util.open(path) as fp:
         lines = fp.readlines()
 
     module_name = f'    "{category}",\n'
@@ -289,7 +288,7 @@ def insert_into_supportedsites(opts):
              category)
 
     path = util.path("scripts", "supportedsites.py")
-    with open(path) as fp:
+    with util.open(path) as fp:
         lines = fp.readlines()
 
     compare = False

scripts/man.py

@@ -228,8 +228,8 @@ and https://github.com/mikf/gallery-dl/graphs/contributors
 
 def parse_docs_configuration():
 
-    doc_path = util.path("docs", "configuration.rst")
+    path = util.path("docs", "configuration.rst")
-    with open(doc_path, encoding="utf-8") as fp:
+    with util.open(path) as fp:
         doc_lines = fp.readlines()
 
     sections = {}

scripts/util.py

@@ -7,17 +7,22 @@
 import os
 import io
 import sys
+import builtins
 
 ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.realpath(ROOTDIR))
 
 
-def path(*segments, join=os.path.join):
+def path(*segments):
-    result = join(ROOTDIR, *segments)
+    result = os.path.join(ROOTDIR, *segments)
     os.makedirs(os.path.dirname(result), exist_ok=True)
     return result
 
 
+def open(path, mode="r"):
+    return builtins.open(path, mode, encoding="utf-8", newline="\n")
+
+
 class lazy():
 
     def __init__(self, path):
@@ -30,7 +35,7 @@ class lazy():
     def __exit__(self, exc_type, exc_value, traceback):
         # get content of old file
         try:
-            with open(self.path, encoding="utf-8") as fp:
+            with builtins.open(self.path, encoding="utf-8", newline="") as fp:
                 old = fp.read()
         except Exception:
             old = None
@@ -40,7 +45,8 @@ class lazy():
 
         if new != old:
             # rewrite entire file
-            with open(self.path, "w", encoding="utf-8") as fp:
+            with builtins.open(
+                    self.path, "w", encoding="utf-8", newline="") as fp:
                 fp.write(new)
         else:
             # only update atime and mtime