Refactor product releases writing

Create a common function to write resulting JSON files to the releases directory.
It makes this task simpler to read and maintain, while making it modifiable at a central point in the future.
One example of such modification could be the sorting of the versions in a uniform way for all the scripts.

src/apple.py

@@ -1,5 +1,4 @@
 import datetime
-import json
 import re
 from bs4 import BeautifulSoup
 from common import endoflife
@@ -97,8 +96,7 @@ for url in URLS:
 
 
 for k in CONFIG.keys():
-    with open("releases/%s.json" % k, "w") as f:
-        data = {v: d.strftime("%Y-%m-%d") for v, d in release_lists[k].items()}
-        f.write(json.dumps(data, indent=2))
+    releases = {v: d.strftime("%Y-%m-%d") for v, d in release_lists[k].items()}
+    endoflife.write_releases(k, releases)
 
 print("::endgroup::")

src/common/endoflife.py

@@ -1,6 +1,6 @@
+import json
 import frontmatter
 import urllib.request
-
 from glob import glob
 from os import path
 
@@ -47,3 +47,8 @@ def fetch_url(url, retry_count=2, timeout=5, data=None, headers=None, encoding='
             continue
 
     raise last_exception
+
+
+def write_releases(product, releases, pathname="releases"):
+    with open(f"{pathname}/{product}.json", "w") as f:
+        f.write(json.dumps(releases, indent=2))

src/cos.py

@@ -1,4 +1,3 @@
-import json
 import re
 from bs4 import BeautifulSoup
 from common import endoflife
@@ -69,7 +68,5 @@ def get_all_versions():
     return all_versions
 
 
-if __name__ == '__main__':
-    v = get_all_versions()
-    with open('releases/cos.json', "w") as f:
-        f.write(json.dumps(v, indent=2))
+releases = get_all_versions()
+endoflife.write_releases('cos', releases)

src/debian.py

@@ -1,6 +1,6 @@
-import json
 import pathlib
 import subprocess
+from common import endoflife
 from hashlib import sha1
 from os.path import exists
 from subprocess import call
@@ -73,21 +73,13 @@ def extract_point_releases(releases):
             releases[version] = date
 
 
-def main():
-    print(f"::group::{PRODUCT}")
-    clone_repository()
-
-    releases = {}
-    extract_major_releases(releases)
-    extract_point_releases(releases)
-    print("::endgroup::")
-
-    with open(f"releases/{PRODUCT}.json", "w") as f:
-        f.write(json.dumps(dict(
-            # sort by date then version (desc)
-            sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
-        ), indent=2))
-
-
-if __name__ == '__main__':
-    main()
+print(f"::group::{PRODUCT}")
+clone_repository()
+releases = {}
+extract_major_releases(releases)
+extract_point_releases(releases)
+endoflife.write_releases(PRODUCT, dict(
+    # sort by date then version (desc)
+    sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
+))
+print("::endgroup::")

src/distrowatch.py

@@ -1,4 +1,3 @@
-import json
 import re
 import sys
 from bs4 import BeautifulSoup
@@ -48,8 +47,7 @@ def update_product(product_name, configs):
             regex = config["regex"]
             releases = releases | fetch_releases(config[METHOD], regex, t)
 
-    with open("releases/%s.json" % product_name, "w") as f:
-        f.write(json.dumps(releases, indent=2))
+    endoflife.write_releases(product_name, releases)
 
 
 p_filter = sys.argv[1] if len(sys.argv) > 1 else None

src/eks.py

@@ -1,5 +1,4 @@
 import datetime
-import json
 import markdown
 import re
 from bs4 import BeautifulSoup
@@ -32,7 +31,5 @@ def parse_platforms_page():
     return all_versions
 
 
-if __name__ == "__main__":
-    versions = parse_platforms_page()
-    with open("releases/eks.json", "w") as f:
-        f.write(json.dumps(versions, indent=2))
+versions = parse_platforms_page()
+endoflife.write_releases('eks', versions)

src/firefox.py

@@ -1,5 +1,4 @@
 import concurrent.futures
-import json
 import re
 import requests
 from bs4 import BeautifulSoup
@@ -134,18 +133,10 @@ def fetch_releases():
 
     return releases
 
-def main():
-    print(f"::group::{PRODUCT}")
-
-    releases = fetch_releases()
-    with open(f"releases/{PRODUCT}.json", "w") as f:
-        f.write(json.dumps(dict(
-            # sort by date then version (desc)
-            sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
-        ), indent=2))
-
-    print("::endgroup::")
-
-
-if __name__ == '__main__':
-    main()
+print(f"::group::{PRODUCT}")
+releases = fetch_releases()
+endoflife.write_releases(PRODUCT, dict(
+    # sort by date then version (desc)
+    sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
+))
+print("::endgroup::")

src/github-releases.py

@@ -1,6 +1,5 @@
 import re
 import sys
-import json
 import subprocess
 from common import endoflife
 
@@ -72,8 +71,7 @@ def update_product(product_name, configs):
         config = config if "regex" in config else config | {"regex": REGEX}
         releases = releases | fetch_releases(config[METHOD], config["regex"])
 
-    with open(f"releases/{product_name}.json", "w") as f:
-        f.write(json.dumps(releases, indent=2))
+    endoflife.write_releases(product_name, releases)
 
 
 p_filter = sys.argv[1] if len(sys.argv) > 1 else None

src/gke.py

@@ -1,4 +1,3 @@
-import json
 import re
 from bs4 import BeautifulSoup
 from common import endoflife
@@ -38,20 +37,10 @@ def parse_soup_for_versions(soup):
 
 CHANNELS = ['nochannel', 'stable', 'regular', 'rapid']
 
-def main():
-    for channel in CHANNELS:
-        soup = fetch_channel(channel)
-        print("::group::GKE - {}".format(channel))
-        versions = parse_soup_for_versions(soup)
-        for version, date in versions.items():
-            print("{}: {}".format(version, date))
-        if channel == 'nochannel':
-            fn = 'releases/gke.json'
-        else:
-            fn = 'releases/gke-{}.json'.format(channel)
-        with open(fn, "w") as f:
-            f.write(json.dumps(versions, indent=2))
-        print("::endgroup::")
-
-if __name__ == '__main__':
-    main()
+for channel in CHANNELS:
+    soup = fetch_channel(channel)
+    print("::group::GKE - {}".format(channel))
+    versions = parse_soup_for_versions(soup)
+    name = 'gke' if channel == 'nochannel' else 'gke-{}'.format(channel)
+    endoflife.write_releases(name, versions)
+    print("::endgroup::")

src/haproxy.py

@@ -1,4 +1,3 @@
-import json
 import re
 from bs4 import BeautifulSoup
 from common import endoflife
@@ -51,19 +50,12 @@ def print_releases(releases):
         print(f"{version} : {date}")
 
 
-def main():
-    print(f"::group::{PRODUCT}")
-    cycles = fetch_cycles()
-    releases = fetch_releases(cycles)
-    print_releases(releases)
-    print("::endgroup::")
-
-    with open(f"releases/{PRODUCT}.json", "w") as f:
-        f.write(json.dumps(dict(
-            # sort by date then version (desc)
-            sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
-        ), indent=2))
-
-
-if __name__ == '__main__':
-    main()
+print(f"::group::{PRODUCT}")
+cycles = fetch_cycles()
+releases = fetch_releases(cycles)
+print_releases(releases)
+endoflife.write_releases(PRODUCT, dict(
+    # sort by date then version (desc)
+    sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
+))
+print("::endgroup::")

src/java.py

@@ -1,6 +1,5 @@
-import json
-
 from requests_html import HTMLSession
+from common import endoflife
 
 """Fetch Java versions with their dates from https://www.java.com/releases/.
 
@@ -37,19 +36,12 @@ def fetch_releases(releases):
             previous_date = date
 
 
-def main():
-    print(f"::group::{PRODUCT}")
-    releases = {}
-    fetch_releases(releases)
-    releases.pop('1.0_alpha') # that's the only version we do not want, regex not needed
-    print("::endgroup::")
-
-    with open(f"releases/{PRODUCT}.json", "w") as f:
-        f.write(json.dumps(dict(
-            # sort by date then version (desc)
-            sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
-        ), indent=2))
-
-
-if __name__ == '__main__':
-    main()
+print(f"::group::{PRODUCT}")
+releases = {}
+fetch_releases(releases)
+releases.pop('1.0_alpha') # that's the only version we do not want, regex not needed
+endoflife.write_releases(PRODUCT, dict(
+    # sort by date then version (desc)
+    sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
+))
+print("::endgroup::")

src/linuxkernel.py

@@ -1,4 +1,3 @@
-import json
 import re
 from bs4 import BeautifulSoup
 from common import endoflife
@@ -51,17 +50,10 @@ def fetch_releases():
     return releases
 
 
-def main():
-    print(f"::group::{PRODUCT}")
-    releases = fetch_releases()
-    print("::endgroup::")
-
-    with open(f"releases/{PRODUCT}.json", "w") as f:
-        f.write(json.dumps(dict(
-            # sort by version then date (asc)
-            sorted(releases.items(), key=lambda x: (x[0], x[1]))
-        ), indent=2))
-
-
-if __name__ == '__main__':
-    main()
+print(f"::group::{PRODUCT}")
+releases = fetch_releases()
+endoflife.write_releases(PRODUCT, dict(
+    # sort by version then date (asc)
+    sorted(releases.items(), key=lambda x: (x[0], x[1]))
+))
+print("::endgroup::")

src/maven.py

@@ -50,11 +50,10 @@ def update_product(product_name, configs):
     for config in configs:
         releases = releases | fetch_releases(config[METHOD])
 
-    with open(f"releases/{product_name}.json", "w") as f:
-        f.write(json.dumps(dict(
-            # sort by date then version (desc)
-            sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
-        ), indent=2))
+    endoflife.write_releases(product_name, dict(
+        # sort by date then version (desc)
+        sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
+    ))
 
 
 p_filter = sys.argv[1] if len(sys.argv) > 1 else None

src/npm.py

@@ -40,8 +40,7 @@ def update_product(product_name, configs):
         config = {"regex": REGEX} | config
         releases = releases | fetch_releases(config[METHOD], config["regex"])
 
-    with open(f"releases/{product_name}.json", "w") as f:
-        f.write(json.dumps(releases, indent=2))
+    endoflife.write_releases(product_name, releases)
 
 
 

src/palo-alto-networks.py

@@ -1,5 +1,4 @@
 import datetime
-import json
 import re
 from bs4 import BeautifulSoup
 from common import endoflife
@@ -19,6 +18,7 @@ def update_releases(html_identifier, file):
     print(f"::group::{html_identifier}")
     response = endoflife.fetch_url(URL)
     soup = BeautifulSoup(response, features="html5lib")
+
     table = soup.find(id=html_identifier)
     for tr in table.findAll("tr")[3:]:
         td_list = tr.findAll("td")
@@ -45,10 +45,9 @@ def update_releases(html_identifier, file):
 
             versions[version] = abs_date
             print("%s: %s" % (version, abs_date))
-    print("::endgroup::")
 
-    with open("releases/%s.json" % file, "w") as f:
-        f.write(json.dumps(versions, indent=2))
+    endoflife.write_releases(file, versions)
+    print("::endgroup::")
 
 
 for html_id in ID_MAPPING:

src/php.py

@@ -26,21 +26,13 @@ def fetch_versions(major_version):
     return data
 
 
-with open("releases/php.json", "w") as f:
-    print("::group::php")
-    releases = {}
+print("::group::php")
+releases = {}
 
-    for major_version in PHP_MAJOR_VERSIONS:
-        releases |= fetch_versions(major_version)
+for major_version in PHP_MAJOR_VERSIONS:
+    releases |= fetch_versions(major_version)
 
-    f.write(
-        json.dumps(
-            dict(sorted(
-                releases.items(),
-                key=lambda x: list(map(int, x[0].split(".")))
-            )),
-            indent=2,
-        )
-    )
-
-    print("::endgroup::")
+endoflife.write_releases('php', dict(sorted(
+    releases.items(), key=lambda x: list(map(int, x[0].split(".")))
+)))
+print("::endgroup::")

src/plesk.py

@@ -1,4 +1,3 @@
-import json
 from bs4 import BeautifulSoup
 from common import endoflife
 from datetime import datetime
@@ -35,18 +34,10 @@ def fetch_releases():
     return result
 
 
-def main():
-    print(f"::group::{PRODUCT}")
-
-    releases = fetch_releases()
-    with open(f"releases/{PRODUCT}.json", "w") as f:
-        f.write(json.dumps(dict(
-            # sort by date then version (desc)
-            sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
-        ), indent=2))
-
-    print("::endgroup::")
-
-
-if __name__ == '__main__':
-    main()
+print(f"::group::{PRODUCT}")
+releases = fetch_releases()
+endoflife.write_releases(PRODUCT, dict(
+    # sort by date then version (desc)
+    sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
+))
+print("::endgroup::")

src/pypi.py

@@ -41,8 +41,7 @@ def update_product(product_name, configs):
         config = {"regex": REGEX} | config
         releases = releases | fetch_releases(config[METHOD], config["regex"])
 
-    with open(f"releases/{product_name}.json", "w") as f:
-        f.write(json.dumps(releases, indent=2))
+    endoflife.write_releases(product_name, releases)
 
 
 p_filter = sys.argv[1] if len(sys.argv) > 1 else None

src/rds.py

@@ -1,4 +1,3 @@
-import json
 import re
 from bs4 import BeautifulSoup
 from common import endoflife
@@ -36,9 +35,8 @@ for db, url in dbs.items():
                     print(f"{version} : {date}")
                     releases[version] = date
 
+    endoflife.write_releases(f"amazon-rds-{db.lower()}", dict(
+        # sort by date then version (desc)
+        sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
+    ))
     print("::endgroup::")
-    with open(f"releases/amazon-rds-{db.lower()}.json", "w") as f:
-        json.dump(dict(
-            # sort by date then version (desc)
-            sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
-        ), f, indent=2)

src/rhel.py

@@ -1,4 +1,3 @@
-import json
 import re
 from bs4 import BeautifulSoup
 from common import endoflife
@@ -7,11 +6,11 @@ URL = "https://access.redhat.com/articles/3078"
 # https://regex101.com/r/877ibq/1
 regex = r"RHEL (?P<major>\d)(\. ?(?P<minor>\d+))?(( Update (?P<minor2>\d))| GA)?"
 
-versions = {}
-
 print("::group::rhel")
 response = endoflife.fetch_url(URL)
 soup = BeautifulSoup(response, features="html5lib")
+
+versions = {}
 for tr in soup.findAll("tr"):
     td_list = tr.findAll("td")
     if len(td_list) > 0:
@@ -25,7 +24,6 @@ for tr in soup.findAll("tr"):
         date = td_list[1].get_text()
         versions[version] = date
         print("%s: %s" % (version, date))
-print("::endgroup::")
 
-with open("releases/redhat.json", "w") as f:
-    f.write(json.dumps(versions, indent=2))
+endoflife.write_releases('redhat', versions)
+print("::endgroup::")

src/ros.py

@@ -1,5 +1,4 @@
 import datetime
-import json
 import re
 from bs4 import BeautifulSoup
 from common import endoflife
@@ -8,11 +7,11 @@ URL = "https://wiki.ros.org/Distributions"
 # https://regex101.com/r/c1ribd/1
 regex = r"^ROS (?P<name>(\w| )+)"
 
-versions = {}
-
 print("::group::ros")
 response = endoflife.fetch_url(URL)
 soup = BeautifulSoup(response, features="html5lib")
+
+versions = {}
 for tr in soup.findAll("tr"):
     td_list = tr.findAll("td")
     if len(td_list) > 0:
@@ -32,7 +31,6 @@ for tr in soup.findAll("tr"):
             abs_date = date.strftime("%Y-%m-%d")
             versions[version] = abs_date
             print("%s: %s" % (version, abs_date))
-print("::endgroup::")
 
-with open("releases/ros.json", "w") as f:
-    f.write(json.dumps(versions, indent=2))
+endoflife.write_releases('ros', versions)
+print("::endgroup::")

src/unrealircd.py

@@ -20,7 +20,6 @@ for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"):
             if re.match(r"\d{4}-\d{2}-\d{2}", maybe_date):
                 versions[maybe_version] = maybe_date
                 print("%s: %s" % (maybe_version, maybe_date))
-print("::endgroup::")
 
-with open("releases/unrealircd.json", "w") as f:
-    f.write(json.dumps(versions, indent=2))
+endoflife.write_releases('unrealircd', versions)
+print("::endgroup::")