From 6ba7828f96814280fb12627c80aa64deabef5e37 Mon Sep 17 00:00:00 2001 From: Marc Wrobel Date: Sat, 20 May 2023 11:00:47 +0200 Subject: [PATCH] Refactor product releases writing Create a common function to write resulting JSON files to the releases directory. It makes this task simpler to read and maintain, while making it modifiable at a central point in the future. One example of such modification could be the sorting of the versions in a uniform way for all the scripts. --- src/apple.py | 6 ++---- src/common/endoflife.py | 7 ++++++- src/cos.py | 7 ++----- src/debian.py | 30 +++++++++++------------------- src/distrowatch.py | 4 +--- src/eks.py | 7 ++----- src/firefox.py | 23 +++++++---------------- src/github-releases.py | 4 +--- src/gke.py | 25 +++++++------------------ src/haproxy.py | 26 +++++++++----------------- src/java.py | 28 ++++++++++------------------ src/linuxkernel.py | 22 +++++++--------------- src/maven.py | 9 ++++----- src/npm.py | 3 +-- src/palo-alto-networks.py | 7 +++---- src/php.py | 24 ++++++++---------------- src/plesk.py | 23 +++++++---------------- src/pypi.py | 3 +-- src/rds.py | 10 ++++------ src/rhel.py | 10 ++++------ src/ros.py | 10 ++++------ src/unrealircd.py | 5 ++--- 22 files changed, 103 insertions(+), 190 deletions(-) diff --git a/src/apple.py b/src/apple.py index 81e38916..2233f945 100644 --- a/src/apple.py +++ b/src/apple.py @@ -1,5 +1,4 @@ import datetime -import json import re from bs4 import BeautifulSoup from common import endoflife @@ -97,8 +96,7 @@ for url in URLS: for k in CONFIG.keys(): - with open("releases/%s.json" % k, "w") as f: - data = {v: d.strftime("%Y-%m-%d") for v, d in release_lists[k].items()} - f.write(json.dumps(data, indent=2)) + releases = {v: d.strftime("%Y-%m-%d") for v, d in release_lists[k].items()} + endoflife.write_releases(k, releases) print("::endgroup::") diff --git a/src/common/endoflife.py b/src/common/endoflife.py index 93832372..87014a1b 100644 --- a/src/common/endoflife.py +++ b/src/common/endoflife.py @@ -1,6 +1,6 @@ +import json import frontmatter import urllib.request - from glob import glob from os import path @@ -47,3 +47,8 @@ def fetch_url(url, retry_count=2, timeout=5, data=None, headers=None, encoding=' continue raise last_exception + + +def write_releases(product, releases, pathname="releases"): + with open(f"{pathname}/{product}.json", "w") as f: + f.write(json.dumps(releases, indent=2)) diff --git a/src/cos.py b/src/cos.py index cab5d0ad..9e1eea30 100644 --- a/src/cos.py +++ b/src/cos.py @@ -1,4 +1,3 @@ -import json import re from bs4 import BeautifulSoup from common import endoflife @@ -69,7 +68,5 @@ def get_all_versions(): return all_versions -if __name__ == '__main__': - v = get_all_versions() - with open('releases/cos.json', "w") as f: - f.write(json.dumps(v, indent=2)) +releases = get_all_versions() +endoflife.write_releases('cos', releases) diff --git a/src/debian.py b/src/debian.py index fc206fa8..5e91471d 100644 --- a/src/debian.py +++ b/src/debian.py @@ -1,6 +1,6 @@ -import json import pathlib import subprocess +from common import endoflife from hashlib import sha1 from os.path import exists from subprocess import call @@ -73,21 +73,13 @@ def extract_point_releases(releases): releases[version] = date -def main(): - print(f"::group::{PRODUCT}") - clone_repository() - - releases = {} - extract_major_releases(releases) - extract_point_releases(releases) - print("::endgroup::") - - with open(f"releases/{PRODUCT}.json", "w") as f: - f.write(json.dumps(dict( - # sort by date then version (desc) - sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) - ), indent=2)) - - -if __name__ == '__main__': - main() +print(f"::group::{PRODUCT}") +clone_repository() +releases = {} +extract_major_releases(releases) +extract_point_releases(releases) +endoflife.write_releases(PRODUCT, dict( + # sort by date then version (desc) + sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) +)) +print("::endgroup::") diff --git a/src/distrowatch.py b/src/distrowatch.py index 7c3f3cc9..18361fbc 100644 --- a/src/distrowatch.py +++ b/src/distrowatch.py @@ -1,4 +1,3 @@ -import json import re import sys from bs4 import BeautifulSoup @@ -48,8 +47,7 @@ def update_product(product_name, configs): regex = config["regex"] releases = releases | fetch_releases(config[METHOD], regex, t) - with open("releases/%s.json" % product_name, "w") as f: - f.write(json.dumps(releases, indent=2)) + endoflife.write_releases(product_name, releases) p_filter = sys.argv[1] if len(sys.argv) > 1 else None diff --git a/src/eks.py b/src/eks.py index 0b1b3f63..f4bcd867 100644 --- a/src/eks.py +++ b/src/eks.py @@ -1,5 +1,4 @@ import datetime -import json import markdown import re from bs4 import BeautifulSoup @@ -32,7 +31,5 @@ def parse_platforms_page(): return all_versions -if __name__ == "__main__": - versions = parse_platforms_page() - with open("releases/eks.json", "w") as f: - f.write(json.dumps(versions, indent=2)) +versions = parse_platforms_page() +endoflife.write_releases('eks', versions) diff --git a/src/firefox.py b/src/firefox.py index d4461f13..a98f57ef 100644 --- a/src/firefox.py +++ b/src/firefox.py @@ -1,5 +1,4 @@ import concurrent.futures -import json import re import requests from bs4 import BeautifulSoup @@ -134,18 +133,10 @@ def fetch_releases(): return releases -def main(): - print(f"::group::{PRODUCT}") - - releases = fetch_releases() - with open(f"releases/{PRODUCT}.json", "w") as f: - f.write(json.dumps(dict( - # sort by date then version (desc) - sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) - ), indent=2)) - - print("::endgroup::") - - -if __name__ == '__main__': - main() +print(f"::group::{PRODUCT}") +releases = fetch_releases() +endoflife.write_releases(PRODUCT, dict( + # sort by date then version (desc) + sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) +)) +print("::endgroup::") diff --git a/src/github-releases.py b/src/github-releases.py index 125be934..f1763823 100644 --- a/src/github-releases.py +++ b/src/github-releases.py @@ -1,6 +1,5 @@ import re import sys -import json import subprocess from common import endoflife @@ -72,8 +71,7 @@ def update_product(product_name, configs): config = config if "regex" in config else config | {"regex": REGEX} releases = releases | fetch_releases(config[METHOD], config["regex"]) - with open(f"releases/{product_name}.json", "w") as f: - f.write(json.dumps(releases, indent=2)) + endoflife.write_releases(product_name, releases) p_filter = sys.argv[1] if len(sys.argv) > 1 else None diff --git a/src/gke.py b/src/gke.py index 841cf7d8..e917b0ca 100644 --- a/src/gke.py +++ b/src/gke.py @@ -1,4 +1,3 @@ -import json import re from bs4 import BeautifulSoup from common import endoflife @@ -38,20 +37,10 @@ def parse_soup_for_versions(soup): CHANNELS = ['nochannel', 'stable', 'regular', 'rapid'] -def main(): - for channel in CHANNELS: - soup = fetch_channel(channel) - print("::group::GKE - {}".format(channel)) - versions = parse_soup_for_versions(soup) - for version, date in versions.items(): - print("{}: {}".format(version, date)) - if channel == 'nochannel': - fn = 'releases/gke.json' - else: - fn = 'releases/gke-{}.json'.format(channel) - with open(fn, "w") as f: - f.write(json.dumps(versions, indent=2)) - print("::endgroup::") - -if __name__ == '__main__': - main() +for channel in CHANNELS: + soup = fetch_channel(channel) + print("::group::GKE - {}".format(channel)) + versions = parse_soup_for_versions(soup) + name = 'gke' if channel == 'nochannel' else 'gke-{}'.format(channel) + endoflife.write_releases(name, versions) + print("::endgroup::") diff --git a/src/haproxy.py b/src/haproxy.py index f82fa239..644331a9 100644 --- a/src/haproxy.py +++ b/src/haproxy.py @@ -1,4 +1,3 @@ -import json import re from bs4 import BeautifulSoup from common import endoflife @@ -51,19 +50,12 @@ def print_releases(releases): print(f"{version} : {date}") -def main(): - print(f"::group::{PRODUCT}") - cycles = fetch_cycles() - releases = fetch_releases(cycles) - print_releases(releases) - print("::endgroup::") - - with open(f"releases/{PRODUCT}.json", "w") as f: - f.write(json.dumps(dict( - # sort by date then version (desc) - sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) - ), indent=2)) - - -if __name__ == '__main__': - main() +print(f"::group::{PRODUCT}") +cycles = fetch_cycles() +releases = fetch_releases(cycles) +print_releases(releases) +endoflife.write_releases(PRODUCT, dict( + # sort by date then version (desc) + sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) +)) +print("::endgroup::") diff --git a/src/java.py b/src/java.py index 55828c9a..ff0f0893 100644 --- a/src/java.py +++ b/src/java.py @@ -1,6 +1,5 @@ -import json - from requests_html import HTMLSession +from common import endoflife """Fetch Java versions with their dates from https://www.java.com/releases/. @@ -37,19 +36,12 @@ def fetch_releases(releases): previous_date = date -def main(): - print(f"::group::{PRODUCT}") - releases = {} - fetch_releases(releases) - releases.pop('1.0_alpha') # that's the only version we do not want, regex not needed - print("::endgroup::") - - with open(f"releases/{PRODUCT}.json", "w") as f: - f.write(json.dumps(dict( - # sort by date then version (desc) - sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) - ), indent=2)) - - -if __name__ == '__main__': - main() +print(f"::group::{PRODUCT}") +releases = {} +fetch_releases(releases) +releases.pop('1.0_alpha') # that's the only version we do not want, regex not needed +endoflife.write_releases(PRODUCT, dict( + # sort by date then version (desc) + sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) +)) +print("::endgroup::") diff --git a/src/linuxkernel.py b/src/linuxkernel.py index 9dd1b8ad..6d927dd2 100644 --- a/src/linuxkernel.py +++ b/src/linuxkernel.py @@ -1,4 +1,3 @@ -import json import re from bs4 import BeautifulSoup from common import endoflife @@ -51,17 +50,10 @@ def fetch_releases(): return releases -def main(): - print(f"::group::{PRODUCT}") - releases = fetch_releases() - print("::endgroup::") - - with open(f"releases/{PRODUCT}.json", "w") as f: - f.write(json.dumps(dict( - # sort by version then date (asc) - sorted(releases.items(), key=lambda x: (x[0], x[1])) - ), indent=2)) - - -if __name__ == '__main__': - main() +print(f"::group::{PRODUCT}") +releases = fetch_releases() +endoflife.write_releases(PRODUCT, dict( + # sort by version then date (asc) + sorted(releases.items(), key=lambda x: (x[0], x[1])) +)) +print("::endgroup::") diff --git a/src/maven.py b/src/maven.py index 1c42b27d..498f102c 100644 --- a/src/maven.py +++ b/src/maven.py @@ -50,11 +50,10 @@ def update_product(product_name, configs): for config in configs: releases = releases | fetch_releases(config[METHOD]) - with open(f"releases/{product_name}.json", "w") as f: - f.write(json.dumps(dict( - # sort by date then version (desc) - sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) - ), indent=2)) + endoflife.write_releases(product_name, dict( + # sort by date then version (desc) + sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) + )) p_filter = sys.argv[1] if len(sys.argv) > 1 else None diff --git a/src/npm.py b/src/npm.py index 79ae13fc..6a9df611 100644 --- a/src/npm.py +++ b/src/npm.py @@ -40,8 +40,7 @@ def update_product(product_name, configs): config = {"regex": REGEX} | config releases = releases | fetch_releases(config[METHOD], config["regex"]) - with open(f"releases/{product_name}.json", "w") as f: - f.write(json.dumps(releases, indent=2)) + endoflife.write_releases(product_name, releases) diff --git a/src/palo-alto-networks.py b/src/palo-alto-networks.py index 60c7737c..bc5351ff 100644 --- a/src/palo-alto-networks.py +++ b/src/palo-alto-networks.py @@ -1,5 +1,4 @@ import datetime -import json import re from bs4 import BeautifulSoup from common import endoflife @@ -19,6 +18,7 @@ def update_releases(html_identifier, file): print(f"::group::{html_identifier}") response = endoflife.fetch_url(URL) soup = BeautifulSoup(response, features="html5lib") + table = soup.find(id=html_identifier) for tr in table.findAll("tr")[3:]: td_list = tr.findAll("td") @@ -45,10 +45,9 @@ def update_releases(html_identifier, file): versions[version] = abs_date print("%s: %s" % (version, abs_date)) - print("::endgroup::") - with open("releases/%s.json" % file, "w") as f: - f.write(json.dumps(versions, indent=2)) + endoflife.write_releases(file, versions) + print("::endgroup::") for html_id in ID_MAPPING: diff --git a/src/php.py b/src/php.py index 4eb56f85..e81f45fe 100644 --- a/src/php.py +++ b/src/php.py @@ -26,21 +26,13 @@ def fetch_versions(major_version): return data -with open("releases/php.json", "w") as f: - print("::group::php") - releases = {} +print("::group::php") +releases = {} - for major_version in PHP_MAJOR_VERSIONS: - releases |= fetch_versions(major_version) +for major_version in PHP_MAJOR_VERSIONS: + releases |= fetch_versions(major_version) - f.write( - json.dumps( - dict(sorted( - releases.items(), - key=lambda x: list(map(int, x[0].split("."))) - )), - indent=2, - ) - ) - - print("::endgroup::") +endoflife.write_releases('php', dict(sorted( + releases.items(), key=lambda x: list(map(int, x[0].split("."))) +))) +print("::endgroup::") diff --git a/src/plesk.py b/src/plesk.py index 8d6e099f..44ad327b 100644 --- a/src/plesk.py +++ b/src/plesk.py @@ -1,4 +1,3 @@ -import json from bs4 import BeautifulSoup from common import endoflife from datetime import datetime @@ -35,18 +34,10 @@ def fetch_releases(): return result -def main(): - print(f"::group::{PRODUCT}") - - releases = fetch_releases() - with open(f"releases/{PRODUCT}.json", "w") as f: - f.write(json.dumps(dict( - # sort by date then version (desc) - sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) - ), indent=2)) - - print("::endgroup::") - - -if __name__ == '__main__': - main() +print(f"::group::{PRODUCT}") +releases = fetch_releases() +endoflife.write_releases(PRODUCT, dict( + # sort by date then version (desc) + sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) +)) +print("::endgroup::") diff --git a/src/pypi.py b/src/pypi.py index c14f8acf..62e762c8 100644 --- a/src/pypi.py +++ b/src/pypi.py @@ -41,8 +41,7 @@ def update_product(product_name, configs): config = {"regex": REGEX} | config releases = releases | fetch_releases(config[METHOD], config["regex"]) - with open(f"releases/{product_name}.json", "w") as f: - f.write(json.dumps(releases, indent=2)) + endoflife.write_releases(product_name, releases) p_filter = sys.argv[1] if len(sys.argv) > 1 else None diff --git a/src/rds.py b/src/rds.py index e8e17d91..32a4a8fe 100644 --- a/src/rds.py +++ b/src/rds.py @@ -1,4 +1,3 @@ -import json import re from bs4 import BeautifulSoup from common import endoflife @@ -36,9 +35,8 @@ for db, url in dbs.items(): print(f"{version} : {date}") releases[version] = date + endoflife.write_releases(f"amazon-rds-{db.lower()}", dict( + # sort by date then version (desc) + sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) + )) print("::endgroup::") - with open(f"releases/amazon-rds-{db.lower()}.json", "w") as f: - json.dump(dict( - # sort by date then version (desc) - sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) - ), f, indent=2) diff --git a/src/rhel.py b/src/rhel.py index 150000fe..bf60dad6 100644 --- a/src/rhel.py +++ b/src/rhel.py @@ -1,4 +1,3 @@ -import json import re from bs4 import BeautifulSoup from common import endoflife @@ -7,11 +6,11 @@ URL = "https://access.redhat.com/articles/3078" # https://regex101.com/r/877ibq/1 regex = r"RHEL (?P\d)(\. ?(?P\d+))?(( Update (?P\d))| GA)?" -versions = {} - print("::group::rhel") response = endoflife.fetch_url(URL) soup = BeautifulSoup(response, features="html5lib") + +versions = {} for tr in soup.findAll("tr"): td_list = tr.findAll("td") if len(td_list) > 0: @@ -25,7 +24,6 @@ for tr in soup.findAll("tr"): date = td_list[1].get_text() versions[version] = date print("%s: %s" % (version, date)) -print("::endgroup::") -with open("releases/redhat.json", "w") as f: - f.write(json.dumps(versions, indent=2)) +endoflife.write_releases('redhat', versions) +print("::endgroup::") diff --git a/src/ros.py b/src/ros.py index b7feebba..11450842 100644 --- a/src/ros.py +++ b/src/ros.py @@ -1,5 +1,4 @@ import datetime -import json import re from bs4 import BeautifulSoup from common import endoflife @@ -8,11 +7,11 @@ URL = "https://wiki.ros.org/Distributions" # https://regex101.com/r/c1ribd/1 regex = r"^ROS (?P(\w| )+)" -versions = {} - print("::group::ros") response = endoflife.fetch_url(URL) soup = BeautifulSoup(response, features="html5lib") + +versions = {} for tr in soup.findAll("tr"): td_list = tr.findAll("td") if len(td_list) > 0: @@ -32,7 +31,6 @@ for tr in soup.findAll("tr"): abs_date = date.strftime("%Y-%m-%d") versions[version] = abs_date print("%s: %s" % (version, abs_date)) -print("::endgroup::") -with open("releases/ros.json", "w") as f: - f.write(json.dumps(versions, indent=2)) +endoflife.write_releases('ros', versions) +print("::endgroup::") diff --git a/src/unrealircd.py b/src/unrealircd.py index 0a3d6da3..a8871cf7 100644 --- a/src/unrealircd.py +++ b/src/unrealircd.py @@ -20,7 +20,6 @@ for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"): if re.match(r"\d{4}-\d{2}-\d{2}", maybe_date): versions[maybe_version] = maybe_date print("%s: %s" % (maybe_version, maybe_date)) -print("::endgroup::") -with open("releases/unrealircd.json", "w") as f: - f.write(json.dumps(versions, indent=2)) +endoflife.write_releases('unrealircd', versions) +print("::endgroup::")