diff --git a/.gitignore b/.gitignore index 2f403d42..cafd421b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ website vendor .bundle pyrightconfig.json +__pycache__ diff --git a/src/common/endoflife.py b/src/common/endoflife.py new file mode 100644 index 00000000..f3afb708 --- /dev/null +++ b/src/common/endoflife.py @@ -0,0 +1,27 @@ +import frontmatter + +from glob import glob +from os import path + + +def list_products(method, products_filter=None, pathname = "website/products"): + """Return a list of products that are using the same given update method. + """ + products_with_method = {} + + for product_file in glob(f"{pathname}/*.md"): + product_name = path.splitext(path.basename(product_file))[0] + if products_filter and product_name != products_filter: + continue + + with open(product_file, "r") as f: + data = frontmatter.load(f) + if "auto" in data: + configs = list(filter( + lambda config: method in config.keys(), + data["auto"] + )) + if len(configs) > 0: + products_with_method[product_name] = configs + + return products_with_method diff --git a/src/distrowatch.py b/src/distrowatch.py index f1c73c78..669ff6fe 100644 --- a/src/distrowatch.py +++ b/src/distrowatch.py @@ -1,15 +1,13 @@ -from glob import glob -import os import re import sys import json -import frontmatter import urllib.request from bs4 import BeautifulSoup from liquid import Template +from common import endoflife -# Same as used in Ruby (update.rb) -DEFAULT_TAG_TEMPLATE = ( +METHOD = 'distrowatch' +DEFAULT_TAG_TEMPLATE = ( # Same as used in Ruby (update.rb) "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}" ) @@ -42,32 +40,21 @@ def fetch_releases(distrowatch_id, regex, template): return releases -def update_releases(product_filter=None): - for product_file in glob("website/products/*.md"): - product_name = os.path.splitext(os.path.basename(product_file))[0] - if product_filter and product_name != product_filter: - continue - with open(product_file, "r") as f: - data = frontmatter.load(f) - if "auto" in data: - for config in data["auto"]: - for key, d_id in config.items(): - if key == "distrowatch": - update_product(product_name, config) +def update_product(product_name, configs): + releases = {} + + for config in configs: + t = config.get("template", DEFAULT_TAG_TEMPLATE) + if "regex" in config: + regex = config["regex"] + releases = releases | fetch_releases(config[METHOD], regex, t) + + with open("releases/%s.json" % product_name, "w") as f: + f.write(json.dumps(releases, indent=2)) -def update_product(product_name, config): - t = config.get("template", DEFAULT_TAG_TEMPLATE) - if "regex" in config: - print("::group::%s" % product_name) - r = fetch_releases(config["distrowatch"], config["regex"], t) - with open("releases/%s.json" % product_name, "w") as f: - f.write(json.dumps(r, indent=2)) - print("::endgroup::") - - -if __name__ == "__main__": - if len(sys.argv) > 1: - update_releases(sys.argv[1]) - else: - update_releases() +p_filter = sys.argv[1] if len(sys.argv) > 1 else None +for product, configs in endoflife.list_products(METHOD, p_filter).items(): + print("::group::%s" % product) + update_product(product, configs) + print("::endgroup::") diff --git a/src/github-releases.py b/src/github-releases.py index 1ba0fe5a..125be934 100644 --- a/src/github-releases.py +++ b/src/github-releases.py @@ -1,13 +1,11 @@ -from glob import glob -import os import re import sys import json -import frontmatter import subprocess +from common import endoflife +METHOD = "github_releases" REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$" -AUTO_KEY = "github_releases" # This script is using the GitHub CLI with the GraphQL API in order to retrieve @@ -68,34 +66,18 @@ def fetch_releases(repo_id, regex): def update_product(product_name, configs): - print("::group::%s" % product_name) releases = {} for config in configs: config = config if "regex" in config else config | {"regex": REGEX} - releases = releases | fetch_releases(config[AUTO_KEY], config["regex"]) + releases = releases | fetch_releases(config[METHOD], config["regex"]) with open(f"releases/{product_name}.json", "w") as f: f.write(json.dumps(releases, indent=2)) + + +p_filter = sys.argv[1] if len(sys.argv) > 1 else None +for product, configs in endoflife.list_products(METHOD, p_filter).items(): + print("::group::%s" % product) + update_product(product, configs) print("::endgroup::") - - -def update_releases(product_filter=None): - for product_file in glob("website/products/*.md"): - product_name = os.path.splitext(os.path.basename(product_file))[0] - if product_filter and product_name != product_filter: - continue - - with open(product_file, "r") as f: - data = frontmatter.load(f) - if "auto" in data: - configs = list(filter(lambda config: AUTO_KEY in config.keys(), data["auto"])) - if len(configs) > 0: - update_product(product_name, configs) - - -if __name__ == "__main__": - if len(sys.argv) > 1: - update_releases(sys.argv[1]) - else: - update_releases() diff --git a/src/maven.py b/src/maven.py index 577ff4c3..238ae07b 100644 --- a/src/maven.py +++ b/src/maven.py @@ -1,13 +1,11 @@ -from glob import glob -import os import sys import json -import frontmatter import urllib.request import datetime import re +from common import endoflife -# Major.Minor + Optional Patch, no RC, nightly releases. +METHOD = "maven" VERSION_REGEX = r'^\d+\.\d+(\.\d+)?$' @@ -59,37 +57,21 @@ def fetch_releases(package_identifier): return releases -def update_releases(product_filter=None): - for product_file in glob("website/products/*.md"): - product_name = os.path.splitext(os.path.basename(product_file))[0] - if product_filter and product_name != product_filter: - continue - with open(product_file, "r") as f: - releases = {} - found_maven = False - print("::group::%s" % product_name) - data = frontmatter.load(f) - if "auto" in data: - for config in data["auto"]: - for key, _ in config.items(): - if key == "maven": - found_maven = True - releases = releases | fetch_releases(config["maven"]) - if found_maven: - write_file(product_name, releases) - print("::endgroup::") +def update_product(product_name, configs): + releases = {} + for config in configs: + releases = releases | fetch_releases(config[METHOD]) -def write_file(product_name, releases): - with open("releases/%s.json" % product_name, "w") as f: + with open(f"releases/{product_name}.json", "w") as f: f.write(json.dumps(dict( # sort by date then version (desc) sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) ), indent=2)) -if __name__ == "__main__": - if len(sys.argv) > 1: - update_releases(sys.argv[1]) - else: - update_releases() +p_filter = sys.argv[1] if len(sys.argv) > 1 else None +for product, configs in endoflife.list_products(METHOD, p_filter).items(): + print("::group::%s" % product) + update_product(product, configs) + print("::endgroup::") diff --git a/src/npm.py b/src/npm.py index 95413ea5..9befb616 100644 --- a/src/npm.py +++ b/src/npm.py @@ -1,12 +1,11 @@ -from glob import glob -import os import re import sys import json -import frontmatter import urllib.request +from common import endoflife -DEFAULT_TAG_TEMPLATE = ( +METHOD = "npm" +DEFAULT_TAG_TEMPLATE = ( # Same as used in Ruby (update.rb) "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}" ) REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$" @@ -35,34 +34,21 @@ def fetch_releases(npm_id, regex): return releases -def update_releases(product_filter=None): - for product_file in glob("website/products/*.md"): - product_name = os.path.splitext(os.path.basename(product_file))[0] - if product_filter and product_name != product_filter: - continue +def update_product(product_name, configs): + releases = {} - with open(product_file, "r") as f: - data = frontmatter.load(f) - if "auto" in data: - for config in data["auto"]: - for key, d_id in config.items(): - if key == "npm": - update_product(product_name, config) - - -def update_product(product_name, config): - if "npm" in config: - print(f"::group::{product_name}") + for config in configs: config = {"regex": REGEX} | config - r = fetch_releases(config["npm"], config["regex"]) - print("::endgroup::") + releases = releases | fetch_releases(config[METHOD], config["regex"]) - with open(f"releases/{product_name}.json", "w") as f: - f.write(json.dumps(r, indent=2)) + with open(f"releases/{product_name}.json", "w") as f: + f.write(json.dumps(releases, indent=2)) -if __name__ == "__main__": - if len(sys.argv) > 1: - update_releases(sys.argv[1]) - else: - update_releases() + + +p_filter = sys.argv[1] if len(sys.argv) > 1 else None +for product, configs in endoflife.list_products(METHOD, p_filter).items(): + print("::group::%s" % product) + update_product(product, configs) + print("::endgroup::") diff --git a/src/pypi.py b/src/pypi.py index 4f94bced..0d0f8e0a 100644 --- a/src/pypi.py +++ b/src/pypi.py @@ -1,14 +1,12 @@ -from glob import glob -import os import re import sys import json -import frontmatter import urllib.request from datetime import datetime +from common import endoflife -# Same as used in Ruby (update.rb) -DEFAULT_TAG_TEMPLATE = ( +METHOD = "pypi" +DEFAULT_TAG_TEMPLATE = ( # Same as used in Ruby (update.rb) "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}" ) REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$" @@ -37,32 +35,19 @@ def fetch_releases(pypi_id, regex): return releases -def update_releases(product_filter=None): - for product_file in glob("website/products/*.md"): - product_name = os.path.splitext(os.path.basename(product_file))[0] - if product_filter and product_name != product_filter: - continue - with open(product_file, "r") as f: - data = frontmatter.load(f) - if "auto" in data: - for config in data["auto"]: - for key, d_id in config.items(): - if key == "pypi": - update_product(product_name, config) +def update_product(product_name, configs): + releases = {} - -def update_product(product_name, config): - if "pypi" in config: - print("::group::%s" % product_name) + for config in configs: config = {"regex": REGEX} | config - r = fetch_releases(config["pypi"], config["regex"]) - with open("releases/%s.json" % product_name, "w") as f: - f.write(json.dumps(r, indent=2)) - print("::endgroup::") + releases = releases | fetch_releases(config[METHOD], config["regex"]) + + with open(f"releases/{product_name}.json", "w") as f: + f.write(json.dumps(releases, indent=2)) -if __name__ == "__main__": - if len(sys.argv) > 1: - update_releases(sys.argv[1]) - else: - update_releases() +p_filter = sys.argv[1] if len(sys.argv) > 1 else None +for product, configs in endoflife.list_products(METHOD, p_filter).items(): + print("::group::%s" % product) + update_product(product, configs) + print("::endgroup::")