Refactor product filtering and loading for generic scripts

Generic scripts are scripts that handle multiple product based on some identifier (URL, coordinates...).
This creates a common function to list and load product configurations for those scripts.

It makes them simpler to read and maintain, while making the way they work much more consistent.

src/common/endoflife.py

@@ -0,0 +1,27 @@
+import frontmatter
+
+from glob import glob
+from os import path
+
+
+def list_products(method, products_filter=None, pathname = "website/products"):
+    """Return a list of products that are using the same given update method.
+    """
+    products_with_method = {}
+
+    for product_file in glob(f"{pathname}/*.md"):
+        product_name = path.splitext(path.basename(product_file))[0]
+        if products_filter and product_name != products_filter:
+            continue
+
+        with open(product_file, "r") as f:
+            data = frontmatter.load(f)
+            if "auto" in data:
+                configs = list(filter(
+                    lambda config: method in config.keys(),
+                    data["auto"]
+                ))
+                if len(configs) > 0:
+                    products_with_method[product_name] = configs
+
+    return products_with_method

src/distrowatch.py

@@ -1,15 +1,13 @@
-from glob import glob
-import os
 import re
 import sys
 import json
-import frontmatter
 import urllib.request
 from bs4 import BeautifulSoup
 from liquid import Template
+from common import endoflife
 
-# Same as used in Ruby (update.rb)
-DEFAULT_TAG_TEMPLATE = (
+METHOD = 'distrowatch'
+DEFAULT_TAG_TEMPLATE = (  # Same as used in Ruby (update.rb)
     "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
 )
 
@@ -42,32 +40,21 @@ def fetch_releases(distrowatch_id, regex, template):
     return releases
 
 
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
-        with open(product_file, "r") as f:
-            data = frontmatter.load(f)
-            if "auto" in data:
-                for config in data["auto"]:
-                    for key, d_id in config.items():
-                        if key == "distrowatch":
-                            update_product(product_name, config)
+def update_product(product_name, configs):
+    releases = {}
+
+    for config in configs:
+        t = config.get("template", DEFAULT_TAG_TEMPLATE)
+        if "regex" in config:
+            regex = config["regex"]
+            releases = releases | fetch_releases(config[METHOD], regex, t)
+
+    with open("releases/%s.json" % product_name, "w") as f:
+        f.write(json.dumps(releases, indent=2))
 
 
-def update_product(product_name, config):
-    t = config.get("template", DEFAULT_TAG_TEMPLATE)
-    if "regex" in config:
-        print("::group::%s" % product_name)
-        r = fetch_releases(config["distrowatch"], config["regex"], t)
-        with open("releases/%s.json" % product_name, "w") as f:
-            f.write(json.dumps(r, indent=2))
-        print("::endgroup::")
-
-
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
+    print("::endgroup::")

src/github-releases.py

@@ -1,13 +1,11 @@
-from glob import glob
-import os
 import re
 import sys
 import json
-import frontmatter
 import subprocess
+from common import endoflife
 
+METHOD = "github_releases"
 REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
-AUTO_KEY = "github_releases"
 
 
 # This script is using the GitHub CLI with the GraphQL API in order to retrieve
@@ -68,34 +66,18 @@ def fetch_releases(repo_id, regex):
 
 
 def update_product(product_name, configs):
-    print("::group::%s" % product_name)
     releases = {}
 
     for config in configs:
         config = config if "regex" in config else config | {"regex": REGEX}
-        releases = releases | fetch_releases(config[AUTO_KEY], config["regex"])
+        releases = releases | fetch_releases(config[METHOD], config["regex"])
 
     with open(f"releases/{product_name}.json", "w") as f:
         f.write(json.dumps(releases, indent=2))
+
+
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
     print("::endgroup::")
-
-
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
-
-        with open(product_file, "r") as f:
-            data = frontmatter.load(f)
-            if "auto" in data:
-                configs = list(filter(lambda config: AUTO_KEY in config.keys(), data["auto"]))
-                if len(configs) > 0:
-                    update_product(product_name, configs)
-
-
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()

src/maven.py

@@ -1,13 +1,11 @@
-from glob import glob
-import os
 import sys
 import json
-import frontmatter
 import urllib.request
 import datetime
 import re
+from common import endoflife
 
-# Major.Minor + Optional Patch, no RC, nightly releases.
+METHOD = "maven"
 VERSION_REGEX = r'^\d+\.\d+(\.\d+)?$'
 
 
@@ -59,37 +57,21 @@ def fetch_releases(package_identifier):
     return releases
 
 
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
-        with open(product_file, "r") as f:
-            releases = {}
-            found_maven = False
-            print("::group::%s" % product_name)
-            data = frontmatter.load(f)
-            if "auto" in data:
-                for config in data["auto"]:
-                    for key, _ in config.items():
-                        if key == "maven":
-                            found_maven = True
-                            releases = releases | fetch_releases(config["maven"])
-            if found_maven:
-                write_file(product_name, releases)
-            print("::endgroup::")
+def update_product(product_name, configs):
+    releases = {}
 
+    for config in configs:
+        releases = releases | fetch_releases(config[METHOD])
 
-def write_file(product_name, releases):
-    with open("releases/%s.json" % product_name, "w") as f:
+    with open(f"releases/{product_name}.json", "w") as f:
         f.write(json.dumps(dict(
             # sort by date then version (desc)
             sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
         ), indent=2))
 
 
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
+    print("::endgroup::")

src/npm.py

@@ -1,12 +1,11 @@
-from glob import glob
-import os
 import re
 import sys
 import json
-import frontmatter
 import urllib.request
+from common import endoflife
 
-DEFAULT_TAG_TEMPLATE = (
+METHOD = "npm"
+DEFAULT_TAG_TEMPLATE = (  # Same as used in Ruby (update.rb)
     "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
 )
 REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
@@ -35,34 +34,21 @@ def fetch_releases(npm_id, regex):
     return releases
 
 
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
+def update_product(product_name, configs):
+    releases = {}
 
-        with open(product_file, "r") as f:
-            data = frontmatter.load(f)
-            if "auto" in data:
-                for config in data["auto"]:
-                    for key, d_id in config.items():
-                        if key == "npm":
-                            update_product(product_name, config)
-
-
-def update_product(product_name, config):
-    if "npm" in config:
-        print(f"::group::{product_name}")
+    for config in configs:
         config = {"regex": REGEX} | config
-        r = fetch_releases(config["npm"], config["regex"])
-        print("::endgroup::")
+        releases = releases | fetch_releases(config[METHOD], config["regex"])
 
-        with open(f"releases/{product_name}.json", "w") as f:
-            f.write(json.dumps(r, indent=2))
+    with open(f"releases/{product_name}.json", "w") as f:
+        f.write(json.dumps(releases, indent=2))
 
 
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()
+
+
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
+    print("::endgroup::")

src/pypi.py

@@ -1,14 +1,12 @@
-from glob import glob
-import os
 import re
 import sys
 import json
-import frontmatter
 import urllib.request
 from datetime import datetime
+from common import endoflife
 
-# Same as used in Ruby (update.rb)
-DEFAULT_TAG_TEMPLATE = (
+METHOD = "pypi"
+DEFAULT_TAG_TEMPLATE = (  # Same as used in Ruby (update.rb)
     "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
 )
 REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
@@ -37,32 +35,19 @@ def fetch_releases(pypi_id, regex):
     return releases
 
 
-def update_releases(product_filter=None):
-    for product_file in glob("website/products/*.md"):
-        product_name = os.path.splitext(os.path.basename(product_file))[0]
-        if product_filter and product_name != product_filter:
-            continue
-        with open(product_file, "r") as f:
-            data = frontmatter.load(f)
-            if "auto" in data:
-                for config in data["auto"]:
-                    for key, d_id in config.items():
-                        if key == "pypi":
-                            update_product(product_name, config)
+def update_product(product_name, configs):
+    releases = {}
 
-
-def update_product(product_name, config):
-    if "pypi" in config:
-        print("::group::%s" % product_name)
+    for config in configs:
         config = {"regex": REGEX} | config
-        r = fetch_releases(config["pypi"], config["regex"])
-        with open("releases/%s.json" % product_name, "w") as f:
-            f.write(json.dumps(r, indent=2))
-        print("::endgroup::")
+        releases = releases | fetch_releases(config[METHOD], config["regex"])
+
+    with open(f"releases/{product_name}.json", "w") as f:
+        f.write(json.dumps(releases, indent=2))
 
 
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        update_releases(sys.argv[1])
-    else:
-        update_releases()
+p_filter = sys.argv[1] if len(sys.argv) > 1 else None
+for product, configs in endoflife.list_products(METHOD, p_filter).items():
+    print("::group::%s" % product)
+    update_product(product, configs)
+    print("::endgroup::")