[github_releases] Refactor script (#226)

Make the script more readable, mostly by:

- using the Product and AutoConfig classes,
- removing the use of functions when unnecessary,
- a little bit of renaming and documentation.

src/common/endoflife.py

@@ -12,7 +12,7 @@ logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.INFO)
 # Major version must be >= 1.
 DEFAULT_VERSION_REGEX = r"^v?(?P<major>[1-9]\d*)\.(?P<minor>\d+)(\.(?P<patch>\d+)(\.(?P<tiny>\d+))?)?$"
 DEFAULT_VERSION_PATTERN = re.compile(DEFAULT_VERSION_REGEX)
-DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{%endif%}{%endif%}{%endif%}"
+DEFAULT_VERSION_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{% endif %}{% endif %}{% endif %}"
 
 PRODUCTS_PATH = os.environ.get("PRODUCTS_PATH", "website/products")
 VERSIONS_PATH = os.environ.get("VERSIONS_PATH", "releases")
@@ -22,7 +22,7 @@ class AutoConfig:
     def __init__(self, method: str, config: dict):
         self.method = method
         self.url = config[method]
-        self.version_template = Template(config.get("template", DEFAULT_TAG_TEMPLATE))
+        self.version_template = Template(config.get("template", DEFAULT_VERSION_TEMPLATE))
 
         regexes = config.get("regex", DEFAULT_VERSION_REGEX)
         regexes = regexes if isinstance(regexes, list) else [regexes]

src/git.py

@@ -38,7 +38,7 @@ def update_product(product_name, configs):
     versions = {}
 
     for config in configs:
-        t = config.get("template", endoflife.DEFAULT_TAG_TEMPLATE)
+        t = config.get("template", endoflife.DEFAULT_VERSION_TEMPLATE)
         regex = config.get("regex", endoflife.DEFAULT_VERSION_REGEX)
         regex = regex.replace("(?<", "(?P<")  # convert ruby regex to python regex
         versions = versions | fetch_releases(config[METHOD], regex, t)

src/github-releases.py

@@ -1,23 +1,22 @@
-import re
+import json
+import logging
 import sys
 import subprocess
+from common import dates
 from common import endoflife
 
+"""Fetches versions from GitHub releases using the GraphQL API and the GitHub CLI.
+
+Note: GraphQL API and GitHub CLI are used because it's simpler: no need to manage pagination and authentication.
+"""
+
 METHOD = "github_releases"
-REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
 
 
-# This script is using the GitHub CLI with the GraphQL API in order to retrieve
+def fetch_releases(repo_id):
-# releases. The reasons are:
+    logging.info(f"fetching {repo_id} GitHub releases")
-# - using 'gh release list' does not return all the releases
-# - using the API, directly or via GitHub CLI, is slow, produces a lot of 502
-#   errors, and is harder due to pagination.
-# - using the GraphQL API directly is hard due to pagination.
-# - using a library, such as graphql-python, is sightly harder than using the
-#   GitHub CLI (and still requires a GITHUB_TOKEN).
-def fetch_json(repo_id):
     (owner, repo) = repo_id.split('/')
-    query = """gh api graphql --paginate -f query='
+    child = subprocess.run("""gh api graphql --paginate -f query='
 query($endCursor: String) {
   repository(name: "%s", owner: "%s") {
     releases(
@@ -35,48 +34,31 @@ query($endCursor: String) {
       }
     }
   }
-}' --jq '.data.repository.releases.edges.[].node | select(.isPrerelease == false) | [.name, .publishedAt] | join(",")'
+}'""" % (repo, owner), capture_output=True, timeout=300, check=True, shell=True)  # noqa: UP031
-""" % (repo, owner)  # noqa: UP031
+    logging.info(f"fetched {repo_id} GitHub releases")
 
-    child = subprocess.Popen(query, shell=True, stdout=subprocess.PIPE)
+    # splitting because response may contain multiple JSON objects on a single line
-    return child.communicate()[0].decode('utf-8')
+    responses = child.stdout.decode("utf-8").strip().replace('}{', '}\n{').split("\n")
-
+    return [json.loads(response) for response in responses]
-
-def fetch_releases(repo_id, regex):
-    """Returns this repository releases using
-    https://docs.github.com/en/rest/releases/releases#list-releases. Only the
-    first page is fetched: there are rate limit rules in place on the GitHub
-    API, and the most recent releases are sufficient.
-    """
-    releases = {}
-    regex = [regex] if not isinstance(regex, list) else regex
-
-    for release in fetch_json(repo_id).splitlines():
-        (raw_version, raw_date) = release.split(',')
-
-        for r in regex:
-            match = re.search(r, raw_version)
-            if match:
-                version = match.group(1)
-                date = raw_date.split("T")[0]
-                releases[version] = date
-                print(f"{version}: {date}")
-
-    return releases
-
-
-def update_product(product_name, configs):
-    versions = {}
-
-    for config in configs:
-        config = config if "regex" in config else config | {"regex": REGEX}
-        versions = versions | fetch_releases(config[METHOD], config["regex"])
-
-    endoflife.write_releases(product_name, versions)
 
 
 p_filter = sys.argv[1] if len(sys.argv) > 1 else None
-for product, configs in endoflife.list_products(METHOD, p_filter).items():
+for product_name, configs in endoflife.list_products(METHOD, p_filter).items():
-    print(f"::group::{product}")
+    print(f"::group::{product_name}")
-    update_product(product, configs)
+    product = endoflife.Product(product_name, load_product_data=True)
+
+    for config in product.get_auto_configs(METHOD):
+        for page in fetch_releases(config.url):
+            releases = [edge['node'] for edge in (page['data']['repository']['releases']['edges'])]
+
+            for release in releases:
+                if not release['isPrerelease']:
+                    version_str = release['name']
+                    version_match = config.first_match(version_str)
+                    if version_match:
+                        version = config.render(version_match)
+                        date = dates.parse_datetime(release['publishedAt'])
+                        product.declare_version(version, date)
+
+    product.write()
     print("::endgroup::")