Add red-hat-openshift script (#119)

The script only get versions 4.x and above, see script comments for more informations.

Note that the code related to git has been extracted to a common script so that it can be reused for the Debian script.

src/common/git.py

@@ -0,0 +1,76 @@
+from subprocess import call, Popen, PIPE, DEVNULL
+from typing import Union, List
+from pathlib import Path
+from hashlib import sha1
+
+class Git:
+    """
+    Git cli wrapper
+    used by debian.py and red-hat-openshift.py
+    """
+
+    def __init__(self, url: str):
+        self.url: str = url
+        self.repo_dir: Path = Path(
+            f"~/.cache/git/{sha1(self.url.encode()).hexdigest()}"
+        ).expanduser()
+
+    def _run(self, cmd: str, return_output: bool = False) -> Union[bool, list]:
+        """
+        Run git command and return True on success or False on failure.
+        Optionaly returns command result instead.
+        """
+        git_opts = f"--git-dir={self.repo_dir}/.git --work-tree={self.repo_dir}"
+
+        if return_output:
+            child = Popen(
+                f"git {git_opts} {cmd}",
+                shell=True,
+                stdout=PIPE,
+            )
+            return child.communicate()[0].decode("utf-8").split("\n")
+
+        return call(f"git {git_opts} {cmd}",
+                    shell=True,
+                    stdout=DEVNULL,
+                    stderr=DEVNULL) == 0
+
+    def setup(self):
+        """
+        Creates the repository path and runs:
+        git init
+        git remote add origin $url
+        """
+        self.repo_dir.mkdir(parents=True, exist_ok=True)
+        if not Path(f"{self.repo_dir}/.git").exists():
+            self._run("init")
+            self._run(f"remote add origin {self.url}")
+
+    def list_branches(self, pattern: str):
+        """
+        Uses ls-remote to fetch the branch names
+        `pattern` uses fnmatch style globbing
+        """
+        raw = self._run(
+            # only list v4+ branches because the format in v3 is different
+            f"ls-remote origin '{pattern}'",
+            return_output=True,
+        )
+
+        # this checks keeps the linter quiet, because _run returns a bool OR list
+        # please dont remove
+        if isinstance(raw, bool):
+            return []
+
+        return [line.split("\t")[1][11:] for line in raw if "\t" in line]
+
+    def checkout(self, branch: str, file_list: List[str] = []):
+        """
+        Checks out a branch
+        If `file_list` is given, sparse-checkout is used to save bandwith
+        and only download the given files
+        """
+        if file_list:
+            self._run(f"sparse-checkout set {' '.join(file_list)}")
+        self._run(f"fetch --filter=blob:none --depth 1 origin {branch}")
+        self._run(f"checkout {branch}")

src/debian.py

@@ -1,39 +1,17 @@
-import pathlib
 import subprocess
 from common import endoflife
-from hashlib import sha1
-from os.path import exists
-from subprocess import call
+from common.git import Git
 
 """Fetch Debian versions with their dates from www.debian.org source repository.
 """
 
 PRODUCT = "debian"
 REPO_URL = "https://salsa.debian.org/webmaster-team/webwml.git"
-REPO_SHA1 = sha1(REPO_URL.encode()).hexdigest()
-REPO_DIR = pathlib.Path(f"~/.cache/git/debian.md_{REPO_SHA1}").expanduser()
 
 
-# Checkout the Debian website repository.
-def clone_repository():
-    git_opts = f"--git-dir={REPO_DIR}/.git --work-tree={REPO_DIR}"
-    REPO_DIR.mkdir(parents=True, exist_ok=True)
-
-    if not exists(f"{REPO_DIR}/.git"):
-        call(f"git {git_opts} init", shell=True)
-        call(f"git {git_opts} remote add origin {REPO_URL}", shell=True)
-
-    call(f"git {git_opts} config core.sparseCheckout true", shell=True)
-    with open(f"{REPO_DIR}/.git/info/sparse-checkout", "w") as f:
-        f.write("english/News/")
-
-    ret_code = call(f"git {git_opts} pull --depth 1 origin master", shell=True)
-    exit(-ret_code) if ret_code < 0 else None
-
-
-def extract_major_releases(releases):
+def extract_major_releases(releases, repo_dir):
     child = subprocess.Popen(
-        f"grep -RhE -A 1 '<define-tag pagetitle>Debian [0-9]+.+</q> released' {REPO_DIR}/english/News "
+        f"grep -RhE -A 1 '<define-tag pagetitle>Debian [0-9]+.+</q> released' {repo_dir}/english/News "
         f"| cut -d '<' -f 2 "
         f"| cut -d '>' -f 2 "
         f"| grep -v -- '--'",
@@ -54,9 +32,9 @@ def extract_major_releases(releases):
                 is_release_line = True
 
 
-def extract_point_releases(releases):
+def extract_point_releases(releases, repo_dir):
     child = subprocess.Popen(
-        f"grep -Rh -B 10 '<define-tag revision>' {REPO_DIR}/english/News "
+        f"grep -Rh -B 10 '<define-tag revision>' {repo_dir}/english/News "
         "| grep -Eo '(release_date>(.*)<|revision>(.*)<)' "
         "| cut -d '>' -f 2,4 "
         "| tr -d '<' "
@@ -73,12 +51,14 @@ def extract_point_releases(releases):
             print(f"{version}: {date}")
             releases[version] = date
 
+git = Git(REPO_URL)
+git.setup()
+git.checkout("master", file_list=["english/News"])
 
 print(f"::group::{PRODUCT}")
-clone_repository()
 all_releases = {}
-extract_major_releases(all_releases)
-extract_point_releases(all_releases)
+extract_major_releases(all_releases, git.repo_dir)
+extract_point_releases(all_releases, git.repo_dir)
 endoflife.write_releases(PRODUCT, dict(
     # sort by date then version (desc)
     sorted(all_releases.items(), key=lambda x: (x[1], x[0]), reverse=True)

src/red-hat-openshift.py

@@ -0,0 +1,52 @@
+import re
+from pathlib import Path
+from common import endoflife
+from common.git import Git
+
+"""
+Fetch Red Hat OpenShift versions from the documentation git repository
+"""
+
+PRODUCT = "red-hat-openshift"
+REPO_URL = "https://github.com/openshift/openshift-docs.git"
+
+
+def get_versions_from_file(release_notes_file: Path) -> dict:
+    if not release_notes_file.exists():
+        return {}
+
+    with open(release_notes_file, "rb") as f:
+        plain = f.read().decode("utf-8")
+
+    return {
+        version: date
+        for (version, date) in re.findall(
+            r"{product-title}\s(?P<version>\d+\.\d+\.\d+).*$\n+Issued:\s(?P<date>\d{4}-\d\d-\d\d)$",
+            plain,
+            re.MULTILINE,
+        )
+    }
+
+git = Git(REPO_URL)
+git.setup()
+versions = {}
+
+# only fetch v4+ branches, because the format was different in openshift v3
+for branch in git.list_branches("refs/heads/enterprise-[4-9]*"):
+    version = branch.split("-")[1].replace(".", "-")
+    release_notes_file = f"release_notes/ocp-{version}-release-notes.adoc"
+    git.checkout(branch, file_list=[release_notes_file])
+    versions = {**versions, **get_versions_from_file(git.repo_dir / release_notes_file)}
+
+print(f"::group::{PRODUCT}")
+for version, date in versions.items():
+    print(f"{version}: {date}")
+print("::endgroup::")
+
+endoflife.write_releases(
+    PRODUCT,
+    dict(
+        # sort by date then version (desc)
+        sorted(versions.items(), key=lambda x: (x[1], x[0]), reverse=True)
+    ),
+)