From 897d514e7f2e569d9425a84828db0937ff9dc829 Mon Sep 17 00:00:00 2001 From: Daniel <33197631+dadav@users.noreply.github.com> Date: Sat, 15 Jul 2023 18:26:11 +0200 Subject: [PATCH] Add red-hat-openshift script (#119) The script only get versions 4.x and above, see script comments for more informations. Note that the code related to git has been extracted to a common script so that it can be reused for the Debian script. --- src/common/__init__.py | 0 src/common/git.py | 76 ++++++++++++++++++++++++++++++++++++++++ src/debian.py | 40 ++++++--------------- src/red-hat-openshift.py | 52 +++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 30 deletions(-) create mode 100644 src/common/__init__.py create mode 100644 src/common/git.py create mode 100644 src/red-hat-openshift.py diff --git a/src/common/__init__.py b/src/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/common/git.py b/src/common/git.py new file mode 100644 index 00000000..9864688c --- /dev/null +++ b/src/common/git.py @@ -0,0 +1,76 @@ +from subprocess import call, Popen, PIPE, DEVNULL +from typing import Union, List +from pathlib import Path +from hashlib import sha1 + +class Git: + """ + Git cli wrapper + used by debian.py and red-hat-openshift.py + """ + + def __init__(self, url: str): + self.url: str = url + self.repo_dir: Path = Path( + f"~/.cache/git/{sha1(self.url.encode()).hexdigest()}" + ).expanduser() + + def _run(self, cmd: str, return_output: bool = False) -> Union[bool, list]: + """ + Run git command and return True on success or False on failure. + Optionaly returns command result instead. + """ + git_opts = f"--git-dir={self.repo_dir}/.git --work-tree={self.repo_dir}" + + if return_output: + child = Popen( + f"git {git_opts} {cmd}", + shell=True, + stdout=PIPE, + ) + return child.communicate()[0].decode("utf-8").split("\n") + + return call(f"git {git_opts} {cmd}", + shell=True, + stdout=DEVNULL, + stderr=DEVNULL) == 0 + + def setup(self): + """ + Creates the repository path and runs: + git init + git remote add origin $url + """ + self.repo_dir.mkdir(parents=True, exist_ok=True) + if not Path(f"{self.repo_dir}/.git").exists(): + self._run("init") + self._run(f"remote add origin {self.url}") + + def list_branches(self, pattern: str): + """ + Uses ls-remote to fetch the branch names + `pattern` uses fnmatch style globbing + """ + raw = self._run( + # only list v4+ branches because the format in v3 is different + f"ls-remote origin '{pattern}'", + return_output=True, + ) + + # this checks keeps the linter quiet, because _run returns a bool OR list + # please dont remove + if isinstance(raw, bool): + return [] + + return [line.split("\t")[1][11:] for line in raw if "\t" in line] + + def checkout(self, branch: str, file_list: List[str] = []): + """ + Checks out a branch + If `file_list` is given, sparse-checkout is used to save bandwith + and only download the given files + """ + if file_list: + self._run(f"sparse-checkout set {' '.join(file_list)}") + self._run(f"fetch --filter=blob:none --depth 1 origin {branch}") + self._run(f"checkout {branch}") diff --git a/src/debian.py b/src/debian.py index 57df3c4f..ef303721 100644 --- a/src/debian.py +++ b/src/debian.py @@ -1,39 +1,17 @@ -import pathlib import subprocess from common import endoflife -from hashlib import sha1 -from os.path import exists -from subprocess import call +from common.git import Git """Fetch Debian versions with their dates from www.debian.org source repository. """ PRODUCT = "debian" REPO_URL = "https://salsa.debian.org/webmaster-team/webwml.git" -REPO_SHA1 = sha1(REPO_URL.encode()).hexdigest() -REPO_DIR = pathlib.Path(f"~/.cache/git/debian.md_{REPO_SHA1}").expanduser() -# Checkout the Debian website repository. -def clone_repository(): - git_opts = f"--git-dir={REPO_DIR}/.git --work-tree={REPO_DIR}" - REPO_DIR.mkdir(parents=True, exist_ok=True) - - if not exists(f"{REPO_DIR}/.git"): - call(f"git {git_opts} init", shell=True) - call(f"git {git_opts} remote add origin {REPO_URL}", shell=True) - - call(f"git {git_opts} config core.sparseCheckout true", shell=True) - with open(f"{REPO_DIR}/.git/info/sparse-checkout", "w") as f: - f.write("english/News/") - - ret_code = call(f"git {git_opts} pull --depth 1 origin master", shell=True) - exit(-ret_code) if ret_code < 0 else None - - -def extract_major_releases(releases): +def extract_major_releases(releases, repo_dir): child = subprocess.Popen( - f"grep -RhE -A 1 'Debian [0-9]+.+ released' {REPO_DIR}/english/News " + f"grep -RhE -A 1 'Debian [0-9]+.+ released' {repo_dir}/english/News " f"| cut -d '<' -f 2 " f"| cut -d '>' -f 2 " f"| grep -v -- '--'", @@ -54,9 +32,9 @@ def extract_major_releases(releases): is_release_line = True -def extract_point_releases(releases): +def extract_point_releases(releases, repo_dir): child = subprocess.Popen( - f"grep -Rh -B 10 '' {REPO_DIR}/english/News " + f"grep -Rh -B 10 '' {repo_dir}/english/News " "| grep -Eo '(release_date>(.*)<|revision>(.*)<)' " "| cut -d '>' -f 2,4 " "| tr -d '<' " @@ -73,12 +51,14 @@ def extract_point_releases(releases): print(f"{version}: {date}") releases[version] = date +git = Git(REPO_URL) +git.setup() +git.checkout("master", file_list=["english/News"]) print(f"::group::{PRODUCT}") -clone_repository() all_releases = {} -extract_major_releases(all_releases) -extract_point_releases(all_releases) +extract_major_releases(all_releases, git.repo_dir) +extract_point_releases(all_releases, git.repo_dir) endoflife.write_releases(PRODUCT, dict( # sort by date then version (desc) sorted(all_releases.items(), key=lambda x: (x[1], x[0]), reverse=True) diff --git a/src/red-hat-openshift.py b/src/red-hat-openshift.py new file mode 100644 index 00000000..4e1ab599 --- /dev/null +++ b/src/red-hat-openshift.py @@ -0,0 +1,52 @@ +import re +from pathlib import Path +from common import endoflife +from common.git import Git + +""" +Fetch Red Hat OpenShift versions from the documentation git repository +""" + +PRODUCT = "red-hat-openshift" +REPO_URL = "https://github.com/openshift/openshift-docs.git" + + +def get_versions_from_file(release_notes_file: Path) -> dict: + if not release_notes_file.exists(): + return {} + + with open(release_notes_file, "rb") as f: + plain = f.read().decode("utf-8") + + return { + version: date + for (version, date) in re.findall( + r"{product-title}\s(?P\d+\.\d+\.\d+).*$\n+Issued:\s(?P\d{4}-\d\d-\d\d)$", + plain, + re.MULTILINE, + ) + } + +git = Git(REPO_URL) +git.setup() +versions = {} + +# only fetch v4+ branches, because the format was different in openshift v3 +for branch in git.list_branches("refs/heads/enterprise-[4-9]*"): + version = branch.split("-")[1].replace(".", "-") + release_notes_file = f"release_notes/ocp-{version}-release-notes.adoc" + git.checkout(branch, file_list=[release_notes_file]) + versions = {**versions, **get_versions_from_file(git.repo_dir / release_notes_file)} + +print(f"::group::{PRODUCT}") +for version, date in versions.items(): + print(f"{version}: {date}") +print("::endgroup::") + +endoflife.write_releases( + PRODUCT, + dict( + # sort by date then version (desc) + sorted(versions.items(), key=lambda x: (x[1], x[0]), reverse=True) + ), +)