From 701c2899d50b7e83c31dca8c8757a4e958e871f1 Mon Sep 17 00:00:00 2001 From: Marc Wrobel Date: Sun, 12 Nov 2023 20:33:29 +0100 Subject: [PATCH] [git] Migrate git method from Ruby to Python (#178) The main reason for doing this is to have some common code between scripts, so that it is easier to change the JSON schema globally and normalize a few things (such as release order). The Ruby code was kept as is so we can quickly roll back if necessary. --- releases/angular-js.json | 124 -------------------------------------- releases/oraclelinux.json | 35 ----------- requirements.txt | 1 + src/common/git.py | 86 ++++++++++++-------------- src/debian.py | 2 +- src/git.py | 55 +++++++++++++++++ update.rb | 2 +- 7 files changed, 96 insertions(+), 209 deletions(-) delete mode 100644 releases/angular-js.json delete mode 100644 releases/oraclelinux.json create mode 100644 src/git.py diff --git a/releases/angular-js.json b/releases/angular-js.json deleted file mode 100644 index d4f05e61..00000000 --- a/releases/angular-js.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "0.0.1": "2012-03-28", - "0.0.2": "2012-07-16", - "0.0.3": "2012-10-28", - "0.0.4": "2012-12-11", - "1.0.0": "2013-11-23", - "1.0.1": "2013-11-23", - "1.0.2": "2013-11-23", - "1.0.3": "2013-11-23", - "1.0.4": "2013-11-23", - "1.0.5": "2013-11-23", - "1.0.6": "2013-11-23", - "1.0.7": "2013-11-23", - "1.0.8": "2013-11-23", - "1.1.0": "2013-11-23", - "1.1.1": "2013-11-23", - "1.1.2": "2013-11-23", - "1.1.3": "2013-11-23", - "1.1.4": "2013-11-23", - "1.1.5": "2013-11-23", - "1.2.0": "2013-11-23", - "1.2.1": "2013-11-23", - "1.2.2": "2013-11-23", - "1.2.7": "2014-01-07", - "1.2.3": "2014-01-07", - "1.2.4": "2014-01-07", - "1.2.5": "2014-01-07", - "1.2.6": "2014-01-07", - "1.2.8": "2014-01-27", - "1.2.9": "2014-01-27", - "1.2.10": "2014-01-27", - "1.2.11": "2014-03-26", - "1.2.12": "2014-03-26", - "1.2.13": "2014-03-26", - "1.2.14": "2014-03-26", - "1.2.15": "2014-03-26", - "1.2.16": "2014-04-18", - "1.2.17": "2014-06-19", - "1.2.18": "2014-06-19", - "1.2.19": "2014-07-02", - "1.2.20": "2014-08-11", - "1.2.21": "2014-08-11", - "1.2.22": "2014-08-31", - "1.2.23": "2014-08-31", - "1.3.0": "2014-10-13", - "1.3.1": "2014-10-31", - "1.3.2": "2014-11-07", - "1.3.3": "2014-11-18", - "1.2.27": "2014-11-20", - "1.3.4": "2014-11-24", - "1.3.5": "2014-12-02", - "1.3.36": "2014-12-09", - "1.3.6": "2014-12-09", - "1.3.7": "2014-12-15", - "1.2.28": "2014-12-15", - "1.3.8": "2014-12-19", - "1.3.9": "2015-01-14", - "1.3.10": "2015-01-20", - "1.3.11": "2015-01-26", - "1.3.12": "2015-02-03", - "1.3.13": "2015-02-09", - "1.3.14": "2015-02-24", - "1.3.15": "2015-03-17", - "1.4.0": "2015-05-27", - "1.3.16": "2015-06-05", - "1.4.1": "2015-06-16", - "1.4.2": "2015-07-06", - "1.3.17": "2015-07-06", - "1.4.3": "2015-07-15", - "1.4.4": "2015-08-13", - "1.3.18": "2015-08-18", - "1.4.5": "2015-08-28", - "1.3.19": "2015-09-16", - "1.4.6": "2015-09-17", - "1.2.29": "2015-09-29", - "1.3.20": "2015-09-29", - "1.4.7": "2015-09-29", - "1.4.8": "2015-11-20", - "1.4.9": "2016-01-21", - "1.5.0": "2016-02-05", - "1.5.1": "2016-03-16", - "1.4.10": "2016-03-16", - "1.5.2": "2016-03-18", - "1.5.3": "2016-03-25", - "1.5.5": "2016-04-18", - "1.4.11": "2016-05-27", - "1.5.6": "2016-05-27", - "1.4.12": "2016-06-15", - "1.5.7": "2016-06-15", - "1.2.30": "2016-07-21", - "1.5.8": "2016-07-22", - "1.4.13": "2016-10-10", - "1.2.31": "2016-10-11", - "1.2.32": "2016-10-11", - "1.4.14": "2016-10-11", - "1.5.9": "2016-11-24", - "1.6.0": "2016-12-08", - "1.5.10": "2016-12-16", - "1.6.1": "2016-12-23", - "1.5.11": "2017-01-12", - "1.6.2": "2017-02-07", - "1.6.3": "2017-03-08", - "1.6.4": "2017-03-31", - "1.6.5": "2017-07-03", - "1.6.6": "2017-08-18", - "1.6.7": "2017-11-24", - "1.6.8": "2017-12-21", - "1.6.9": "2018-02-02", - "1.6.10": "2018-04-17", - "1.7.0": "2018-05-11", - "1.7.1": "2018-06-08", - "1.7.2": "2018-06-12", - "1.7.3": "2018-08-08", - "1.7.4": "2018-09-07", - "1.7.5": "2018-10-04", - "1.7.6": "2019-01-17", - "1.7.7": "2019-02-04", - "1.7.8": "2019-03-11", - "1.7.9": "2019-11-19", - "1.8.0": "2020-06-04", - "1.8.1": "2020-10-05", - "1.8.2": "2020-10-21", - "1.8.3": "2022-04-07" -} \ No newline at end of file diff --git a/releases/oraclelinux.json b/releases/oraclelinux.json deleted file mode 100644 index 2a7cff11..00000000 --- a/releases/oraclelinux.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "9.2": "2023-05-26", - "9.1": "2022-11-30", - "8.7": "2022-11-21", - "9.0": "2022-07-06", - "8.6": "2022-05-20", - "8.4": "2021-05-27", - "8.3": "2020-11-13", - "7.9": "2020-10-08", - "8.2": "2020-06-21", - "8.1": "2019-11-17", - "8.0": "2019-07-19", - "7.6": "2018-11-07", - "7.5": "2018-04-18", - "7.4": "2017-09-09", - "6.9": "2017-03-28", - "7.3": "2016-11-11", - "7.2": "2015-11-26", - "6.7": "2015-07-31", - "7.1": "2015-03-17", - "6.6": "2014-10-23", - "5.11": "2014-09-26", - "7.0": "2014-07-23", - "6.5": "2013-12-01", - "6.3": "2012-06-29", - "5.8": "2012-03-03", - "6.2": "2011-12-20", - "5.7": "2011-08-02", - "6.1": "2011-06-01", - "6": "2011-02-12", - "5.6": "2011-01-23", - "5.5": "2010-04-14", - "5.4": "2009-09-16", - "4.4": "2006-10-26" -} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0e782929..b3a649e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ soupsieve==2.5 typing_extensions==4.8.0 webencodings==0.5.1 requests-html==0.10.0 +regex==2023.10.3 diff --git a/src/common/git.py b/src/common/git.py index 9864688c..5456cbfe 100644 --- a/src/common/git.py +++ b/src/common/git.py @@ -1,76 +1,66 @@ -from subprocess import call, Popen, PIPE, DEVNULL -from typing import Union, List -from pathlib import Path from hashlib import sha1 +from pathlib import Path +from subprocess import run, PIPE + class Git: - """ - Git cli wrapper - used by debian.py and red-hat-openshift.py + """Git cli wrapper """ def __init__(self, url: str): self.url: str = url - self.repo_dir: Path = Path( - f"~/.cache/git/{sha1(self.url.encode()).hexdigest()}" - ).expanduser() + self.repo_dir: Path = Path(f"~/.cache/git/{sha1(url.encode()).hexdigest()}").expanduser() - def _run(self, cmd: str, return_output: bool = False) -> Union[bool, list]: + def _run(self, cmd: str) -> list: + """Run git command and return command result as a list of lines. """ - Run git command and return True on success or False on failure. - Optionaly returns command result instead. - """ - git_opts = f"--git-dir={self.repo_dir}/.git --work-tree={self.repo_dir}" + try: + child = run(f"git {cmd}", capture_output=True, timeout=300, check=True, shell=True, cwd=self.repo_dir) + return child.stdout.decode("utf-8").strip().split("\n") + except ChildProcessError as ex: + raise RuntimeError(f"Failed to run '{git_command}': {ex}") - if return_output: - child = Popen( - f"git {git_opts} {cmd}", - shell=True, - stdout=PIPE, - ) - return child.communicate()[0].decode("utf-8").split("\n") - - return call(f"git {git_opts} {cmd}", - shell=True, - stdout=DEVNULL, - stderr=DEVNULL) == 0 - - def setup(self): - """ - Creates the repository path and runs: + def setup(self, bare: bool = False): + """Creates the repository path and runs: git init git remote add origin $url """ - self.repo_dir.mkdir(parents=True, exist_ok=True) - if not Path(f"{self.repo_dir}/.git").exists(): - self._run("init") + if not Path(f"{self.repo_dir}").exists(): + self.repo_dir.mkdir(parents=True, exist_ok=True) + bare = "--bare" if bare else "" + self._run(f"init {bare}") self._run(f"remote add origin {self.url}") + # See https://stackoverflow.com/a/65746233/374236 + def list_tags(self): + """Fetch and return tags matching the given`pattern`""" + # See https://stackoverflow.com/a/65746233/374236 + self._run("config --local extensions.partialClone true") + # Using --force to avoid error like "would clobber existing tag". + # See https://stackoverflow.com/questions/58031165/how-to-get-rid-of-would-clobber-existing-tag. + self._run("fetch --force --tags --filter=blob:none --depth=1 origin") + tags_with_date = self._run("tag --list --format='%(refname:strip=2) %(creatordate:short)'") + return [tag_with_date.split(" ") for tag_with_date in tags_with_date] + def list_branches(self, pattern: str): - """ - Uses ls-remote to fetch the branch names + """Uses ls-remote to fetch the branch names `pattern` uses fnmatch style globbing """ - raw = self._run( - # only list v4+ branches because the format in v3 is different - f"ls-remote origin '{pattern}'", - return_output=True, - ) + lines = self._run(f"ls-remote origin '{pattern}'") # this checks keeps the linter quiet, because _run returns a bool OR list - # please dont remove - if isinstance(raw, bool): + if isinstance(lines, bool): return [] - return [line.split("\t")[1][11:] for line in raw if "\t" in line] + return [line.split("\t")[1][11:] for line in lines if "\t" in line] - def checkout(self, branch: str, file_list: List[str] = []): - """ - Checks out a branch - If `file_list` is given, sparse-checkout is used to save bandwith + def checkout(self, branch: str, file_list=None): + """Checks out a branch + If `file_list` is given, sparse-checkout is used to save bandwidth and only download the given files """ if file_list: - self._run(f"sparse-checkout set {' '.join(file_list)}") + # --skip-checks needed to avoid error when file_list contains a file + self._run(f"sparse-checkout set --skip-checks {' '.join(file_list)}") self._run(f"fetch --filter=blob:none --depth 1 origin {branch}") self._run(f"checkout {branch}") diff --git a/src/debian.py b/src/debian.py index ef303721..9a4f10d0 100644 --- a/src/debian.py +++ b/src/debian.py @@ -51,11 +51,11 @@ def extract_point_releases(releases, repo_dir): print(f"{version}: {date}") releases[version] = date +print(f"::group::{PRODUCT}") git = Git(REPO_URL) git.setup() git.checkout("master", file_list=["english/News"]) -print(f"::group::{PRODUCT}") all_releases = {} extract_major_releases(all_releases, git.repo_dir) extract_point_releases(all_releases, git.repo_dir) diff --git a/src/git.py b/src/git.py new file mode 100644 index 00000000..f153dc6f --- /dev/null +++ b/src/git.py @@ -0,0 +1,55 @@ +import regex as re +import sys +from common import endoflife +from common.git import Git +from liquid import Template + +"""Fetch versions with their dates from tags in a git repository. + +This replace the old update.rb script. + +Note that this script is using the regex module because the Python re module does not support +identically named groups (as used in the mariadb product). +""" + +# Default tag template and regex should include tiny version to properly handle blender, +# craft-cms, exim, gerrit, jquery, kdeplasma, kirby, logstash, nexus, silverstripe +# and tarantool versions. +METHOD = 'git' +DEFAULT_VERSION_REGEX = r"^v?(?P[1-9]\d*)\.(?P\d+)(\.(?P\d+)(\.(?P\d+))?)?$" +DEFAULT_TAG_TEMPLATE = "{{major}}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{%endif%}{%endif%}" + + +def fetch_releases(product_name, url, regex, template): + releases = {} + + git = Git(url) + git.setup(bare=True) + + tags = git.list_tags() + for tag, date in tags: + match = re.match(regex, tag) + if match: + version = Template(template).render(match.groupdict()) + releases[version] = date + print(f"{version}: {date}") + + return releases + +def update_product(product_name, configs): + releases = {} + + for config in configs: + t = config.get("template", DEFAULT_TAG_TEMPLATE) + regex = config.get("regex", DEFAULT_VERSION_REGEX) + regex = regex.replace("(?<", "(?P<") # convert ruby regex to python regex + releases = releases | fetch_releases(product_name, config[METHOD], regex, t) + + endoflife.write_releases(product_name, releases) + + +p_filter = sys.argv[1] if len(sys.argv) > 1 else None +for product, configs in endoflife.list_products(METHOD, p_filter).items(): + print(f"::group::{product}") + update_product(product, configs) + print("::endgroup::") diff --git a/update.rb b/update.rb index 1a13071b..40d1d2c9 100644 --- a/update.rb +++ b/update.rb @@ -108,7 +108,7 @@ end def get_releases(product, config, i) type = get_update_type(config) - if type == 'git' + if type == 'oldgit' # replaced by the git.py script, code kept for now to facilitate rollbacks dir = get_cache_dir('git', product, config) fetch_git_releases(dir, config) return get_releases_from_git(dir, config)