[git] Migrate git method from Ruby to Python (#178)
The main reason for doing this is to have some common code between scripts, so that it is easier to change the JSON schema globally and normalize a few things (such as release order). The Ruby code was kept as is so we can quickly roll back if necessary.
This commit is contained in:
Marc Wrobel
@@ -1,124 +0,0 @@
|
||||
{
|
||||
"0.0.1": "2012-03-28",
|
||||
"0.0.2": "2012-07-16",
|
||||
"0.0.3": "2012-10-28",
|
||||
"0.0.4": "2012-12-11",
|
||||
"1.0.0": "2013-11-23",
|
||||
"1.0.1": "2013-11-23",
|
||||
"1.0.2": "2013-11-23",
|
||||
"1.0.3": "2013-11-23",
|
||||
"1.0.4": "2013-11-23",
|
||||
"1.0.5": "2013-11-23",
|
||||
"1.0.6": "2013-11-23",
|
||||
"1.0.7": "2013-11-23",
|
||||
"1.0.8": "2013-11-23",
|
||||
"1.1.0": "2013-11-23",
|
||||
"1.1.1": "2013-11-23",
|
||||
"1.1.2": "2013-11-23",
|
||||
"1.1.3": "2013-11-23",
|
||||
"1.1.4": "2013-11-23",
|
||||
"1.1.5": "2013-11-23",
|
||||
"1.2.0": "2013-11-23",
|
||||
"1.2.1": "2013-11-23",
|
||||
"1.2.2": "2013-11-23",
|
||||
"1.2.7": "2014-01-07",
|
||||
"1.2.3": "2014-01-07",
|
||||
"1.2.4": "2014-01-07",
|
||||
"1.2.5": "2014-01-07",
|
||||
"1.2.6": "2014-01-07",
|
||||
"1.2.8": "2014-01-27",
|
||||
"1.2.9": "2014-01-27",
|
||||
"1.2.10": "2014-01-27",
|
||||
"1.2.11": "2014-03-26",
|
||||
"1.2.12": "2014-03-26",
|
||||
"1.2.13": "2014-03-26",
|
||||
"1.2.14": "2014-03-26",
|
||||
"1.2.15": "2014-03-26",
|
||||
"1.2.16": "2014-04-18",
|
||||
"1.2.17": "2014-06-19",
|
||||
"1.2.18": "2014-06-19",
|
||||
"1.2.19": "2014-07-02",
|
||||
"1.2.20": "2014-08-11",
|
||||
"1.2.21": "2014-08-11",
|
||||
"1.2.22": "2014-08-31",
|
||||
"1.2.23": "2014-08-31",
|
||||
"1.3.0": "2014-10-13",
|
||||
"1.3.1": "2014-10-31",
|
||||
"1.3.2": "2014-11-07",
|
||||
"1.3.3": "2014-11-18",
|
||||
"1.2.27": "2014-11-20",
|
||||
"1.3.4": "2014-11-24",
|
||||
"1.3.5": "2014-12-02",
|
||||
"1.3.36": "2014-12-09",
|
||||
"1.3.6": "2014-12-09",
|
||||
"1.3.7": "2014-12-15",
|
||||
"1.2.28": "2014-12-15",
|
||||
"1.3.8": "2014-12-19",
|
||||
"1.3.9": "2015-01-14",
|
||||
"1.3.10": "2015-01-20",
|
||||
"1.3.11": "2015-01-26",
|
||||
"1.3.12": "2015-02-03",
|
||||
"1.3.13": "2015-02-09",
|
||||
"1.3.14": "2015-02-24",
|
||||
"1.3.15": "2015-03-17",
|
||||
"1.4.0": "2015-05-27",
|
||||
"1.3.16": "2015-06-05",
|
||||
"1.4.1": "2015-06-16",
|
||||
"1.4.2": "2015-07-06",
|
||||
"1.3.17": "2015-07-06",
|
||||
"1.4.3": "2015-07-15",
|
||||
"1.4.4": "2015-08-13",
|
||||
"1.3.18": "2015-08-18",
|
||||
"1.4.5": "2015-08-28",
|
||||
"1.3.19": "2015-09-16",
|
||||
"1.4.6": "2015-09-17",
|
||||
"1.2.29": "2015-09-29",
|
||||
"1.3.20": "2015-09-29",
|
||||
"1.4.7": "2015-09-29",
|
||||
"1.4.8": "2015-11-20",
|
||||
"1.4.9": "2016-01-21",
|
||||
"1.5.0": "2016-02-05",
|
||||
"1.5.1": "2016-03-16",
|
||||
"1.4.10": "2016-03-16",
|
||||
"1.5.2": "2016-03-18",
|
||||
"1.5.3": "2016-03-25",
|
||||
"1.5.5": "2016-04-18",
|
||||
"1.4.11": "2016-05-27",
|
||||
"1.5.6": "2016-05-27",
|
||||
"1.4.12": "2016-06-15",
|
||||
"1.5.7": "2016-06-15",
|
||||
"1.2.30": "2016-07-21",
|
||||
"1.5.8": "2016-07-22",
|
||||
"1.4.13": "2016-10-10",
|
||||
"1.2.31": "2016-10-11",
|
||||
"1.2.32": "2016-10-11",
|
||||
"1.4.14": "2016-10-11",
|
||||
"1.5.9": "2016-11-24",
|
||||
"1.6.0": "2016-12-08",
|
||||
"1.5.10": "2016-12-16",
|
||||
"1.6.1": "2016-12-23",
|
||||
"1.5.11": "2017-01-12",
|
||||
"1.6.2": "2017-02-07",
|
||||
"1.6.3": "2017-03-08",
|
||||
"1.6.4": "2017-03-31",
|
||||
"1.6.5": "2017-07-03",
|
||||
"1.6.6": "2017-08-18",
|
||||
"1.6.7": "2017-11-24",
|
||||
"1.6.8": "2017-12-21",
|
||||
"1.6.9": "2018-02-02",
|
||||
"1.6.10": "2018-04-17",
|
||||
"1.7.0": "2018-05-11",
|
||||
"1.7.1": "2018-06-08",
|
||||
"1.7.2": "2018-06-12",
|
||||
"1.7.3": "2018-08-08",
|
||||
"1.7.4": "2018-09-07",
|
||||
"1.7.5": "2018-10-04",
|
||||
"1.7.6": "2019-01-17",
|
||||
"1.7.7": "2019-02-04",
|
||||
"1.7.8": "2019-03-11",
|
||||
"1.7.9": "2019-11-19",
|
||||
"1.8.0": "2020-06-04",
|
||||
"1.8.1": "2020-10-05",
|
||||
"1.8.2": "2020-10-21",
|
||||
"1.8.3": "2022-04-07"
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
{
|
||||
"9.2": "2023-05-26",
|
||||
"9.1": "2022-11-30",
|
||||
"8.7": "2022-11-21",
|
||||
"9.0": "2022-07-06",
|
||||
"8.6": "2022-05-20",
|
||||
"8.4": "2021-05-27",
|
||||
"8.3": "2020-11-13",
|
||||
"7.9": "2020-10-08",
|
||||
"8.2": "2020-06-21",
|
||||
"8.1": "2019-11-17",
|
||||
"8.0": "2019-07-19",
|
||||
"7.6": "2018-11-07",
|
||||
"7.5": "2018-04-18",
|
||||
"7.4": "2017-09-09",
|
||||
"6.9": "2017-03-28",
|
||||
"7.3": "2016-11-11",
|
||||
"7.2": "2015-11-26",
|
||||
"6.7": "2015-07-31",
|
||||
"7.1": "2015-03-17",
|
||||
"6.6": "2014-10-23",
|
||||
"5.11": "2014-09-26",
|
||||
"7.0": "2014-07-23",
|
||||
"6.5": "2013-12-01",
|
||||
"6.3": "2012-06-29",
|
||||
"5.8": "2012-03-03",
|
||||
"6.2": "2011-12-20",
|
||||
"5.7": "2011-08-02",
|
||||
"6.1": "2011-06-01",
|
||||
"6": "2011-02-12",
|
||||
"5.6": "2011-01-23",
|
||||
"5.5": "2010-04-14",
|
||||
"5.4": "2009-09-16",
|
||||
"4.4": "2006-10-26"
|
||||
}
|
||||
@@ -14,3 +14,4 @@ soupsieve==2.5
|
||||
typing_extensions==4.8.0
|
||||
webencodings==0.5.1
|
||||
requests-html==0.10.0
|
||||
regex==2023.10.3
|
||||
|
||||
@@ -1,76 +1,66 @@
|
||||
from subprocess import call, Popen, PIPE, DEVNULL
|
||||
from typing import Union, List
|
||||
from pathlib import Path
|
||||
from hashlib import sha1
|
||||
from pathlib import Path
|
||||
from subprocess import run, PIPE
|
||||
|
||||
|
||||
class Git:
|
||||
"""
|
||||
Git cli wrapper
|
||||
used by debian.py and red-hat-openshift.py
|
||||
"""Git cli wrapper
|
||||
"""
|
||||
|
||||
def __init__(self, url: str):
|
||||
self.url: str = url
|
||||
self.repo_dir: Path = Path(
|
||||
f"~/.cache/git/{sha1(self.url.encode()).hexdigest()}"
|
||||
).expanduser()
|
||||
self.repo_dir: Path = Path(f"~/.cache/git/{sha1(url.encode()).hexdigest()}").expanduser()
|
||||
|
||||
def _run(self, cmd: str, return_output: bool = False) -> Union[bool, list]:
|
||||
def _run(self, cmd: str) -> list:
|
||||
"""Run git command and return command result as a list of lines.
|
||||
"""
|
||||
Run git command and return True on success or False on failure.
|
||||
Optionaly returns command result instead.
|
||||
"""
|
||||
git_opts = f"--git-dir={self.repo_dir}/.git --work-tree={self.repo_dir}"
|
||||
try:
|
||||
child = run(f"git {cmd}", capture_output=True, timeout=300, check=True, shell=True, cwd=self.repo_dir)
|
||||
return child.stdout.decode("utf-8").strip().split("\n")
|
||||
except ChildProcessError as ex:
|
||||
raise RuntimeError(f"Failed to run '{git_command}': {ex}")
|
||||
|
||||
if return_output:
|
||||
child = Popen(
|
||||
f"git {git_opts} {cmd}",
|
||||
shell=True,
|
||||
stdout=PIPE,
|
||||
)
|
||||
return child.communicate()[0].decode("utf-8").split("\n")
|
||||
|
||||
return call(f"git {git_opts} {cmd}",
|
||||
shell=True,
|
||||
stdout=DEVNULL,
|
||||
stderr=DEVNULL) == 0
|
||||
|
||||
def setup(self):
|
||||
"""
|
||||
Creates the repository path and runs:
|
||||
def setup(self, bare: bool = False):
|
||||
"""Creates the repository path and runs:
|
||||
git init
|
||||
git remote add origin $url
|
||||
"""
|
||||
self.repo_dir.mkdir(parents=True, exist_ok=True)
|
||||
if not Path(f"{self.repo_dir}/.git").exists():
|
||||
self._run("init")
|
||||
if not Path(f"{self.repo_dir}").exists():
|
||||
self.repo_dir.mkdir(parents=True, exist_ok=True)
|
||||
bare = "--bare" if bare else ""
|
||||
self._run(f"init {bare}")
|
||||
self._run(f"remote add origin {self.url}")
|
||||
|
||||
# See https://stackoverflow.com/a/65746233/374236
|
||||
def list_tags(self):
|
||||
"""Fetch and return tags matching the given`pattern`"""
|
||||
# See https://stackoverflow.com/a/65746233/374236
|
||||
self._run("config --local extensions.partialClone true")
|
||||
# Using --force to avoid error like "would clobber existing tag".
|
||||
# See https://stackoverflow.com/questions/58031165/how-to-get-rid-of-would-clobber-existing-tag.
|
||||
self._run("fetch --force --tags --filter=blob:none --depth=1 origin")
|
||||
tags_with_date = self._run("tag --list --format='%(refname:strip=2) %(creatordate:short)'")
|
||||
return [tag_with_date.split(" ") for tag_with_date in tags_with_date]
|
||||
|
||||
def list_branches(self, pattern: str):
|
||||
"""
|
||||
Uses ls-remote to fetch the branch names
|
||||
"""Uses ls-remote to fetch the branch names
|
||||
`pattern` uses fnmatch style globbing
|
||||
"""
|
||||
raw = self._run(
|
||||
# only list v4+ branches because the format in v3 is different
|
||||
f"ls-remote origin '{pattern}'",
|
||||
return_output=True,
|
||||
)
|
||||
lines = self._run(f"ls-remote origin '{pattern}'")
|
||||
|
||||
# this checks keeps the linter quiet, because _run returns a bool OR list
|
||||
# please dont remove
|
||||
if isinstance(raw, bool):
|
||||
if isinstance(lines, bool):
|
||||
return []
|
||||
|
||||
return [line.split("\t")[1][11:] for line in raw if "\t" in line]
|
||||
return [line.split("\t")[1][11:] for line in lines if "\t" in line]
|
||||
|
||||
def checkout(self, branch: str, file_list: List[str] = []):
|
||||
"""
|
||||
Checks out a branch
|
||||
If `file_list` is given, sparse-checkout is used to save bandwith
|
||||
def checkout(self, branch: str, file_list=None):
|
||||
"""Checks out a branch
|
||||
If `file_list` is given, sparse-checkout is used to save bandwidth
|
||||
and only download the given files
|
||||
"""
|
||||
if file_list:
|
||||
self._run(f"sparse-checkout set {' '.join(file_list)}")
|
||||
# --skip-checks needed to avoid error when file_list contains a file
|
||||
self._run(f"sparse-checkout set --skip-checks {' '.join(file_list)}")
|
||||
self._run(f"fetch --filter=blob:none --depth 1 origin {branch}")
|
||||
self._run(f"checkout {branch}")
|
||||
|
||||
@@ -51,11 +51,11 @@ def extract_point_releases(releases, repo_dir):
|
||||
print(f"{version}: {date}")
|
||||
releases[version] = date
|
||||
|
||||
print(f"::group::{PRODUCT}")
|
||||
git = Git(REPO_URL)
|
||||
git.setup()
|
||||
git.checkout("master", file_list=["english/News"])
|
||||
|
||||
print(f"::group::{PRODUCT}")
|
||||
all_releases = {}
|
||||
extract_major_releases(all_releases, git.repo_dir)
|
||||
extract_point_releases(all_releases, git.repo_dir)
|
||||
|
||||
55
src/git.py
Normal file
55
src/git.py
Normal file
@@ -0,0 +1,55 @@
|
||||
import regex as re
|
||||
import sys
|
||||
from common import endoflife
|
||||
from common.git import Git
|
||||
from liquid import Template
|
||||
|
||||
"""Fetch versions with their dates from tags in a git repository.
|
||||
|
||||
This replace the old update.rb script.
|
||||
|
||||
Note that this script is using the regex module because the Python re module does not support
|
||||
identically named groups (as used in the mariadb product).
|
||||
"""
|
||||
|
||||
# Default tag template and regex should include tiny version to properly handle blender,
|
||||
# craft-cms, exim, gerrit, jquery, kdeplasma, kirby, logstash, nexus, silverstripe
|
||||
# and tarantool versions.
|
||||
METHOD = 'git'
|
||||
DEFAULT_VERSION_REGEX = r"^v?(?P<major>[1-9]\d*)\.(?P<minor>\d+)(\.(?P<patch>\d+)(\.(?P<tiny>\d+))?)?$"
|
||||
DEFAULT_TAG_TEMPLATE = "{{major}}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{%endif%}{%endif%}"
|
||||
|
||||
|
||||
def fetch_releases(product_name, url, regex, template):
|
||||
releases = {}
|
||||
|
||||
git = Git(url)
|
||||
git.setup(bare=True)
|
||||
|
||||
tags = git.list_tags()
|
||||
for tag, date in tags:
|
||||
match = re.match(regex, tag)
|
||||
if match:
|
||||
version = Template(template).render(match.groupdict())
|
||||
releases[version] = date
|
||||
print(f"{version}: {date}")
|
||||
|
||||
return releases
|
||||
|
||||
def update_product(product_name, configs):
|
||||
releases = {}
|
||||
|
||||
for config in configs:
|
||||
t = config.get("template", DEFAULT_TAG_TEMPLATE)
|
||||
regex = config.get("regex", DEFAULT_VERSION_REGEX)
|
||||
regex = regex.replace("(?<", "(?P<") # convert ruby regex to python regex
|
||||
releases = releases | fetch_releases(product_name, config[METHOD], regex, t)
|
||||
|
||||
endoflife.write_releases(product_name, releases)
|
||||
|
||||
|
||||
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
for product, configs in endoflife.list_products(METHOD, p_filter).items():
|
||||
print(f"::group::{product}")
|
||||
update_product(product, configs)
|
||||
print("::endgroup::")
|
||||
@@ -108,7 +108,7 @@ end
|
||||
|
||||
def get_releases(product, config, i)
|
||||
type = get_update_type(config)
|
||||
if type == 'git'
|
||||
if type == 'oldgit' # replaced by the git.py script, code kept for now to facilitate rollbacks
|
||||
dir = get_cache_dir('git', product, config)
|
||||
fetch_git_releases(dir, config)
|
||||
return get_releases_from_git(dir, config)
|
||||
|
||||
Reference in New Issue
Block a user