[git] Migrate git method from Ruby to Python (#178)
The main reason for doing this is to have some common code between scripts, so that it is easier to change the JSON schema globally and normalize a few things (such as release order). The Ruby code was kept as is so we can quickly roll back if necessary.
This commit is contained in:
Marc Wrobel
@@ -1,124 +0,0 @@
|
|||||||
{
|
|
||||||
"0.0.1": "2012-03-28",
|
|
||||||
"0.0.2": "2012-07-16",
|
|
||||||
"0.0.3": "2012-10-28",
|
|
||||||
"0.0.4": "2012-12-11",
|
|
||||||
"1.0.0": "2013-11-23",
|
|
||||||
"1.0.1": "2013-11-23",
|
|
||||||
"1.0.2": "2013-11-23",
|
|
||||||
"1.0.3": "2013-11-23",
|
|
||||||
"1.0.4": "2013-11-23",
|
|
||||||
"1.0.5": "2013-11-23",
|
|
||||||
"1.0.6": "2013-11-23",
|
|
||||||
"1.0.7": "2013-11-23",
|
|
||||||
"1.0.8": "2013-11-23",
|
|
||||||
"1.1.0": "2013-11-23",
|
|
||||||
"1.1.1": "2013-11-23",
|
|
||||||
"1.1.2": "2013-11-23",
|
|
||||||
"1.1.3": "2013-11-23",
|
|
||||||
"1.1.4": "2013-11-23",
|
|
||||||
"1.1.5": "2013-11-23",
|
|
||||||
"1.2.0": "2013-11-23",
|
|
||||||
"1.2.1": "2013-11-23",
|
|
||||||
"1.2.2": "2013-11-23",
|
|
||||||
"1.2.7": "2014-01-07",
|
|
||||||
"1.2.3": "2014-01-07",
|
|
||||||
"1.2.4": "2014-01-07",
|
|
||||||
"1.2.5": "2014-01-07",
|
|
||||||
"1.2.6": "2014-01-07",
|
|
||||||
"1.2.8": "2014-01-27",
|
|
||||||
"1.2.9": "2014-01-27",
|
|
||||||
"1.2.10": "2014-01-27",
|
|
||||||
"1.2.11": "2014-03-26",
|
|
||||||
"1.2.12": "2014-03-26",
|
|
||||||
"1.2.13": "2014-03-26",
|
|
||||||
"1.2.14": "2014-03-26",
|
|
||||||
"1.2.15": "2014-03-26",
|
|
||||||
"1.2.16": "2014-04-18",
|
|
||||||
"1.2.17": "2014-06-19",
|
|
||||||
"1.2.18": "2014-06-19",
|
|
||||||
"1.2.19": "2014-07-02",
|
|
||||||
"1.2.20": "2014-08-11",
|
|
||||||
"1.2.21": "2014-08-11",
|
|
||||||
"1.2.22": "2014-08-31",
|
|
||||||
"1.2.23": "2014-08-31",
|
|
||||||
"1.3.0": "2014-10-13",
|
|
||||||
"1.3.1": "2014-10-31",
|
|
||||||
"1.3.2": "2014-11-07",
|
|
||||||
"1.3.3": "2014-11-18",
|
|
||||||
"1.2.27": "2014-11-20",
|
|
||||||
"1.3.4": "2014-11-24",
|
|
||||||
"1.3.5": "2014-12-02",
|
|
||||||
"1.3.36": "2014-12-09",
|
|
||||||
"1.3.6": "2014-12-09",
|
|
||||||
"1.3.7": "2014-12-15",
|
|
||||||
"1.2.28": "2014-12-15",
|
|
||||||
"1.3.8": "2014-12-19",
|
|
||||||
"1.3.9": "2015-01-14",
|
|
||||||
"1.3.10": "2015-01-20",
|
|
||||||
"1.3.11": "2015-01-26",
|
|
||||||
"1.3.12": "2015-02-03",
|
|
||||||
"1.3.13": "2015-02-09",
|
|
||||||
"1.3.14": "2015-02-24",
|
|
||||||
"1.3.15": "2015-03-17",
|
|
||||||
"1.4.0": "2015-05-27",
|
|
||||||
"1.3.16": "2015-06-05",
|
|
||||||
"1.4.1": "2015-06-16",
|
|
||||||
"1.4.2": "2015-07-06",
|
|
||||||
"1.3.17": "2015-07-06",
|
|
||||||
"1.4.3": "2015-07-15",
|
|
||||||
"1.4.4": "2015-08-13",
|
|
||||||
"1.3.18": "2015-08-18",
|
|
||||||
"1.4.5": "2015-08-28",
|
|
||||||
"1.3.19": "2015-09-16",
|
|
||||||
"1.4.6": "2015-09-17",
|
|
||||||
"1.2.29": "2015-09-29",
|
|
||||||
"1.3.20": "2015-09-29",
|
|
||||||
"1.4.7": "2015-09-29",
|
|
||||||
"1.4.8": "2015-11-20",
|
|
||||||
"1.4.9": "2016-01-21",
|
|
||||||
"1.5.0": "2016-02-05",
|
|
||||||
"1.5.1": "2016-03-16",
|
|
||||||
"1.4.10": "2016-03-16",
|
|
||||||
"1.5.2": "2016-03-18",
|
|
||||||
"1.5.3": "2016-03-25",
|
|
||||||
"1.5.5": "2016-04-18",
|
|
||||||
"1.4.11": "2016-05-27",
|
|
||||||
"1.5.6": "2016-05-27",
|
|
||||||
"1.4.12": "2016-06-15",
|
|
||||||
"1.5.7": "2016-06-15",
|
|
||||||
"1.2.30": "2016-07-21",
|
|
||||||
"1.5.8": "2016-07-22",
|
|
||||||
"1.4.13": "2016-10-10",
|
|
||||||
"1.2.31": "2016-10-11",
|
|
||||||
"1.2.32": "2016-10-11",
|
|
||||||
"1.4.14": "2016-10-11",
|
|
||||||
"1.5.9": "2016-11-24",
|
|
||||||
"1.6.0": "2016-12-08",
|
|
||||||
"1.5.10": "2016-12-16",
|
|
||||||
"1.6.1": "2016-12-23",
|
|
||||||
"1.5.11": "2017-01-12",
|
|
||||||
"1.6.2": "2017-02-07",
|
|
||||||
"1.6.3": "2017-03-08",
|
|
||||||
"1.6.4": "2017-03-31",
|
|
||||||
"1.6.5": "2017-07-03",
|
|
||||||
"1.6.6": "2017-08-18",
|
|
||||||
"1.6.7": "2017-11-24",
|
|
||||||
"1.6.8": "2017-12-21",
|
|
||||||
"1.6.9": "2018-02-02",
|
|
||||||
"1.6.10": "2018-04-17",
|
|
||||||
"1.7.0": "2018-05-11",
|
|
||||||
"1.7.1": "2018-06-08",
|
|
||||||
"1.7.2": "2018-06-12",
|
|
||||||
"1.7.3": "2018-08-08",
|
|
||||||
"1.7.4": "2018-09-07",
|
|
||||||
"1.7.5": "2018-10-04",
|
|
||||||
"1.7.6": "2019-01-17",
|
|
||||||
"1.7.7": "2019-02-04",
|
|
||||||
"1.7.8": "2019-03-11",
|
|
||||||
"1.7.9": "2019-11-19",
|
|
||||||
"1.8.0": "2020-06-04",
|
|
||||||
"1.8.1": "2020-10-05",
|
|
||||||
"1.8.2": "2020-10-21",
|
|
||||||
"1.8.3": "2022-04-07"
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
{
|
|
||||||
"9.2": "2023-05-26",
|
|
||||||
"9.1": "2022-11-30",
|
|
||||||
"8.7": "2022-11-21",
|
|
||||||
"9.0": "2022-07-06",
|
|
||||||
"8.6": "2022-05-20",
|
|
||||||
"8.4": "2021-05-27",
|
|
||||||
"8.3": "2020-11-13",
|
|
||||||
"7.9": "2020-10-08",
|
|
||||||
"8.2": "2020-06-21",
|
|
||||||
"8.1": "2019-11-17",
|
|
||||||
"8.0": "2019-07-19",
|
|
||||||
"7.6": "2018-11-07",
|
|
||||||
"7.5": "2018-04-18",
|
|
||||||
"7.4": "2017-09-09",
|
|
||||||
"6.9": "2017-03-28",
|
|
||||||
"7.3": "2016-11-11",
|
|
||||||
"7.2": "2015-11-26",
|
|
||||||
"6.7": "2015-07-31",
|
|
||||||
"7.1": "2015-03-17",
|
|
||||||
"6.6": "2014-10-23",
|
|
||||||
"5.11": "2014-09-26",
|
|
||||||
"7.0": "2014-07-23",
|
|
||||||
"6.5": "2013-12-01",
|
|
||||||
"6.3": "2012-06-29",
|
|
||||||
"5.8": "2012-03-03",
|
|
||||||
"6.2": "2011-12-20",
|
|
||||||
"5.7": "2011-08-02",
|
|
||||||
"6.1": "2011-06-01",
|
|
||||||
"6": "2011-02-12",
|
|
||||||
"5.6": "2011-01-23",
|
|
||||||
"5.5": "2010-04-14",
|
|
||||||
"5.4": "2009-09-16",
|
|
||||||
"4.4": "2006-10-26"
|
|
||||||
}
|
|
||||||
@@ -14,3 +14,4 @@ soupsieve==2.5
|
|||||||
typing_extensions==4.8.0
|
typing_extensions==4.8.0
|
||||||
webencodings==0.5.1
|
webencodings==0.5.1
|
||||||
requests-html==0.10.0
|
requests-html==0.10.0
|
||||||
|
regex==2023.10.3
|
||||||
|
|||||||
@@ -1,76 +1,66 @@
|
|||||||
from subprocess import call, Popen, PIPE, DEVNULL
|
|
||||||
from typing import Union, List
|
|
||||||
from pathlib import Path
|
|
||||||
from hashlib import sha1
|
from hashlib import sha1
|
||||||
|
from pathlib import Path
|
||||||
|
from subprocess import run, PIPE
|
||||||
|
|
||||||
|
|
||||||
class Git:
|
class Git:
|
||||||
"""
|
"""Git cli wrapper
|
||||||
Git cli wrapper
|
|
||||||
used by debian.py and red-hat-openshift.py
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, url: str):
|
def __init__(self, url: str):
|
||||||
self.url: str = url
|
self.url: str = url
|
||||||
self.repo_dir: Path = Path(
|
self.repo_dir: Path = Path(f"~/.cache/git/{sha1(url.encode()).hexdigest()}").expanduser()
|
||||||
f"~/.cache/git/{sha1(self.url.encode()).hexdigest()}"
|
|
||||||
).expanduser()
|
|
||||||
|
|
||||||
def _run(self, cmd: str, return_output: bool = False) -> Union[bool, list]:
|
def _run(self, cmd: str) -> list:
|
||||||
|
"""Run git command and return command result as a list of lines.
|
||||||
"""
|
"""
|
||||||
Run git command and return True on success or False on failure.
|
try:
|
||||||
Optionaly returns command result instead.
|
child = run(f"git {cmd}", capture_output=True, timeout=300, check=True, shell=True, cwd=self.repo_dir)
|
||||||
"""
|
return child.stdout.decode("utf-8").strip().split("\n")
|
||||||
git_opts = f"--git-dir={self.repo_dir}/.git --work-tree={self.repo_dir}"
|
except ChildProcessError as ex:
|
||||||
|
raise RuntimeError(f"Failed to run '{git_command}': {ex}")
|
||||||
|
|
||||||
if return_output:
|
def setup(self, bare: bool = False):
|
||||||
child = Popen(
|
"""Creates the repository path and runs:
|
||||||
f"git {git_opts} {cmd}",
|
|
||||||
shell=True,
|
|
||||||
stdout=PIPE,
|
|
||||||
)
|
|
||||||
return child.communicate()[0].decode("utf-8").split("\n")
|
|
||||||
|
|
||||||
return call(f"git {git_opts} {cmd}",
|
|
||||||
shell=True,
|
|
||||||
stdout=DEVNULL,
|
|
||||||
stderr=DEVNULL) == 0
|
|
||||||
|
|
||||||
def setup(self):
|
|
||||||
"""
|
|
||||||
Creates the repository path and runs:
|
|
||||||
git init
|
git init
|
||||||
git remote add origin $url
|
git remote add origin $url
|
||||||
"""
|
"""
|
||||||
|
if not Path(f"{self.repo_dir}").exists():
|
||||||
self.repo_dir.mkdir(parents=True, exist_ok=True)
|
self.repo_dir.mkdir(parents=True, exist_ok=True)
|
||||||
if not Path(f"{self.repo_dir}/.git").exists():
|
bare = "--bare" if bare else ""
|
||||||
self._run("init")
|
self._run(f"init {bare}")
|
||||||
self._run(f"remote add origin {self.url}")
|
self._run(f"remote add origin {self.url}")
|
||||||
|
|
||||||
|
# See https://stackoverflow.com/a/65746233/374236
|
||||||
|
def list_tags(self):
|
||||||
|
"""Fetch and return tags matching the given`pattern`"""
|
||||||
|
# See https://stackoverflow.com/a/65746233/374236
|
||||||
|
self._run("config --local extensions.partialClone true")
|
||||||
|
# Using --force to avoid error like "would clobber existing tag".
|
||||||
|
# See https://stackoverflow.com/questions/58031165/how-to-get-rid-of-would-clobber-existing-tag.
|
||||||
|
self._run("fetch --force --tags --filter=blob:none --depth=1 origin")
|
||||||
|
tags_with_date = self._run("tag --list --format='%(refname:strip=2) %(creatordate:short)'")
|
||||||
|
return [tag_with_date.split(" ") for tag_with_date in tags_with_date]
|
||||||
|
|
||||||
def list_branches(self, pattern: str):
|
def list_branches(self, pattern: str):
|
||||||
"""
|
"""Uses ls-remote to fetch the branch names
|
||||||
Uses ls-remote to fetch the branch names
|
|
||||||
`pattern` uses fnmatch style globbing
|
`pattern` uses fnmatch style globbing
|
||||||
"""
|
"""
|
||||||
raw = self._run(
|
lines = self._run(f"ls-remote origin '{pattern}'")
|
||||||
# only list v4+ branches because the format in v3 is different
|
|
||||||
f"ls-remote origin '{pattern}'",
|
|
||||||
return_output=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
# this checks keeps the linter quiet, because _run returns a bool OR list
|
# this checks keeps the linter quiet, because _run returns a bool OR list
|
||||||
# please dont remove
|
if isinstance(lines, bool):
|
||||||
if isinstance(raw, bool):
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
return [line.split("\t")[1][11:] for line in raw if "\t" in line]
|
return [line.split("\t")[1][11:] for line in lines if "\t" in line]
|
||||||
|
|
||||||
def checkout(self, branch: str, file_list: List[str] = []):
|
def checkout(self, branch: str, file_list=None):
|
||||||
"""
|
"""Checks out a branch
|
||||||
Checks out a branch
|
If `file_list` is given, sparse-checkout is used to save bandwidth
|
||||||
If `file_list` is given, sparse-checkout is used to save bandwith
|
|
||||||
and only download the given files
|
and only download the given files
|
||||||
"""
|
"""
|
||||||
if file_list:
|
if file_list:
|
||||||
self._run(f"sparse-checkout set {' '.join(file_list)}")
|
# --skip-checks needed to avoid error when file_list contains a file
|
||||||
|
self._run(f"sparse-checkout set --skip-checks {' '.join(file_list)}")
|
||||||
self._run(f"fetch --filter=blob:none --depth 1 origin {branch}")
|
self._run(f"fetch --filter=blob:none --depth 1 origin {branch}")
|
||||||
self._run(f"checkout {branch}")
|
self._run(f"checkout {branch}")
|
||||||
|
|||||||
@@ -51,11 +51,11 @@ def extract_point_releases(releases, repo_dir):
|
|||||||
print(f"{version}: {date}")
|
print(f"{version}: {date}")
|
||||||
releases[version] = date
|
releases[version] = date
|
||||||
|
|
||||||
|
print(f"::group::{PRODUCT}")
|
||||||
git = Git(REPO_URL)
|
git = Git(REPO_URL)
|
||||||
git.setup()
|
git.setup()
|
||||||
git.checkout("master", file_list=["english/News"])
|
git.checkout("master", file_list=["english/News"])
|
||||||
|
|
||||||
print(f"::group::{PRODUCT}")
|
|
||||||
all_releases = {}
|
all_releases = {}
|
||||||
extract_major_releases(all_releases, git.repo_dir)
|
extract_major_releases(all_releases, git.repo_dir)
|
||||||
extract_point_releases(all_releases, git.repo_dir)
|
extract_point_releases(all_releases, git.repo_dir)
|
||||||
|
|||||||
55
src/git.py
Normal file
55
src/git.py
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
import regex as re
|
||||||
|
import sys
|
||||||
|
from common import endoflife
|
||||||
|
from common.git import Git
|
||||||
|
from liquid import Template
|
||||||
|
|
||||||
|
"""Fetch versions with their dates from tags in a git repository.
|
||||||
|
|
||||||
|
This replace the old update.rb script.
|
||||||
|
|
||||||
|
Note that this script is using the regex module because the Python re module does not support
|
||||||
|
identically named groups (as used in the mariadb product).
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Default tag template and regex should include tiny version to properly handle blender,
|
||||||
|
# craft-cms, exim, gerrit, jquery, kdeplasma, kirby, logstash, nexus, silverstripe
|
||||||
|
# and tarantool versions.
|
||||||
|
METHOD = 'git'
|
||||||
|
DEFAULT_VERSION_REGEX = r"^v?(?P<major>[1-9]\d*)\.(?P<minor>\d+)(\.(?P<patch>\d+)(\.(?P<tiny>\d+))?)?$"
|
||||||
|
DEFAULT_TAG_TEMPLATE = "{{major}}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{%endif%}{%endif%}"
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_releases(product_name, url, regex, template):
|
||||||
|
releases = {}
|
||||||
|
|
||||||
|
git = Git(url)
|
||||||
|
git.setup(bare=True)
|
||||||
|
|
||||||
|
tags = git.list_tags()
|
||||||
|
for tag, date in tags:
|
||||||
|
match = re.match(regex, tag)
|
||||||
|
if match:
|
||||||
|
version = Template(template).render(match.groupdict())
|
||||||
|
releases[version] = date
|
||||||
|
print(f"{version}: {date}")
|
||||||
|
|
||||||
|
return releases
|
||||||
|
|
||||||
|
def update_product(product_name, configs):
|
||||||
|
releases = {}
|
||||||
|
|
||||||
|
for config in configs:
|
||||||
|
t = config.get("template", DEFAULT_TAG_TEMPLATE)
|
||||||
|
regex = config.get("regex", DEFAULT_VERSION_REGEX)
|
||||||
|
regex = regex.replace("(?<", "(?P<") # convert ruby regex to python regex
|
||||||
|
releases = releases | fetch_releases(product_name, config[METHOD], regex, t)
|
||||||
|
|
||||||
|
endoflife.write_releases(product_name, releases)
|
||||||
|
|
||||||
|
|
||||||
|
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
|
||||||
|
for product, configs in endoflife.list_products(METHOD, p_filter).items():
|
||||||
|
print(f"::group::{product}")
|
||||||
|
update_product(product, configs)
|
||||||
|
print("::endgroup::")
|
||||||
@@ -108,7 +108,7 @@ end
|
|||||||
|
|
||||||
def get_releases(product, config, i)
|
def get_releases(product, config, i)
|
||||||
type = get_update_type(config)
|
type = get_update_type(config)
|
||||||
if type == 'git'
|
if type == 'oldgit' # replaced by the git.py script, code kept for now to facilitate rollbacks
|
||||||
dir = get_cache_dir('git', product, config)
|
dir = get_cache_dir('git', product, config)
|
||||||
fetch_git_releases(dir, config)
|
fetch_git_releases(dir, config)
|
||||||
return get_releases_from_git(dir, config)
|
return get_releases_from_git(dir, config)
|
||||||
|
|||||||
Reference in New Issue
Block a user