diff --git a/src/apple.py b/src/apple.py index bb406d37..dfeffed6 100644 --- a/src/apple.py +++ b/src/apple.py @@ -51,10 +51,7 @@ VERSION_PATTERNS = { DATE_PATTERN = re.compile(r"\b\d+\s[A-Za-z]+\s\d+\b") -logging.info("::group::apple") soups = [BeautifulSoup(response.text, features="html5lib") for response in http.fetch_urls(URLS)] -logging.info("::endgroup::") - for product_name in VERSION_PATTERNS: with releasedata.ProductData(product_name) as product_data: for soup in soups: diff --git a/src/aws-lambda.py b/src/aws-lambda.py index 4c75d617..d06001e8 100644 --- a/src/aws-lambda.py +++ b/src/aws-lambda.py @@ -28,8 +28,6 @@ with releasedata.ProductData("aws-lambda") as product_data: identifier = cells[identifier_index].get_text().strip() date = product_frontmatter.get_release_date(identifier) # use the product releaseDate if available - if date is None: - date = product_data.get_previous_version(identifier).date() # else use the previously found date if date is None: date = dates.today() # else use today's date diff --git a/src/cgit.py b/src/cgit.py index c5948c4f..2845d50a 100644 --- a/src/cgit.py +++ b/src/cgit.py @@ -9,28 +9,28 @@ Ideally we would want to use the git repository directly, but cgit-managed repos METHOD = "cgit" p_filter = sys.argv[1] if len(sys.argv) > 1 else None -for product in endoflife.list_products(METHOD, p_filter): - with releasedata.ProductData(product.name) as product_data: - for config in product.get_auto_configs(METHOD): - response = http.fetch_url(config.url + '/refs/tags') - soup = BeautifulSoup(response.text, features="html5lib") +m_filter = sys.argv[2] if len(sys.argv) > 2 else None +for config in endoflife.list_configs(p_filter, METHOD, m_filter): + with releasedata.ProductData(config.product) as product_data: + response = http.fetch_url(config.url + '/refs/tags') + soup = BeautifulSoup(response.text, features="html5lib") - for table in soup.find_all("table", class_="list"): - for row in table.find_all("tr"): - columns = row.find_all("td") - if len(columns) != 4: - continue + for table in soup.find_all("table", class_="list"): + for row in table.find_all("tr"): + columns = row.find_all("td") + if len(columns) != 4: + continue - version_str = columns[0].text.strip() - version_match = config.first_match(version_str) - if not version_match: - continue + version_str = columns[0].text.strip() + version_match = config.first_match(version_str) + if not version_match: + continue - datetime_td = columns[3].find_next("span") - datetime_str = datetime_td.attrs["title"] if datetime_td else None - if not datetime_str: - continue + datetime_td = columns[3].find_next("span") + datetime_str = datetime_td.attrs["title"] if datetime_td else None + if not datetime_str: + continue - version = config.render(version_match) - date = dates.parse_datetime(datetime_str) - product_data.declare_version(version, date) + version = config.render(version_match) + date = dates.parse_datetime(datetime_str) + product_data.declare_version(version, date) diff --git a/src/common/endoflife.py b/src/common/endoflife.py index 32d3b748..0df7dcd1 100644 --- a/src/common/endoflife.py +++ b/src/common/endoflife.py @@ -1,3 +1,4 @@ +import itertools import logging import os import re @@ -17,11 +18,14 @@ PRODUCTS_PATH = Path(os.environ.get("PRODUCTS_PATH", "website/products")) class AutoConfig: - def __init__(self, method: str, config: dict) -> None: - self.method = method - self.url = config[method] + def __init__(self, product: str, config: dict) -> None: + self.product = product + self.method = next(key for key in config if key not in ("template", "regex", "regex_exclude")) + self.url = config[self.method] self.version_template = Template(config.get("template", DEFAULT_VERSION_TEMPLATE)) + self.script = f"{self.url}.py" if self.method == "custom" else f"{self.method}.py" + regexes_include = config.get("regex", DEFAULT_VERSION_REGEX) regexes_include = regexes_include if isinstance(regexes_include, list) else [regexes_include] self.include_version_patterns = [re.compile(r) for r in regexes_include] @@ -45,6 +49,9 @@ class AutoConfig: def render(self, match: re.Match) -> str: return self.version_template.render(**match.groupdict()) + def __repr__(self) -> str: + return f"{self.product}#{self.method}({self.url})" + class ProductFrontmatter: def __init__(self, name: str) -> None: @@ -59,17 +66,23 @@ class ProductFrontmatter: else: logging.warning(f"no product data found for {self.name} at {self.path}") - def get_auto_configs(self, method: str) -> list[AutoConfig]: + def has_auto_configs(self) -> bool: + return self.data and "methods" in self.data.get("auto", {}) + + def is_auto_update_cumulative(self) -> bool: + return self.data.get("auto", {}).get("cumulative", False) + + def auto_configs(self, method_filter: str = None, url_filter: str = None) -> list[AutoConfig]: configs = [] - all_configs = self.data.get("auto", {}).get("methods", []) - for config in all_configs: - if method in config: - configs.append(AutoConfig(method, config)) + configs_data = self.data.get("auto", {}).get("methods", []) + for config_data in configs_data: + config = AutoConfig(self.name, config_data) + if ((method_filter and config.method != method_filter) + or (url_filter and config.url != url_filter)): + continue - if len(configs) > 0 and len(configs) != len(all_configs): - message = f"mixed auto-update methods declared for {self.name}, this is not yet supported" - raise ValueError(message) + configs.append(config) return configs @@ -80,7 +93,7 @@ class ProductFrontmatter: return None -def list_products(method: str, products_filter: str = None) -> list[ProductFrontmatter]: +def list_products(products_filter: str = None) -> list[ProductFrontmatter]: """Return a list of products that are using the same given update method.""" products = [] @@ -89,9 +102,12 @@ def list_products(method: str, products_filter: str = None) -> list[ProductFront if products_filter and product_name != products_filter: continue - product = ProductFrontmatter(product_name) - configs = product.get_auto_configs(method) - if len(configs) > 0: - products.append(product) + products.append(ProductFrontmatter(product_name)) return products + + +def list_configs(products_filter: str = None, methods_filter: str = None, urls_filter: str = None) -> list[AutoConfig]: + products = list_products(products_filter) + configs_by_product = [p.auto_configs(methods_filter, urls_filter) for p in products] + return list(itertools.chain.from_iterable(configs_by_product)) # flatten the list of lists diff --git a/src/common/gha.py b/src/common/gha.py index 42a6b90c..fd8b0ad2 100644 --- a/src/common/gha.py +++ b/src/common/gha.py @@ -45,3 +45,14 @@ class GitHubStepSummary: if var_exists: with open(os.environ["GITHUB_STEP_SUMMARY"], 'a') as github_step_summary: # NOQA: PTH123 print(self.value, file=github_step_summary) + + +class GitHubGroup: + def __init__(self, name: str) -> None: + self.name = name + + def __enter__(self) -> None: + logging.info(f"::group::{self.name}") + + def __exit__(self, exc_type: any, exc_value: any, traceback: any) -> None: + logging.info("::endgroup::") diff --git a/src/common/releasedata.py b/src/common/releasedata.py index 993aecbf..7b6e17f0 100644 --- a/src/common/releasedata.py +++ b/src/common/releasedata.py @@ -37,71 +37,53 @@ class ProductVersion: def replace_date(self, date: datetime) -> None: self.data["date"] = date.strftime("%Y-%m-%d") - def copy(self) -> "ProductVersion": - return ProductVersion(self.product, self.data.copy()) - def __repr__(self) -> str: return f"{self.product}#{self.name()} ({self.date()})" class ProductData: - def __init__(self, name: str, cumulative_update: bool = False) -> None: + def __init__(self, name: str) -> None: self.name: str = name - self.cumulative_update: bool = cumulative_update self.versions_path: Path = VERSIONS_PATH / f"{name}.json" self.versions: dict[str, ProductVersion] = {} - self.previous_versions: dict[str, ProductVersion] = {} def __enter__(self) -> "ProductData": - logging.info(f"::group::{self}") - if self.versions_path.is_file(): with self.versions_path.open() as f: for json_version in json.load(f)["versions"].values(): version = ProductVersion(self.name, json_version) - self.previous_versions[version.name()] = version - logging.info(f"loaded previous versions data for {self} from {self.versions_path}") + self.versions[version.name()] = version + logging.info(f"loaded versions data for {self} from {self.versions_path}") else: - logging.info(f"no previous versions data found for {self} at {self.versions_path}") - - if self.cumulative_update: - logging.info(f"cumulative update is enabled for {self}, will reuse previous versions data") - for name, version in self.previous_versions.items(): - self.versions[name] = version.copy() + logging.info(f"no versions data found for {self} at {self.versions_path}") return self def __exit__(self, exc_type: Optional[Type[BaseException]], exc_value: Optional[BaseException], exc_traceback: Optional[TracebackType]) -> None: - try: - if exc_value: - message = f"an unexpected error occurred while updating {self} data" - logging.error(message, exc_info=exc_value) - raise ProductUpdateError(message) from exc_value + if exc_value: + message = f"an unexpected error occurred while updating {self} data" + logging.error(message, exc_info=exc_value) + raise ProductUpdateError(message) from exc_value - logging.info("updating %s data",self) - # sort by date then version (desc) - ordered_versions = sorted(self.versions.values(), key=lambda v: (v.date(), v.name()), reverse=True) - with self.versions_path.open("w") as f: - f.write(json.dumps({ - "versions": {version.name(): version.data for version in ordered_versions}, - }, indent=2)) - finally: - logging.info("::endgroup::") + logging.info("updating %s data",self.versions_path) + # sort by date then version (desc) + ordered_versions = sorted(self.versions.values(), key=lambda v: (v.date(), v.name()), reverse=True) + with self.versions_path.open("w") as f: + f.write(json.dumps({ + "versions": {version.name(): version.data for version in ordered_versions}, + }, indent=2)) def get_version(self, version: str) -> ProductVersion: return self.versions[version] if version in self.versions else None - def get_previous_version(self, version: str) -> ProductVersion: - return self.previous_versions[version] if version in self.previous_versions else None - def declare_version(self, version: str, date: datetime) -> None: if version in self.versions and self.versions[version].date() != date: logging.info(f"overwriting {version} ({self.get_version(version).date()} -> {date}) for {self}") self.versions[version].replace_date(date) else: logging.info(f"adding version {version} ({date}) to {self}") - self.versions[version] = ProductVersion.of(self, version, date) + self.versions[version] = ProductVersion.of(self.name, version, date) def declare_versions(self, dates_by_version: dict[str, datetime]) -> None: for (version, date) in dates_by_version.items(): diff --git a/src/distrowatch.py b/src/distrowatch.py index 50f64095..300bf371 100644 --- a/src/distrowatch.py +++ b/src/distrowatch.py @@ -6,21 +6,21 @@ from common import dates, endoflife, http, releasedata METHOD = 'distrowatch' p_filter = sys.argv[1] if len(sys.argv) > 1 else None -for product in endoflife.list_products(METHOD, p_filter): - with releasedata.ProductData(product.name) as product_data: - for config in product.get_auto_configs(METHOD): - response = http.fetch_url(f"https://distrowatch.com/index.php?distribution={config.url}") - soup = BeautifulSoup(response.text, features="html5lib") +m_filter = sys.argv[2] if len(sys.argv) > 2 else None +for config in endoflife.list_configs(p_filter, METHOD, m_filter): + with releasedata.ProductData(config.product) as product_data: + response = http.fetch_url(f"https://distrowatch.com/index.php?distribution={config.url}") + soup = BeautifulSoup(response.text, features="html5lib") - for table in soup.select("td.News1>table.News"): - headline = table.select_one("td.NewsHeadline a[href]").get_text().strip() - versions_match = config.first_match(headline) - if not versions_match: - continue + for table in soup.select("td.News1>table.News"): + headline = table.select_one("td.NewsHeadline a[href]").get_text().strip() + versions_match = config.first_match(headline) + if not versions_match: + continue - # multiple versions may be released at once (e.g. Ubuntu 16.04.7 and 18.04.5) - versions = config.render(versions_match).split("\n") - date = dates.parse_date(table.select_one("td.NewsDate").get_text()) + # multiple versions may be released at once (e.g. Ubuntu 16.04.7 and 18.04.5) + versions = config.render(versions_match).split("\n") + date = dates.parse_date(table.select_one("td.NewsDate").get_text()) - for version in versions: - product_data.declare_version(version, date) + for version in versions: + product_data.declare_version(version, date) diff --git a/src/docker_hub.py b/src/docker_hub.py index 3ca20554..86db332c 100644 --- a/src/docker_hub.py +++ b/src/docker_hub.py @@ -23,7 +23,7 @@ def fetch_releases(p: releasedata.ProductData, c: endoflife.AutoConfig, url: str p_filter = sys.argv[1] if len(sys.argv) > 1 else None -for product in endoflife.list_products(METHOD, p_filter): - with releasedata.ProductData(product.name) as product_data: - for config in product.get_auto_configs(METHOD): - fetch_releases(product_data, config, f"https://hub.docker.com/v2/repositories/{config.url}/tags?page_size=100&page=1") +m_filter = sys.argv[2] if len(sys.argv) > 2 else None +for config in endoflife.list_configs(p_filter, METHOD, m_filter): + with releasedata.ProductData(config.product) as product_data: + fetch_releases(product_data, config, f"https://hub.docker.com/v2/repositories/{config.url}/tags?page_size=100&page=1") diff --git a/src/firefox.py b/src/firefox.py index 5a338f6c..3ee3280c 100644 --- a/src/firefox.py +++ b/src/firefox.py @@ -20,7 +20,7 @@ The script will need to be updated if someday those conditions are not met.""" MAX_VERSIONS_LIMIT = 50 -with releasedata.ProductData("firefox", cumulative_update=True) as product_data: +with releasedata.ProductData("firefox") as product_data: releases_page = http.fetch_url("https://www.mozilla.org/en-US/firefox/releases/") releases_soup = BeautifulSoup(releases_page.text, features="html5lib") releases_list = releases_soup.find_all("ol", class_="c-release-list") diff --git a/src/git.py b/src/git.py index 915b804c..c3253d9e 100644 --- a/src/git.py +++ b/src/git.py @@ -8,16 +8,16 @@ from common.git import Git METHOD = 'git' p_filter = sys.argv[1] if len(sys.argv) > 1 else None -for product in endoflife.list_products(METHOD, p_filter): - with releasedata.ProductData(product.name) as product_data: - for config in product.get_auto_configs(METHOD): - git = Git(config.url) - git.setup(bare=True) +m_filter = sys.argv[2] if len(sys.argv) > 2 else None +for config in endoflife.list_configs(p_filter, METHOD, m_filter): + with releasedata.ProductData(config.product) as product_data: + git = Git(config.url) + git.setup(bare=True) - tags = git.list_tags() - for tag, date_str in tags: - version_match = config.first_match(tag) - if version_match: - version = config.render(version_match) - date = dates.parse_date(date_str) - product_data.declare_version(version, date) + tags = git.list_tags() + for tag, date_str in tags: + version_match = config.first_match(tag) + if version_match: + version = config.render(version_match) + date = dates.parse_date(date_str) + product_data.declare_version(version, date) diff --git a/src/github-releases.py b/src/github_releases.py similarity index 61% rename from src/github-releases.py rename to src/github_releases.py index ea4e8fc9..e3b0e0fc 100644 --- a/src/github-releases.py +++ b/src/github_releases.py @@ -43,17 +43,17 @@ query($endCursor: String) { p_filter = sys.argv[1] if len(sys.argv) > 1 else None -for product in endoflife.list_products(METHOD, p_filter): - with releasedata.ProductData(product.name) as product_data: - for config in product.get_auto_configs(METHOD): - for page in fetch_releases(config.url): - releases = [edge['node'] for edge in (page['data']['repository']['releases']['edges'])] +m_filter = sys.argv[2] if len(sys.argv) > 2 else None +for config in endoflife.list_configs(p_filter, METHOD, m_filter): + with releasedata.ProductData(config.product) as product_data: + for page in fetch_releases(config.url): + releases = [edge['node'] for edge in (page['data']['repository']['releases']['edges'])] - for release in releases: - if not release['isPrerelease']: - version_str = release['name'] - version_match = config.first_match(version_str) - if version_match: - version = config.render(version_match) - date = dates.parse_datetime(release['publishedAt']) - product_data.declare_version(version, date) + for release in releases: + if not release['isPrerelease']: + version_str = release['name'] + version_match = config.first_match(version_str) + if version_match: + version = config.render(version_match) + date = dates.parse_datetime(release['publishedAt']) + product_data.declare_version(version, date) diff --git a/src/maven.py b/src/maven.py index 28d7b2e4..ef5ce614 100644 --- a/src/maven.py +++ b/src/maven.py @@ -6,23 +6,23 @@ from common import endoflife, http, releasedata METHOD = "maven" p_filter = sys.argv[1] if len(sys.argv) > 1 else None -for product in endoflife.list_products(METHOD, p_filter): - with releasedata.ProductData(product.name) as product_data: - for config in product.get_auto_configs(METHOD): - start = 0 - group_id, artifact_id = config.url.split("/") +m_filter = sys.argv[2] if len(sys.argv) > 2 else None +for config in endoflife.list_configs(p_filter, METHOD, m_filter): + with releasedata.ProductData(config.product) as product_data: + start = 0 + group_id, artifact_id = config.url.split("/") - while True: - url = f"https://search.maven.org/solrsearch/select?q=g:{group_id}+AND+a:{artifact_id}&core=gav&wt=json&start={start}&rows=100" - data = http.fetch_url(url).json() + while True: + url = f"https://search.maven.org/solrsearch/select?q=g:{group_id}+AND+a:{artifact_id}&core=gav&wt=json&start={start}&rows=100" + data = http.fetch_url(url).json() - for row in data["response"]["docs"]: - version_match = config.first_match(row["v"]) - if version_match: - version = config.render(version_match) - date = datetime.fromtimestamp(row["timestamp"] / 1000, tz=timezone.utc) - product_data.declare_version(version, date) + for row in data["response"]["docs"]: + version_match = config.first_match(row["v"]) + if version_match: + version = config.render(version_match) + date = datetime.fromtimestamp(row["timestamp"] / 1000, tz=timezone.utc) + product_data.declare_version(version, date) - start += 100 - if data["response"]["numFound"] <= start: - break + start += 100 + if data["response"]["numFound"] <= start: + break diff --git a/src/npm.py b/src/npm.py index f1cce2a7..377051da 100644 --- a/src/npm.py +++ b/src/npm.py @@ -5,13 +5,13 @@ from common import dates, endoflife, http, releasedata METHOD = "npm" p_filter = sys.argv[1] if len(sys.argv) > 1 else None -for product in endoflife.list_products(METHOD, p_filter): - with releasedata.ProductData(product.name) as product_data: - for config in product.get_auto_configs(METHOD): - data = http.fetch_url(f"https://registry.npmjs.org/{config.url}").json() - for version_str in data["versions"]: - version_match = config.first_match(version_str) - if version_match: - version = config.render(version_match) - date = dates.parse_datetime(data["time"][version_str]) - product_data.declare_version(version, date) +m_filter = sys.argv[2] if len(sys.argv) > 2 else None +for config in endoflife.list_configs(p_filter, METHOD, m_filter): + with releasedata.ProductData(config.product) as product_data: + data = http.fetch_url(f"https://registry.npmjs.org/{config.url}").json() + for version_str in data["versions"]: + version_match = config.first_match(version_str) + if version_match: + version = config.render(version_match) + date = dates.parse_datetime(data["time"][version_str]) + product_data.declare_version(version, date) diff --git a/src/palo-alto-networks.py b/src/palo-alto-networks.py index f7929785..8207892b 100644 --- a/src/palo-alto-networks.py +++ b/src/palo-alto-networks.py @@ -1,4 +1,3 @@ -import logging import re from bs4 import BeautifulSoup @@ -11,11 +10,8 @@ IDENTIFIERS_BY_PRODUCT = { } # all products are on the same page, it's faster to fetch it only once -logging.info("::group::palo-alto-networks") response = http.fetch_url("https://www.paloaltonetworks.com/services/support/end-of-life-announcements/end-of-life-summary") soup = BeautifulSoup(response.text, features="html5lib") -logging.info("::endgroup::") - for product_name, identifier in IDENTIFIERS_BY_PRODUCT.items(): with releasedata.ProductData(product_name) as product_data: table = soup.find(id=identifier) diff --git a/src/pypi.py b/src/pypi.py index d78c229f..6ce57111 100644 --- a/src/pypi.py +++ b/src/pypi.py @@ -5,16 +5,16 @@ from common import dates, endoflife, http, releasedata METHOD = "pypi" p_filter = sys.argv[1] if len(sys.argv) > 1 else None -for product in endoflife.list_products(METHOD, p_filter): - with releasedata.ProductData(product.name) as product_data: - for config in product.get_auto_configs(METHOD): - data = http.fetch_url(f"https://pypi.org/pypi/{config.url}/json").json() +m_filter = sys.argv[2] if len(sys.argv) > 2 else None +for config in endoflife.list_configs(p_filter, METHOD, m_filter): + with releasedata.ProductData(config.product) as product_data: + data = http.fetch_url(f"https://pypi.org/pypi/{config.url}/json").json() - for version_str in data["releases"]: - version_match = config.first_match(version_str) - version_data = data["releases"][version_str] + for version_str in data["releases"]: + version_match = config.first_match(version_str) + version_data = data["releases"][version_str] - if version_match and version_data: - version = config.render(version_match) - date = dates.parse_datetime(version_data[0]["upload_time_iso_8601"]) - product_data.declare_version(version, date) + if version_match and version_data: + version = config.render(version_match) + date = dates.parse_datetime(version_data[0]["upload_time_iso_8601"]) + product_data.declare_version(version, date) diff --git a/src/unity.py b/src/unity.py index 2aaf7e38..2980c186 100644 --- a/src/unity.py +++ b/src/unity.py @@ -17,7 +17,7 @@ Note that it was assumed that: The script will need to be updated if someday those conditions are not met.""" -with releasedata.ProductData("unity", cumulative_update=True) as product_data: +with releasedata.ProductData("unity") as product_data: response = http.fetch_url("https://unity.com/releases/editor/qa/lts-releases") soup = BeautifulSoup(response.text, features="html5lib") diff --git a/update.py b/update.py index 1989692d..2599e389 100644 --- a/update.py +++ b/update.py @@ -1,6 +1,5 @@ import json import logging -import os import subprocess import sys import time @@ -8,45 +7,106 @@ from pathlib import Path from deepdiff import DeepDiff -from src.common.gha import GitHubOutput, GitHubStepSummary +from src.common.endoflife import AutoConfig, ProductFrontmatter, list_products +from src.common.gha import GitHubGroup, GitHubOutput, GitHubStepSummary SRC_DIR = Path('src') DATA_DIR = Path('releases') -class ScriptResult: - def __init__(self, path: Path, duration: float, success: bool) -> None: - self.name = path.stem - self.duration = duration - self.success = success +class ScriptExecutionSummary: + def __init__(self) -> None: + self.success_by_product = {} + self.success_by_script = {} + self.durations_by_product = {} + self.durations_by_script = {} + self.scripts_by_product = {} + self.products_by_script = {} - def __lt__(self, other: "ScriptResult") -> bool: - return self.duration < other.duration + def register(self, script: str, product: str, duration: float, success: bool) -> None: + self.success_by_product[product] = self.success_by_product.get(product, True) and success + self.success_by_script[script] = self.success_by_script.get(script, True) and success + self.durations_by_product[product] = self.durations_by_product.get(product, 0) + duration + self.durations_by_script[script] = self.durations_by_script.get(script, 0) + duration + self.scripts_by_product[product] = self.scripts_by_product.get(product, []) + [script] + self.products_by_script[script] = self.products_by_script.get(script, []) + [product] + + def print_summary(self, summary: GitHubStepSummary, min_duration: float = 3) -> None: + summary.println("## Script execution summary\n") + summary.println(f"\nExecutions below {min_duration} seconds are hidden except in case of failure.\n") + summary.println("### By products\n") + summary.println("| Name | Duration | Scripts | Succeeded |") + summary.println("|------|----------|---------|-----------|") + for product, duration in sorted(self.durations_by_product.items(), key=lambda x: x[1], reverse=True): + if duration >= min_duration or not self.success_by_product[product]: + scripts = ', '.join(self.scripts_by_product[product]) + success = '✅' if self.success_by_product[product] else '❌' + summary.println(f"| {product} | {duration:.2f}s | {scripts} | {success} |") + + summary.println("\n### By scripts\n") + summary.println("| Name | Duration | #Products | Succeeded |") + summary.println("|------|----------|-----------|-----------|") + for script, duration in sorted(self.durations_by_script.items(), key=lambda x: x[1], reverse=True): + if duration >= min_duration or not self.products_by_script[script]: + product_count = len(self.products_by_script[script]) + success = '✅' if self.success_by_script[script] else '❌' + summary.println(f"| {script} | {duration:.2f}s | {product_count} | {success} |") + + summary.println("") + + def any_failure(self) -> bool: + return not all(self.success_by_product.values()) -def run_scripts(summary: GitHubStepSummary) -> bool: - results = [] +def __delete_data(product: ProductFrontmatter) -> None: + release_data_path = DATA_DIR / f"{product.name}.json" + if not release_data_path.exists() or product.is_auto_update_cumulative(): + return - for script in sorted([SRC_DIR / file for file in os.listdir(SRC_DIR) if file.endswith('.py')]): - logging.info(f"start running {script}") + release_data_path.unlink() + logging.info(f"deleted {release_data_path} before running scripts") - start = time.perf_counter() - child = subprocess.run([sys.executable, script]) # timeout handled in subscripts - elapsed_seconds = time.perf_counter() - start - result = ScriptResult(script, elapsed_seconds, child.returncode == 0) - log_level = logging.ERROR if not result.success else logging.INFO - logging.log(log_level, f"ran {script}, took {elapsed_seconds:.2f}s (success={result.success})") - results.append(result) +def __revert_data(product: ProductFrontmatter) -> None: + release_data_path = DATA_DIR / f"{product.name}.json" + subprocess.run(f'git checkout HEAD -- {release_data_path}', timeout=10, check=True, shell=True) + logging.warning(f"reverted changes in {release_data_path}") - summary.println("## Script execution summary\n") - summary.println("| Name | Duration | Succeeded |") - summary.println("|------|----------|-----------|") - for result in sorted(results, reverse=True): - summary.println(f"| {result.name} | {result.duration:.2f}s | {'✅' if result.success else '❌'} |") - summary.println("") - return not all(result.success for result in results) +def __run_script(product: ProductFrontmatter, config: AutoConfig, summary: ScriptExecutionSummary) -> bool: + script = SRC_DIR / config.script + + logging.info(f"start running {script} for {config}") + start = time.perf_counter() + # timeout is handled in child scripts + child = subprocess.run([sys.executable, script, config.product, str(config.url)]) + success = child.returncode == 0 + elapsed_seconds = time.perf_counter() - start + + summary.register(script.stem, product.name, elapsed_seconds, success) + logging.log(logging.ERROR if not success else logging.INFO, + f"ran {script} for {config}, took {elapsed_seconds:.2f}s (success={success})") + + return success + + +def run_scripts(summary: GitHubStepSummary, product_filter: str) -> bool: + exec_summary = ScriptExecutionSummary() + + for product in list_products(product_filter): + if not product.has_auto_configs(): + continue + + with GitHubGroup(product.name): + __delete_data(product) + for config in product.auto_configs(): + success = __run_script(product, config, exec_summary) + if not success: + __revert_data(product) + break # stop running scripts for this product + + exec_summary.print_summary(summary) + return exec_summary.any_failure() def get_updated_products() -> list[Path]: @@ -92,9 +152,10 @@ def generate_commit_message(old_content: dict[Path, dict], new_content: dict[Pat summary.println("") -logging.basicConfig(format=logging.BASIC_FORMAT, level=logging.INFO) +logging.basicConfig(format="%(message)s", level=logging.INFO) +p_filter = sys.argv[1] if len(sys.argv) > 1 else None with GitHubStepSummary() as step_summary: - some_script_failed = run_scripts(step_summary) + some_script_failed = run_scripts(step_summary, p_filter) updated_products = get_updated_products() step_summary.println("## Update summary\n")