From 1add3a71d14087d2c36c72d04bea2152bb384eb6 Mon Sep 17 00:00:00 2001 From: Marc Wrobel Date: Sun, 24 Dec 2023 17:48:40 +0100 Subject: [PATCH] Minor refactoring (#262) - Move frontmatter-related operation from Product to ProductFrontmatter. This makes more senses, as we are manipulating different files / kind of data. - Use Product directly to load old versions. --- src/aws-lambda.py | 10 +++-- src/cgit.py | 7 ++-- src/common/endoflife.py | 82 +++++++++++++++++++---------------------- src/distrowatch.py | 7 ++-- src/docker_hub.py | 7 ++-- src/git.py | 8 ++-- src/github-releases.py | 7 ++-- src/maven.py | 7 ++-- src/npm.py | 8 ++-- src/pypi.py | 7 ++-- 10 files changed, 78 insertions(+), 72 deletions(-) diff --git a/src/aws-lambda.py b/src/aws-lambda.py index 77e367c7..27fc2094 100644 --- a/src/aws-lambda.py +++ b/src/aws-lambda.py @@ -13,8 +13,10 @@ If one day release dates are available in the AWS documentation, it would be bet them though. Note that this would also be unnecessary if it was possible to disable release/latest release dates updates in the latest.py script.""" -print("::group::aws-lambda") -product = endoflife.Product("aws-lambda", load_product_data=True, load_versions_data=True) +product = endoflife.Product("aws-lambda") +print(f"::group::{product.name}") +old_product = endoflife.Product.from_file(product.name) +product_frontmatter = endoflife.ProductFrontmatter(product.name) response = http.fetch_url("https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html") soup = BeautifulSoup(response.text, features="html5lib") @@ -28,9 +30,9 @@ for table in soup.find_all("table"): cells = row.find_all("td") identifier = cells[identifier_index].get_text().strip() - date = product.get_release_date(identifier) # use the product releaseDate if available + date = product_frontmatter.get_release_date(identifier) # use the product releaseDate if available if date is None: - date = product.get_old_version_date(identifier) # else use the previously found date + date = old_product.get_version_date(identifier) # else use the previously found date if date is None: date = datetime.date.today() # else use today's date diff --git a/src/cgit.py b/src/cgit.py index 5439853b..5689ece1 100644 --- a/src/cgit.py +++ b/src/cgit.py @@ -11,10 +11,11 @@ METHOD = "cgit" p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product_name, configs in endoflife.list_products(METHOD, p_filter).items(): - print(f"::group::{product_name}") - product = endoflife.Product(product_name, load_product_data=True) + product = endoflife.Product(product_name) + print(f"::group::{product.name}") - for auto_config in product.get_auto_configs(METHOD): + product_frontmatter = endoflife.ProductFrontmatter(product.name) + for auto_config in product_frontmatter.get_auto_configs(METHOD): response = http.fetch_url(auto_config.url + '/refs/tags') soup = BeautifulSoup(response.text, features="html5lib") diff --git a/src/common/endoflife.py b/src/common/endoflife.py index cdbb6ff0..3ce36f54 100644 --- a/src/common/endoflife.py +++ b/src/common/endoflife.py @@ -38,38 +38,24 @@ class AutoConfig: return self.version_template.render(**match.groupdict()) -class Product: - """Model an endoflife.date product.""" - - def __init__(self, name: str, load_product_data: bool = False, load_versions_data: bool = False): +class ProductFrontmatter: + def __init__(self, name: str): self.name: str = name - self.versions = {} - self.versions_path: str = f"{VERSIONS_PATH}/{name}.json" - self.product_path: str = f"{PRODUCTS_PATH}/{name}.md" + self.path: str = f"{PRODUCTS_PATH}/{name}.md" - if load_product_data: - if os.path.isfile(self.product_path): - with open(self.product_path) as f: - self.product_data = frontmatter.load(f) - logging.info(f"loaded product data for {self.name} from {self.product_path}") - else: - logging.warning(f"no product data found for {self.name} at {self.product_path}") - self.product_data = None - - if load_versions_data: - if os.path.isfile(self.versions_path): - with open(self.versions_path) as f: - logging.info(f"loaded versions data for {self.name} from {self.versions_path}") - self.old_versions = json.load(f) - else: - logging.warning(f"no versions data found for {self.name} at {self.versions_path}") - self.old_versions = None + self.data = None + if os.path.isfile(self.path): + with open(self.path) as f: + self.data = frontmatter.load(f) + logging.info(f"loaded product data for {self.name} from {self.path}") + else: + logging.warning(f"no product data found for {self.name} at {self.path}") def get_auto_configs(self, method: str) -> list[AutoConfig]: configs = [] - if "auto" in self.product_data: - for config in self.product_data["auto"]: + if "auto" in self.data: + for config in self.data["auto"]: if method in config.keys(): configs.append(AutoConfig(method, config)) else: @@ -77,23 +63,38 @@ class Product: return configs + def get_release_date(self, release_cycle: str) -> datetime: + for release in self.data["releases"]: + if release["releaseCycle"] == release_cycle: + return release["releaseDate"] + + +class Product: + def __init__(self, name: str): + self.name: str = name + self.versions_path: str = f"{VERSIONS_PATH}/{name}.json" + self.versions = {} + + @staticmethod + def from_file(name: str): + product = Product(name) + + if not os.path.isfile(product.versions_path): + with open(product.versions_path) as f: + for version, date in json.load(f).items(): + product.versions[version] = datetime.strptime(date, "%Y-%m-%d") + logging.info(f"loaded versions data for {product.name} from {product.versions_path}") + else: + logging.warning(f"no versions data found for {product.name} at {product.versions_path}") + + return product + def has_version(self, version: str) -> bool: return version in self.versions def get_version_date(self, version: str) -> datetime: return self.versions[version] if version in self.versions else None - def get_old_version_date(self, version: str) -> datetime: - return datetime.strptime(self.old_versions[version], "%Y-%m-%d") if ( - self.old_versions - and version in self.old_versions - ) else None - - def get_release_date(self, release_cycle: str) -> datetime: - for release in self.product_data["releases"]: - if release["releaseCycle"] == release_cycle: - return release["releaseDate"] - def declare_version(self, version: str, date: datetime) -> None: if version in self.versions: if self.versions[version] != date: @@ -134,13 +135,6 @@ class Product: return f"<{self.name}>" -def load_product(product_name) -> frontmatter.Post: - """Load the product's file frontmatter. - """ - with open(f"{PRODUCTS_PATH}/{product_name}.md") as f: - return frontmatter.load(f) - - def list_products(method, products_filter=None) -> dict[str, list[dict]]: """Return a list of products that are using the same given update method. """ diff --git a/src/distrowatch.py b/src/distrowatch.py index d34434a9..319f44bb 100644 --- a/src/distrowatch.py +++ b/src/distrowatch.py @@ -8,10 +8,11 @@ METHOD = 'distrowatch' p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product_name, configs in endoflife.list_products(METHOD, p_filter).items(): - print(f"::group::{product_name}") - product = endoflife.Product(product_name, load_product_data=True) + product = endoflife.Product(product_name) + print(f"::group::{product.name}") - for config in product.get_auto_configs(METHOD): + product_frontmatter = endoflife.ProductFrontmatter(product.name) + for config in product_frontmatter.get_auto_configs(METHOD): response = http.fetch_url(f"https://distrowatch.com/index.php?distribution={config.url}") soup = BeautifulSoup(response.text, features="html5lib") diff --git a/src/docker_hub.py b/src/docker_hub.py index c4345103..44b184c7 100644 --- a/src/docker_hub.py +++ b/src/docker_hub.py @@ -25,10 +25,11 @@ def fetch_releases(product, config, url): p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product_name, configs in endoflife.list_products(METHOD, p_filter).items(): - print(f"::group::{product_name}") - product = endoflife.Product(product_name, load_product_data=True) + product = endoflife.Product(product_name) + print(f"::group::{product.name}") - for config in product.get_auto_configs(METHOD): + product_frontmatter = endoflife.ProductFrontmatter(product.name) + for config in product_frontmatter.get_auto_configs(METHOD): url = f"https://hub.docker.com/v2/repositories/{config.url}/tags?page_size=100&page=1" fetch_releases(product, config, url) diff --git a/src/git.py b/src/git.py index 345d18bc..c164d5ce 100644 --- a/src/git.py +++ b/src/git.py @@ -9,9 +9,11 @@ METHOD = 'git' p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product_name, configs in endoflife.list_products(METHOD, p_filter).items(): - print(f"::group::{product_name}") - product = endoflife.Product(product_name, load_product_data=True) - for config in product.get_auto_configs(METHOD): + product = endoflife.Product(product_name) + print(f"::group::{product.name}") + + product_frontmatter = endoflife.ProductFrontmatter(product.name) + for config in product_frontmatter.get_auto_configs(METHOD): git = Git(config.url) git.setup(bare=True) diff --git a/src/github-releases.py b/src/github-releases.py index 34aa9297..95f4d15b 100644 --- a/src/github-releases.py +++ b/src/github-releases.py @@ -44,10 +44,11 @@ query($endCursor: String) { p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product_name, configs in endoflife.list_products(METHOD, p_filter).items(): - print(f"::group::{product_name}") - product = endoflife.Product(product_name, load_product_data=True) + product = endoflife.Product(product_name) + print(f"::group::{product.name}") - for config in product.get_auto_configs(METHOD): + product_frontmatter = endoflife.ProductFrontmatter(product.name) + for config in product_frontmatter.get_auto_configs(METHOD): for page in fetch_releases(config.url): releases = [edge['node'] for edge in (page['data']['repository']['releases']['edges'])] diff --git a/src/maven.py b/src/maven.py index 97237c1a..f187688e 100644 --- a/src/maven.py +++ b/src/maven.py @@ -7,10 +7,11 @@ METHOD = "maven" p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product_name, configs in endoflife.list_products(METHOD, p_filter).items(): - print(f"::group::{product_name}") - product = endoflife.Product(product_name, load_product_data=True) + product = endoflife.Product(product_name) + print(f"::group::{product.name}") - for config in product.get_auto_configs(METHOD): + product_frontmatter = endoflife.ProductFrontmatter(product.name) + for config in product_frontmatter.get_auto_configs(METHOD): start = 0 group_id, artifact_id = config.url.split("/") diff --git a/src/npm.py b/src/npm.py index dc835b75..4a1e9f2a 100644 --- a/src/npm.py +++ b/src/npm.py @@ -7,9 +7,11 @@ METHOD = "npm" p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product_name, configs in endoflife.list_products(METHOD, p_filter).items(): - print(f"::group::{product_name}") - product = endoflife.Product(product_name, load_product_data=True) - for config in product.get_auto_configs(METHOD): + product = endoflife.Product(product_name) + print(f"::group::{product.name}") + + product_frontmatter = endoflife.ProductFrontmatter(product.name) + for config in product_frontmatter.get_auto_configs(METHOD): data = http.fetch_url(f"https://registry.npmjs.org/{config.url}").json() for version_str in data["versions"]: version_match = config.first_match(version_str) diff --git a/src/pypi.py b/src/pypi.py index a4ddf543..231d1c48 100644 --- a/src/pypi.py +++ b/src/pypi.py @@ -7,10 +7,11 @@ METHOD = "pypi" p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product_name, configs in endoflife.list_products(METHOD, p_filter).items(): - print(f"::group::{product_name}") - product = endoflife.Product(product_name, load_product_data=True) + product = endoflife.Product(product_name) + print(f"::group::{product.name}") - for config in product.get_auto_configs(METHOD): + product_frontmatter = endoflife.ProductFrontmatter(product.name) + for config in product_frontmatter.get_auto_configs(METHOD): data = http.fetch_url(f"https://pypi.org/pypi/{config.url}/json").json() for version_str in data["releases"]: