Refactor scripts arguments handling (#456)

- remove the use of environment variables to get directory paths,
- make use of arguments / argparse instead of environment variables in `update.py` and `report.py`,
- automatically guess the data directory in `latest.py` based on the script's location,
- propagate log level to auto scripts,
- move `list_configs_from_argv` from `endoflife` module to `releasedata` module,
- use `list_products` in `latest.py` to load the product's frontmatters.

src/amazon-eks.py

@@ -1,11 +1,11 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches EKS versions from AWS docs.
 Now that AWS no longer publishes docs on GitHub, we use the Web Archive to get the older versions."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/amazon-neptune.py

@@ -1,10 +1,10 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches Amazon Neptune versions from its RSS feed on docs.aws.amazon.com."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         rss = http.fetch_xml(config.url)
 

src/apache-http-server.py

@@ -1,10 +1,10 @@
-from common import dates, endoflife, releasedata
+from common import dates, releasedata
 from common.git import Git
 
 """Fetches Apache HTTP Server versions and release date from its git repository
 by looking at the STATUS file of each <major>.<minor>.x branch."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         git = Git(config.url)
         git.setup()

src/apache-subversion.py

@@ -1,8 +1,8 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/apple.py

@@ -2,7 +2,7 @@ import logging
 import re
 
 from bs4 import BeautifulSoup
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches and parses version and release date information from Apple's support website."""
 
@@ -22,7 +22,7 @@ URLS = [
 
 DATE_PATTERN = re.compile(r"\b\d+\s[A-Za-z]+\s\d+\b")
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         # URLs are cached to avoid rate limiting by support.apple.com.
         soups = [BeautifulSoup(response.text, features="html5lib") for response in http.fetch_urls(URLS)]

src/artifactory.py

@@ -1,10 +1,10 @@
 from bs4 import BeautifulSoup
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches Artifactory versions from https://jfrog.com, using requests_html because JavaScript is
 needed to render the page."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         content = http.fetch_javascript_url(config.url, wait_until = 'networkidle')
         soup = BeautifulSoup(content, 'html.parser')

src/atlassian_eol.py

@@ -1,7 +1,7 @@
 import logging
 
 from bs4 import BeautifulSoup
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches EOL dates from Atlassian EOL page.
 
@@ -9,7 +9,7 @@ This script takes a selector argument which is the product title identifier on t
 `AtlassianSupportEndofLifePolicy-JiraSoftware`.
 """
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         content = http.fetch_javascript_url(config.url)
         soup = BeautifulSoup(content, features="html5lib")

src/atlassian_versions.py

@@ -1,5 +1,5 @@
 from bs4 import BeautifulSoup
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches versions from Atlassian download-archives pages.
 
@@ -7,7 +7,7 @@ This script takes a single argument which is the url of the product's download-a
 `https://www.atlassian.com/software/confluence/download-archives`.
 """
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         content = http.fetch_javascript_url(config.url, wait_until='networkidle')
         soup = BeautifulSoup(content, 'html5lib')

src/aws-lambda.py

@@ -1,10 +1,10 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches AWS lambda runtimes with their support / EOL dates from https://docs.aws.amazon.com."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/cgit.py

@@ -1,9 +1,9 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches versions from repositories managed with cgit, such as the Linux kernel repository.
 Ideally we would want to use the git repository directly, but cgit-managed repositories don't support partial clone."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url + '/refs/tags')
 

src/chef-infra.py

@@ -1,4 +1,4 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 from common.git import Git
 
 """Fetch released versions from docs.chef.io and retrieve their date from GitHub.
@@ -7,7 +7,7 @@ docs.chef.io needs to be scraped because not all tagged versions are actually re
 More context on https://github.com/endoflife-date/endoflife.date/pull/4425#discussion_r1447932411.
 """
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
         released_versions = [h2.get('id') for h2 in html.find_all('h2', id=True) if h2.get('id')]

src/chef-inspec.py

@@ -1,4 +1,4 @@
-from common import dates, endoflife, github, http, releasedata
+from common import dates, github, http, releasedata
 
 """Fetch released versions from docs.chef.io and retrieve their date from GitHub.
 docs.chef.io needs to be scraped because not all tagged versions are actually released.
@@ -6,7 +6,7 @@ docs.chef.io needs to be scraped because not all tagged versions are actually re
 More context on https://github.com/endoflife-date/endoflife.date/pull/4425#discussion_r1447932411.
 """
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
         released_versions = [h2.get('id') for h2 in html.find_all('h2', id=True) if h2.get('id')]

src/coldfusion.py

@@ -1,6 +1,6 @@
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches versions from Adobe ColdFusion release notes on helpx.adobe.com.
 
@@ -21,7 +21,7 @@ FIXED_VERSIONS = {
     "2023.0.0": dates.date(2022, 5, 16),  # https://coldfusion.adobe.com/2023/05/coldfusion2023-release/
 }
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/common/endoflife.py

@@ -1,8 +1,5 @@
-import itertools
 import logging
-import os
 import re
-import sys
 from datetime import datetime
 from pathlib import Path
 
@@ -15,8 +12,6 @@ DEFAULT_VERSION_REGEX = r"^v?(?P<major>[1-9]\d*)\.(?P<minor>\d+)(\.(?P<patch>\d+
 DEFAULT_VERSION_PATTERN = re.compile(DEFAULT_VERSION_REGEX)
 DEFAULT_VERSION_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{% if tiny %}.{{tiny}}{% endif %}{% endif %}{% endif %}"
 
-PRODUCTS_PATH = Path(os.environ.get("PRODUCTS_PATH", "website/products"))
-
 
 class AutoConfig:
     def __init__(self, product: str, data: dict) -> None:
@@ -58,9 +53,9 @@ class AutoConfig:
 
 
 class ProductFrontmatter:
-    def __init__(self, name: str) -> None:
-        self.name: str = name
-        self.path: Path = PRODUCTS_PATH / f"{name}.md"
+    def __init__(self, path: Path) -> None:
+        self.path: Path = path
+        self.name: str = path.stem
 
         self.data = None
         if self.path.is_file():
@@ -109,37 +104,19 @@ class ProductFrontmatter:
         return None
 
 
-def list_products(products_filter: str = None) -> list[ProductFrontmatter]:
-    """Return a list of products that are using the same given update method."""
+def list_products(products_dir: Path, product_name: str = None) -> list[ProductFrontmatter]:
+    product_names = [product_name] if product_name else sorted([p.stem for p in products_dir.glob("*.md")])
+
     products = []
-
-    for product_file in sorted(PRODUCTS_PATH.glob("*.md")):
-        product_name = product_file.stem
-        if products_filter and product_name != products_filter:
-            continue
-
+    for product_name in product_names:
         try:
-            products.append(ProductFrontmatter(product_name))
+            products.append(ProductFrontmatter(products_dir / f"{product_name}.md"))
         except Exception as e:
             logging.exception(f"failed to load product data for {product_name}: {e}")
 
     return products
 
 
-def list_configs(products_filter: str = None, methods_filter: str = None, urls_filter: str = None) -> list[AutoConfig]:
-    """Return a list of auto configs, filtering by product name, method, and URL."""
-    products = list_products(products_filter)
-    configs_by_product = [p.auto_configs(methods_filter, urls_filter) for p in products]
-    return list(itertools.chain.from_iterable(configs_by_product))  # flatten the list of lists
-
-
-def list_configs_from_argv() -> list[AutoConfig]:
-    products_filter = sys.argv[1] if len(sys.argv) > 1 else None
-    methods_filter = sys.argv[2] if len(sys.argv) > 1 else None
-    urls_filter = sys.argv[3] if len(sys.argv) > 2 else None
-    return list_configs(products_filter, methods_filter, urls_filter)
-
-
 def to_identifier(s: str) -> str:
     """Convert a string to a valid endoflife.date identifier."""
     identifier = s.strip().lower()

src/common/releasedata.py

@@ -1,15 +1,16 @@
+import argparse
 import json
 import logging
-import os
+import sys
 from datetime import datetime, timezone
 from pathlib import Path
 from types import TracebackType
 from typing import Optional, Type
 
-# Do not update the format: it's also used to declare groups in the GitHub Actions logs.
-logging.basicConfig(format="%(message)s", level=logging.INFO)
+from . import endoflife
 
-VERSIONS_PATH = Path(os.environ.get("VERSIONS_PATH", "releases"))
+SRC_DIR = Path('src')
+DATA_DIR = Path('releases')
 
 
 class ProductUpdateError(Exception):
@@ -108,7 +109,7 @@ class ProductVersion:
 class ProductData:
     def __init__(self, name: str) -> None:
         self.name: str = name
-        self.versions_path: Path = VERSIONS_PATH / f"{name}.json"
+        self.versions_path: Path = DATA_DIR / f"{name}.json"
         self.releases = {}
         self.versions: dict[str, ProductVersion] = {}
         self.updated = False
@@ -190,3 +191,21 @@ class ProductData:
 
     def __repr__(self) -> str:
         return self.name
+
+
+def list_configs_from_argv() -> list[endoflife.AutoConfig]:
+    return parse_argv()[1]
+
+def parse_argv() -> tuple[endoflife.ProductFrontmatter, list[endoflife.AutoConfig]]:
+    parser = argparse.ArgumentParser(description=sys.argv[0])
+    parser.add_argument('-p', '--product', required=True, help='path to the product')
+    parser.add_argument('-m', '--method', required=True, help='method to filter by')
+    parser.add_argument('-u', '--url', required=True, help='url to filter by')
+    parser.add_argument('-v', '--verbose', action='store_true', help='enable verbose logging')
+    args = parser.parse_args()
+
+    # Do not update the format: it's also used to declare groups in the GitHub Actions logs.
+    logging.basicConfig(format="%(message)s", level=(logging.DEBUG if args.verbose else logging.INFO))
+
+    product = endoflife.ProductFrontmatter(Path(args.product))
+    return product, product.auto_configs(args.method, args.url)

src/cos.py

@@ -2,7 +2,7 @@ import datetime
 import re
 
 from bs4 import BeautifulSoup
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 MILESTONE_PATTERN = re.compile(r'COS \d+ LTS')
 VERSION_PATTERN = re.compile(r"^(cos-\d+-\d+-\d+-\d+)")
@@ -14,7 +14,7 @@ def parse_date(date_text: str) -> datetime:
     return dates.parse_date(date_text)
 
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         main = http.fetch_url(config.url)
         main_soup = BeautifulSoup(main.text, features="html5lib")

src/couchbase-server.py

@@ -1,7 +1,7 @@
 import logging
 
 from bs4 import BeautifulSoup
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches versions from release notes of each minor version on docs.couchbase.com.
 
@@ -16,7 +16,7 @@ MANUAL_VERSIONS = {
     "7.2.0": dates.date(2023, 6, 1),  # https://www.couchbase.com/blog/couchbase-capella-spring-release-72/
 }
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(f"{config.url}/current/install/install-intro.html")
 

src/debian.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from subprocess import run
 
-from common import dates, endoflife, releasedata
+from common import dates, releasedata
 from common.git import Git
 
 """Fetch Debian versions by parsing news in www.debian.org source repository."""
@@ -40,7 +40,7 @@ def extract_point_versions(p: releasedata.ProductData, repo_dir: Path) -> None:
         (date, version) = line.split(' ')
         p.declare_version(version, dates.parse_date(date))
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         git = Git(config.url)
         git.setup()

src/distrowatch.py

@@ -1,6 +1,6 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(f"https://distrowatch.com/index.php?distribution={config.url}")
 

src/docker_hub.py

@@ -17,6 +17,6 @@ def fetch_releases(p: releasedata.ProductData, c: endoflife.AutoConfig, url: str
         fetch_releases(p, c, data["next"])
 
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         fetch_releases(product_data, config, f"https://hub.docker.com/v2/repositories/{config.url}/tags?page_size=100&page=1")

src/firefox.py

@@ -1,7 +1,7 @@
 import urllib.parse
 
 from bs4 import BeautifulSoup
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetch Firefox versions with their dates from https://www.mozilla.org/.
 
@@ -20,7 +20,7 @@ The script will need to be updated if someday those conditions are not met."""
 
 MAX_VERSIONS_LIMIT = 100
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         releases_page = http.fetch_url(config.url)
         releases_soup = BeautifulSoup(releases_page.text, features="html5lib")

src/ghc-wiki.py

@@ -14,7 +14,7 @@ References:
 import re
 from typing import Any, Generator, Iterator
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 
 def parse_markdown_tables(lineiter: Iterator[str]) -> Generator[list[list[Any]], Any, None]:
@@ -50,7 +50,7 @@ def maybe_markdown_table_row(line: str) -> list[str] | None:
         return None
     return [x.strip() for x in line.strip('|').split('|')]
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product:
         resp = http.fetch_url(config.url)
         resp.raise_for_status()

src/git.py

@@ -1,9 +1,9 @@
-from common import dates, endoflife, releasedata
+from common import dates, releasedata
 from common.git import Git
 
 """Fetches versions from tags in a git repository. This replace the old update.rb script."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         git = Git(config.url)
         git.setup(bare=True)

src/github_releases.py

@@ -1,11 +1,11 @@
-from common import dates, endoflife, github, releasedata
+from common import dates, github, releasedata
 
 """Fetches versions from GitHub releases using the GraphQL API and the GitHub CLI.
 
 Note: GraphQL API and GitHub CLI are used because it's simpler: no need to manage pagination and authentication.
 """
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         for release in github.fetch_releases(config.url):
             if release.is_prerelease:

src/github_tags.py

@@ -1,11 +1,11 @@
-from common import dates, endoflife, github, releasedata
+from common import dates, github, releasedata
 
 """Fetches versions from GitHub tags using the GraphQL API and the GitHub CLI.
 
 Note: GraphQL API and GitHub CLI are used because it's simpler: no need to manage pagination and authentication.
 """
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         for tag in github.fetch_tags(config.url):
             version_str = tag.name

src/google-kubernetes-engine.py

@@ -1,6 +1,6 @@
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 # https://regex101.com/r/zPxBqT/1
 VERSION_PATTERN = re.compile(r"\d.\d+\.\d+-gke\.\d+")
@@ -11,7 +11,7 @@ URL_BY_PRODUCT = {
     "google-kubernetes-engine-rapid": "https://cloud.google.com/kubernetes-engine/docs/release-notes-rapid",
 }
 
-for config in endoflife.list_configs_from_argv(): # noqa: B007 multiple JSON produced for historical reasons
+for config in releasedata.list_configs_from_argv(): # noqa: B007 multiple JSON produced for historical reasons
     for product_name, url in URL_BY_PRODUCT.items():
         with releasedata.ProductData(product_name) as product_data:
             html = http.fetch_html(url)

src/graalvm.py

@@ -1,8 +1,8 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
         table_selector = config.data.get("table_selector", "#previous-releases + table").strip()

src/haproxy.py

@@ -1,11 +1,11 @@
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 CYCLE_PATTERN = re.compile(r"^(\d+\.\d+)/$")
 DATE_AND_VERSION_PATTERN = re.compile(r"^(\d{4})/(\d{2})/(\d{2})\s+:\s+(\d+\.\d+\.\d.?)$")  # https://regex101.com/r/1JCnFC/1
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         # First, get all minor releases from the download page
         download_html = http.fetch_html(config.url)

src/ibm-aix.py

@@ -1,6 +1,6 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/kuma.py

@@ -1,6 +1,6 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetch version data for Kuma from https://raw.githubusercontent.com/kumahq/kuma/master/versions.yml.
 """
@@ -9,7 +9,7 @@ RELEASE_FIELD = 'release'
 RELEASE_DATE_FIELD = 'releaseDate'
 EOL_FIELD = 'endOfLifeDate'
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         versions_data = http.fetch_yaml(config.url)
 

src/libreoffice.py

@@ -1,10 +1,10 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches LibreOffice versions from https://downloadarchive.documentfoundation.org/libreoffice/old/"""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/looker.py

@@ -1,14 +1,14 @@
 import re
 
 from bs4 import BeautifulSoup
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetch Looker versions from the Google Cloud release notes RSS feed.
 """
 
 ANNOUNCEMENT_PATTERN = re.compile(r"includes\s+the\s+following\s+changes", re.IGNORECASE)
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         rss = http.fetch_xml(config.url)
 

src/lua.py

@@ -1,13 +1,13 @@
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches Lua releases from lua.org."""
 
 RELEASED_AT_PATTERN = re.compile(r"Lua\s*(?P<release>\d+\.\d+)\s*was\s*released\s*on\s*(?P<release_date>\d+\s*\w+\s*\d{4})")
 VERSION_PATTERN = re.compile(r"(?P<version>\d+\.\d+\.\d+),\s*released\s*on\s*(?P<version_date>\d+\s*\w+\s*\d{4})")
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url, features = 'html.parser')
         page_text = html.text # HTML is broken, no way to parse it with beautifulsoup

src/maven.py

@@ -1,8 +1,8 @@
 from datetime import datetime, timezone
 
-from common import endoflife, http, releasedata
+from common import http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         start = 0
         group_id, artifact_id = config.url.split("/")

src/netbsd.py

@@ -1,10 +1,10 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches NetBSD versions and EOL information from https://www.netbsd.org/."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/npm.py

@@ -1,6 +1,6 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         data = http.fetch_json(f"https://registry.npmjs.org/{config.url}")
         for version_str in data["versions"]:

src/nutanix.py

@@ -1,8 +1,8 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetch Nutanix products versions from https://portal.nutanix.com/api/v1."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         data = http.fetch_json(f"https://portal.nutanix.com/api/v1/eol/find?type={config.url}")
 

src/oracle-jdk.py

@@ -1,11 +1,11 @@
 from bs4 import BeautifulSoup
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetch Java versions from https://www.java.com/releases/.
 
 This script is using requests-html because the page needs JavaScript to render correctly."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         content = http.fetch_javascript_url(config.url, wait_until='networkidle')
         soup = BeautifulSoup(content, 'html.parser')

src/pan-os.py

@@ -1,8 +1,8 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches pan-os versions from https://github.com/mrjcap/panos-versions/."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         versions = http.fetch_json(config.url)
 

src/php.py

@@ -1,6 +1,6 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         # Fetch major versions
         latest_by_major = http.fetch_url(config.url).json()

src/plesk.py

@@ -1,11 +1,11 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches versions from Plesk's change log.
 
 Only 18.0.20.3 and later will be picked up, as the format of the change log for 18.0.20 and 18.0.19 are different and
 there is no entry for GA of version 18.0.18 and older."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/pypi.py

@@ -1,6 +1,6 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         data = http.fetch_json(f"https://pypi.org/pypi/{config.url}/json")
 

src/rds.py

@@ -1,6 +1,6 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches Amazon RDS versions from the version management pages on AWS docs.
 
@@ -8,7 +8,7 @@ Pages parsed by this script are expected to have version tables with a version i
 in the third column (usually named 'RDS release date').
 """
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/red-hat-jboss-eap-7.py

@@ -1,10 +1,10 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches RedHat JBoss EAP version data for JBoss 7"""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/red-hat-jboss-eap-8.py

@@ -1,10 +1,10 @@
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches the latest RedHat JBoss EAP version data for JBoss 8.0"""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         xml = http.fetch_xml(config.url)
 

src/red-hat-openshift.py

@@ -1,6 +1,6 @@
 import re
 
-from common import dates, endoflife, releasedata
+from common import dates, releasedata
 from common.git import Git
 
 """Fetches Red Hat OpenShift versions from the documentation's git repository"""
@@ -10,7 +10,7 @@ VERSION_AND_DATE_PATTERN = re.compile(
     re.MULTILINE,
 )
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         git = Git(config.url)
         git.setup()

src/red-hat-satellite.py

@@ -1,12 +1,12 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches Satellite versions from access.redhat.com.
 
 A few of the older versions, such as 'Satellite 6.1 GA Release (Build 6.1.1)', were ignored because too hard to parse."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/redhat_lifecycles.py

@@ -1,7 +1,7 @@
 import logging
 import urllib.parse
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches EOL dates from the Red Hat Product Life Cycle Data API.
 
@@ -17,7 +17,7 @@ class Mapping:
     def get_field_for(self, phase_name: str) -> str | None:
         return self.fields_by_phase.get(phase_name.lower(), None)
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         name = urllib.parse.quote(config.url)
         mapping = Mapping(config.data["fields"])

src/release_table.py

@@ -150,7 +150,7 @@ class Field:
         return f"{self.name}({self.column})"
 
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         render_javascript = config.data.get("render_javascript", False)
         render_javascript_click_selector = config.data.get("render_javascript_click_selector", None)

src/rhel.py

@@ -1,11 +1,11 @@
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 # https://regex101.com/r/877ibq/1
 VERSION_PATTERN = re.compile(r"RHEL (?P<major>\d)(\. ?(?P<minor>\d+))?(( Update (?P<minor2>\d))| GA)?")
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/rocky-linux.py

@@ -1,6 +1,6 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         response = http.fetch_url(config.url)
         for line in response.text.strip().split('\n'):

src/ros.py

@@ -1,8 +1,8 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/samsung-security.py

@@ -12,9 +12,9 @@ it retains the date and use it as the model's EOL date.
 
 TODAY = dates.today()
 
-for config in endoflife.list_configs_from_argv():
+frontmatter, configs = releasedata.parse_argv()
+for config in configs:
     with releasedata.ProductData(config.product) as product_data:
-        frontmatter = endoflife.ProductFrontmatter(product_data.name)
         frontmatter_release_names = frontmatter.get_release_names()
 
         # Copy EOL dates from frontmatter to product data

src/sles.py

@@ -1,8 +1,8 @@
 import logging
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/splunk.py

@@ -1,6 +1,6 @@
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 VERSION_DATE_PATTERN = re.compile(r"Splunk Enterprise (?P<version>\d+\.\d+(?:\.\d+)*) was (?:first )?released on (?P<date>\w+\s\d\d?,\s\d{4})\.", re.MULTILINE)
 
@@ -29,7 +29,7 @@ def get_latest_minor_versions(versions: list[str]) -> list[str]:
     return latest_versions
 
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/typo3.py

@@ -1,6 +1,6 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         data = http.fetch_json(config.url)
         for v in data:

src/unity.py

@@ -1,4 +1,4 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches the Unity LTS releases from the Unity website. Non-LTS releases are not listed there, so this automation
 is only partial.
@@ -16,7 +16,7 @@ Note that it was assumed that:
 
 The script will need to be updated if someday those conditions are not met."""
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/unrealircd.py

@@ -1,10 +1,10 @@
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 DATE_PATTERN = re.compile(r"\d{4}-\d{2}-\d{2}")
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         wikicode = http.fetch_markdown(config.url)
 

src/veeam.py

@@ -1,7 +1,7 @@
 import logging
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches Veeam products versions from https://www.veeam.com.
 
@@ -9,7 +9,7 @@ This script takes a single argument which is the url of the versions page on htt
 such as `https://www.veeam.com/kb2680`.
 """
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/virtualbox.py

@@ -1,13 +1,13 @@
 import logging
 import re
 
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
 """Fetches releases from VirtualBox download page."""
 
 EOL_REGEX = re.compile(r"^\(no longer supported, support ended (?P<value>\d{4}/\d{2})\)$")
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)
 

src/visual-studio.py

@@ -1,6 +1,6 @@
-from common import dates, endoflife, http, releasedata
+from common import dates, http, releasedata
 
-for config in endoflife.list_configs_from_argv():
+for config in releasedata.list_configs_from_argv():
     with releasedata.ProductData(config.product) as product_data:
         html = http.fetch_html(config.url)