github/endoflife-date-release-data
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Simplify endoflife.list_products usage (#292)

Browse Source 
Make endoflife.list_products return product instead of just the product name, to avoid having to reload the product a second time to get more information.
This commit is contained in:
Marc Wrobel
2024-02-03 00:26:23 +01:00
committed by GitHub
parent e4fc52c2a9
commit 8f6efad2cc
9 changed files with 49 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/cgit.py
View File

@@ -9,11 +9,10 @@ Ideally we would want to use the git repository directly, but cgit-managed repos
METHOD = "cgit"
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
for product_name in endoflife.list_products(METHOD, p_filter):
product = releasedata.Product(product_name)
product_frontmatter = endoflife.ProductFrontmatter(product.name)
for auto_config in product_frontmatter.get_auto_configs(METHOD):
response = http.fetch_url(auto_config.url + '/refs/tags')
for product in endoflife.list_products(METHOD, p_filter):
product_data = releasedata.Product(product.name)
for config in product.get_auto_configs(METHOD):
response = http.fetch_url(config.url + '/refs/tags')
soup = BeautifulSoup(response.text, features="html5lib")
for table in soup.find_all("table", class_="list"):
@@ -23,7 +22,7 @@ for product_name in endoflife.list_products(METHOD, p_filter):
continue
version_str = columns[0].text.strip()
version_match = auto_config.first_match(version_str)
version_match = config.first_match(version_str)
if not version_match:
continue
@@ -32,8 +31,8 @@ for product_name in endoflife.list_products(METHOD, p_filter):
if not datetime_str:
continue
version = auto_config.render(version_match)
version = config.render(version_match)
date = dates.parse_datetime(datetime_str)
product.declare_version(version, date)
product_data.declare_version(version, date)
product.write()
product_data.write()

10
src/common/endoflife.py
View File

@@ -68,7 +68,8 @@ class ProductFrontmatter:
configs.append(AutoConfig(method, config))
if len(configs) > 0 and len(configs) != len(self.data["auto"]):
logging.error(f"mixed auto-update methods declared for {self.name}, this is not yet supported")
message = f"mixed auto-update methods declared for {self.name}, this is not yet supported"
raise ValueError(message)
return configs
@@ -79,7 +80,7 @@ class ProductFrontmatter:
return None
def list_products(method: str, products_filter: str = None) -> list[str]:
def list_products(method: str, products_filter: str = None) -> list[ProductFrontmatter]:
"""Return a list of products that are using the same given update method."""
products = []
@@ -89,7 +90,8 @@ def list_products(method: str, products_filter: str = None) -> list[str]:
continue
product = ProductFrontmatter(product_name)
if len(product.get_auto_configs(method)) > 0:
products.append(product_name)
configs = product.get_auto_configs(method)
if len(configs) > 0:
products.append(product)
return products

11
src/distrowatch.py
View File

@@ -6,10 +6,9 @@ from common import dates, endoflife, http, releasedata
METHOD = 'distrowatch'
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
for product_name in endoflife.list_products(METHOD, p_filter):
product = releasedata.Product(product_name)
product_frontmatter = endoflife.ProductFrontmatter(product.name)
for config in product_frontmatter.get_auto_configs(METHOD):
for product in endoflife.list_products(METHOD, p_filter):
product_data = releasedata.Product(product.name)
for config in product.get_auto_configs(METHOD):
response = http.fetch_url(f"https://distrowatch.com/index.php?distribution={config.url}")
soup = BeautifulSoup(response.text, features="html5lib")
@@ -24,6 +23,6 @@ for product_name in endoflife.list_products(METHOD, p_filter):
date = dates.parse_date(table.select_one("td.NewsDate").get_text())
for version in versions:
product.declare_version(version, date)
product_data.declare_version(version, date)
product.write()
product_data.write()

11
src/docker_hub.py
View File

@@ -23,9 +23,8 @@ def fetch_releases(p: releasedata.Product, c: endoflife.AutoConfig, url: str) ->
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
for product_name in endoflife.list_products(METHOD, p_filter):
product = releasedata.Product(product_name)
product_frontmatter = endoflife.ProductFrontmatter(product.name)
for config in product_frontmatter.get_auto_configs(METHOD):
fetch_releases(product, config, f"https://hub.docker.com/v2/repositories/{config.url}/tags?page_size=100&page=1")
product.write()
for product in endoflife.list_products(METHOD, p_filter):
product_data = releasedata.Product(product.name)
for config in product.get_auto_configs(METHOD):
fetch_releases(product_data, config, f"https://hub.docker.com/v2/repositories/{config.url}/tags?page_size=100&page=1")
product_data.write()

11
src/git.py
View File

@@ -8,10 +8,9 @@ from common.git import Git
METHOD = 'git'
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
for product_name in endoflife.list_products(METHOD, p_filter):
product = releasedata.Product(product_name)
product_frontmatter = endoflife.ProductFrontmatter(product.name)
for config in product_frontmatter.get_auto_configs(METHOD):
for product in endoflife.list_products(METHOD, p_filter):
product_data = releasedata.Product(product.name)
for config in product.get_auto_configs(METHOD):
git = Git(config.url)
git.setup(bare=True)
@@ -21,6 +20,6 @@ for product_name in endoflife.list_products(METHOD, p_filter):
if version_match:
version = config.render(version_match)
date = dates.parse_date(date_str)
product.declare_version(version, date)
product_data.declare_version(version, date)
product.write()
product_data.write()

11
src/github-releases.py
View File

@@ -43,10 +43,9 @@ query($endCursor: String) {
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
for product_name in endoflife.list_products(METHOD, p_filter):
product = releasedata.Product(product_name)
product_frontmatter = endoflife.ProductFrontmatter(product.name)
for config in product_frontmatter.get_auto_configs(METHOD):
for product in endoflife.list_products(METHOD, p_filter):
product_data = releasedata.Product(product.name)
for config in product.get_auto_configs(METHOD):
for page in fetch_releases(config.url):
releases = [edge['node'] for edge in (page['data']['repository']['releases']['edges'])]
@@ -57,6 +56,6 @@ for product_name in endoflife.list_products(METHOD, p_filter):
if version_match:
version = config.render(version_match)
date = dates.parse_datetime(release['publishedAt'])
product.declare_version(version, date)
product_data.declare_version(version, date)
product.write()
product_data.write()

11
src/maven.py
View File

@@ -6,10 +6,9 @@ from common import endoflife, http, releasedata
METHOD = "maven"
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
for product_name in endoflife.list_products(METHOD, p_filter):
product = releasedata.Product(product_name)
product_frontmatter = endoflife.ProductFrontmatter(product.name)
for config in product_frontmatter.get_auto_configs(METHOD):
for product in endoflife.list_products(METHOD, p_filter):
product_data = releasedata.Product(product.name)
for config in product.get_auto_configs(METHOD):
start = 0
group_id, artifact_id = config.url.split("/")
@@ -22,10 +21,10 @@ for product_name in endoflife.list_products(METHOD, p_filter):
if version_match:
version = config.render(version_match)
date = datetime.fromtimestamp(row["timestamp"] / 1000, tz=timezone.utc)
product.declare_version(version, date)
product_data.declare_version(version, date)
start += 100
if data["response"]["numFound"] <= start:
break
product.write()
product_data.write()

11
src/npm.py
View File

@@ -5,16 +5,15 @@ from common import dates, endoflife, http, releasedata
METHOD = "npm"
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
for product_name in endoflife.list_products(METHOD, p_filter):
product = releasedata.Product(product_name)
product_frontmatter = endoflife.ProductFrontmatter(product.name)
for config in product_frontmatter.get_auto_configs(METHOD):
for product in endoflife.list_products(METHOD, p_filter):
product_data = releasedata.Product(product.name)
for config in product.get_auto_configs(METHOD):
data = http.fetch_url(f"https://registry.npmjs.org/{config.url}").json()
for version_str in data["versions"]:
version_match = config.first_match(version_str)
if version_match:
version = config.render(version_match)
date = dates.parse_datetime(data["time"][version_str])
product.declare_version(version, date)
product_data.declare_version(version, date)
product.write()
product_data.write()

11
src/pypi.py
View File

@@ -5,10 +5,9 @@ from common import dates, endoflife, http, releasedata
METHOD = "pypi"
p_filter = sys.argv[1] if len(sys.argv) > 1 else None
for product_name in endoflife.list_products(METHOD, p_filter):
product = releasedata.Product(product_name)
product_frontmatter = endoflife.ProductFrontmatter(product.name)
for config in product_frontmatter.get_auto_configs(METHOD):
for product in endoflife.list_products(METHOD, p_filter):
product_data = releasedata.Product(product.name)
for config in product.get_auto_configs(METHOD):
data = http.fetch_url(f"https://pypi.org/pypi/{config.url}/json").json()
for version_str in data["releases"]:
@@ -18,6 +17,6 @@ for product_name in endoflife.list_products(METHOD, p_filter):
if version_match and version_data:
version = config.render(version_match)
date = dates.parse_datetime(version_data[0]["upload_time_iso_8601"])
product.declare_version(version, date)
product_data.declare_version(version, date)
product.write()
product_data.write()