github/endoflife-date-release-data
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Refactor product releases writing

Browse Source 
Create a common function to write resulting JSON files to the releases directory.
It makes this task simpler to read and maintain, while making it modifiable at a central point in the future.
One example of such modification could be the sorting of the versions in a uniform way for all the scripts.
This commit is contained in:
Marc Wrobel
2023-05-20 11:00:47 +02:00
parent a16d9090d3
commit 6ba7828f96
22 changed files with 103 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/apple.py
View File

@@ -1,5 +1,4 @@
import datetime import datetime
import json
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
@@ -97,8 +96,7 @@ for url in URLS:
for k in CONFIG.keys(): for k in CONFIG.keys():
with open("releases/%s.json" % k, "w") as f: releases = {v: d.strftime("%Y-%m-%d") for v, d in release_lists[k].items()}
data = {v: d.strftime("%Y-%m-%d") for v, d in release_lists[k].items()} endoflife.write_releases(k, releases)
f.write(json.dumps(data, indent=2))
print("::endgroup::") print("::endgroup::")

7
src/common/endoflife.py
View File

@@ -1,6 +1,6 @@
import json
import frontmatter import frontmatter
import urllib.request import urllib.request
from glob import glob from glob import glob
from os import path from os import path
@@ -47,3 +47,8 @@ def fetch_url(url, retry_count=2, timeout=5, data=None, headers=None, encoding='
continue continue
raise last_exception raise last_exception
def write_releases(product, releases, pathname="releases"):
with open(f"{pathname}/{product}.json", "w") as f:
f.write(json.dumps(releases, indent=2))

7
src/cos.py
View File

@@ -1,4 +1,3 @@
import json
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
@@ -69,7 +68,5 @@ def get_all_versions():
return all_versions return all_versions
if __name__ == '__main__': releases = get_all_versions()
v = get_all_versions() endoflife.write_releases('cos', releases)
with open('releases/cos.json', "w") as f:
f.write(json.dumps(v, indent=2))

30
src/debian.py
View File

@@ -1,6 +1,6 @@
import json
import pathlib import pathlib
import subprocess import subprocess
from common import endoflife
from hashlib import sha1 from hashlib import sha1
from os.path import exists from os.path import exists
from subprocess import call from subprocess import call
@@ -73,21 +73,13 @@ def extract_point_releases(releases):
releases[version] = date releases[version] = date
def main(): print(f"::group::{PRODUCT}")
print(f"::group::{PRODUCT}") clone_repository()
clone_repository() releases = {}
extract_major_releases(releases)
releases = {} extract_point_releases(releases)
extract_major_releases(releases) endoflife.write_releases(PRODUCT, dict(
extract_point_releases(releases) # sort by date then version (desc)
print("::endgroup::") sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
))
with open(f"releases/{PRODUCT}.json", "w") as f: print("::endgroup::")
f.write(json.dumps(dict(
# sort by date then version (desc)
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
), indent=2))
if __name__ == '__main__':
main()

4
src/distrowatch.py
View File

@@ -1,4 +1,3 @@
import json
import re import re
import sys import sys
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
@@ -48,8 +47,7 @@ def update_product(product_name, configs):
regex = config["regex"] regex = config["regex"]
releases = releases | fetch_releases(config[METHOD], regex, t) releases = releases | fetch_releases(config[METHOD], regex, t)
with open("releases/%s.json" % product_name, "w") as f: endoflife.write_releases(product_name, releases)
f.write(json.dumps(releases, indent=2))
p_filter = sys.argv[1] if len(sys.argv) > 1 else None p_filter = sys.argv[1] if len(sys.argv) > 1 else None

7
src/eks.py
View File

@@ -1,5 +1,4 @@
import datetime import datetime
import json
import markdown import markdown
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
@@ -32,7 +31,5 @@ def parse_platforms_page():
return all_versions return all_versions
if __name__ == "__main__": versions = parse_platforms_page()
versions = parse_platforms_page() endoflife.write_releases('eks', versions)
with open("releases/eks.json", "w") as f:
f.write(json.dumps(versions, indent=2))

23
src/firefox.py
View File

@@ -1,5 +1,4 @@
import concurrent.futures import concurrent.futures
import json
import re import re
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
@@ -134,18 +133,10 @@ def fetch_releases():
return releases return releases
def main(): print(f"::group::{PRODUCT}")
print(f"::group::{PRODUCT}") releases = fetch_releases()
endoflife.write_releases(PRODUCT, dict(
releases = fetch_releases() # sort by date then version (desc)
with open(f"releases/{PRODUCT}.json", "w") as f: sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
f.write(json.dumps(dict( ))
# sort by date then version (desc) print("::endgroup::")
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
), indent=2))
print("::endgroup::")
if __name__ == '__main__':
main()

4
src/github-releases.py
View File

@@ -1,6 +1,5 @@
import re import re
import sys import sys
import json
import subprocess import subprocess
from common import endoflife from common import endoflife
@@ -72,8 +71,7 @@ def update_product(product_name, configs):
config = config if "regex" in config else config | {"regex": REGEX} config = config if "regex" in config else config | {"regex": REGEX}
releases = releases | fetch_releases(config[METHOD], config["regex"]) releases = releases | fetch_releases(config[METHOD], config["regex"])
with open(f"releases/{product_name}.json", "w") as f: endoflife.write_releases(product_name, releases)
f.write(json.dumps(releases, indent=2))
p_filter = sys.argv[1] if len(sys.argv) > 1 else None p_filter = sys.argv[1] if len(sys.argv) > 1 else None

25
src/gke.py
View File

@@ -1,4 +1,3 @@
import json
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
@@ -38,20 +37,10 @@ def parse_soup_for_versions(soup):
CHANNELS = ['nochannel', 'stable', 'regular', 'rapid'] CHANNELS = ['nochannel', 'stable', 'regular', 'rapid']
def main(): for channel in CHANNELS:
for channel in CHANNELS: soup = fetch_channel(channel)
soup = fetch_channel(channel) print("::group::GKE - {}".format(channel))
print("::group::GKE - {}".format(channel)) versions = parse_soup_for_versions(soup)
versions = parse_soup_for_versions(soup) name = 'gke' if channel == 'nochannel' else 'gke-{}'.format(channel)
for version, date in versions.items(): endoflife.write_releases(name, versions)
print("{}: {}".format(version, date)) print("::endgroup::")
if channel == 'nochannel':
fn = 'releases/gke.json'
else:
fn = 'releases/gke-{}.json'.format(channel)
with open(fn, "w") as f:
f.write(json.dumps(versions, indent=2))
print("::endgroup::")
if __name__ == '__main__':
main()

26
src/haproxy.py
View File

@@ -1,4 +1,3 @@
import json
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
@@ -51,19 +50,12 @@ def print_releases(releases):
print(f"{version} : {date}") print(f"{version} : {date}")
def main(): print(f"::group::{PRODUCT}")
print(f"::group::{PRODUCT}") cycles = fetch_cycles()
cycles = fetch_cycles() releases = fetch_releases(cycles)
releases = fetch_releases(cycles) print_releases(releases)
print_releases(releases) endoflife.write_releases(PRODUCT, dict(
print("::endgroup::") # sort by date then version (desc)
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
with open(f"releases/{PRODUCT}.json", "w") as f: ))
f.write(json.dumps(dict( print("::endgroup::")
# sort by date then version (desc)
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
), indent=2))
if __name__ == '__main__':
main()

28
src/java.py
View File

@@ -1,6 +1,5 @@
import json
from requests_html import HTMLSession from requests_html import HTMLSession
from common import endoflife
"""Fetch Java versions with their dates from https://www.java.com/releases/. """Fetch Java versions with their dates from https://www.java.com/releases/.
@@ -37,19 +36,12 @@ def fetch_releases(releases):
previous_date = date previous_date = date
def main(): print(f"::group::{PRODUCT}")
print(f"::group::{PRODUCT}") releases = {}
releases = {} fetch_releases(releases)
fetch_releases(releases) releases.pop('1.0_alpha') # that's the only version we do not want, regex not needed
releases.pop('1.0_alpha') # that's the only version we do not want, regex not needed endoflife.write_releases(PRODUCT, dict(
print("::endgroup::") # sort by date then version (desc)
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
with open(f"releases/{PRODUCT}.json", "w") as f: ))
f.write(json.dumps(dict( print("::endgroup::")
# sort by date then version (desc)
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
), indent=2))
if __name__ == '__main__':
main()

22
src/linuxkernel.py
View File

@@ -1,4 +1,3 @@
import json
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
@@ -51,17 +50,10 @@ def fetch_releases():
return releases return releases
def main(): print(f"::group::{PRODUCT}")
print(f"::group::{PRODUCT}") releases = fetch_releases()
releases = fetch_releases() endoflife.write_releases(PRODUCT, dict(
print("::endgroup::") # sort by version then date (asc)
sorted(releases.items(), key=lambda x: (x[0], x[1]))
with open(f"releases/{PRODUCT}.json", "w") as f: ))
f.write(json.dumps(dict( print("::endgroup::")
# sort by version then date (asc)
sorted(releases.items(), key=lambda x: (x[0], x[1]))
), indent=2))
if __name__ == '__main__':
main()

9
src/maven.py
View File

@@ -50,11 +50,10 @@ def update_product(product_name, configs):
for config in configs: for config in configs:
releases = releases | fetch_releases(config[METHOD]) releases = releases | fetch_releases(config[METHOD])
with open(f"releases/{product_name}.json", "w") as f: endoflife.write_releases(product_name, dict(
f.write(json.dumps(dict( # sort by date then version (desc)
# sort by date then version (desc) sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) ))
), indent=2))
p_filter = sys.argv[1] if len(sys.argv) > 1 else None p_filter = sys.argv[1] if len(sys.argv) > 1 else None

3
src/npm.py
View File

@@ -40,8 +40,7 @@ def update_product(product_name, configs):
config = {"regex": REGEX} | config config = {"regex": REGEX} | config
releases = releases | fetch_releases(config[METHOD], config["regex"]) releases = releases | fetch_releases(config[METHOD], config["regex"])
with open(f"releases/{product_name}.json", "w") as f: endoflife.write_releases(product_name, releases)
f.write(json.dumps(releases, indent=2))

7
src/palo-alto-networks.py
View File

@@ -1,5 +1,4 @@
import datetime import datetime
import json
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
@@ -19,6 +18,7 @@ def update_releases(html_identifier, file):
print(f"::group::{html_identifier}") print(f"::group::{html_identifier}")
response = endoflife.fetch_url(URL) response = endoflife.fetch_url(URL)
soup = BeautifulSoup(response, features="html5lib") soup = BeautifulSoup(response, features="html5lib")
table = soup.find(id=html_identifier) table = soup.find(id=html_identifier)
for tr in table.findAll("tr")[3:]: for tr in table.findAll("tr")[3:]:
td_list = tr.findAll("td") td_list = tr.findAll("td")
@@ -45,10 +45,9 @@ def update_releases(html_identifier, file):
versions[version] = abs_date versions[version] = abs_date
print("%s: %s" % (version, abs_date)) print("%s: %s" % (version, abs_date))
print("::endgroup::")
with open("releases/%s.json" % file, "w") as f: endoflife.write_releases(file, versions)
f.write(json.dumps(versions, indent=2)) print("::endgroup::")
for html_id in ID_MAPPING: for html_id in ID_MAPPING:

24
src/php.py
View File

@@ -26,21 +26,13 @@ def fetch_versions(major_version):
return data return data
with open("releases/php.json", "w") as f: print("::group::php")
print("::group::php") releases = {}
releases = {}
for major_version in PHP_MAJOR_VERSIONS: for major_version in PHP_MAJOR_VERSIONS:
releases |= fetch_versions(major_version) releases |= fetch_versions(major_version)
f.write( endoflife.write_releases('php', dict(sorted(
json.dumps( releases.items(), key=lambda x: list(map(int, x[0].split(".")))
dict(sorted( )))
releases.items(), print("::endgroup::")
key=lambda x: list(map(int, x[0].split(".")))
)),
indent=2,
)
)
print("::endgroup::")

23
src/plesk.py
View File

@@ -1,4 +1,3 @@
import json
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
from datetime import datetime from datetime import datetime
@@ -35,18 +34,10 @@ def fetch_releases():
return result return result
def main(): print(f"::group::{PRODUCT}")
print(f"::group::{PRODUCT}") releases = fetch_releases()
endoflife.write_releases(PRODUCT, dict(
releases = fetch_releases() # sort by date then version (desc)
with open(f"releases/{PRODUCT}.json", "w") as f: sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
f.write(json.dumps(dict( ))
# sort by date then version (desc) print("::endgroup::")
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
), indent=2))
print("::endgroup::")
if __name__ == '__main__':
main()

3
src/pypi.py
View File

@@ -41,8 +41,7 @@ def update_product(product_name, configs):
config = {"regex": REGEX} | config config = {"regex": REGEX} | config
releases = releases | fetch_releases(config[METHOD], config["regex"]) releases = releases | fetch_releases(config[METHOD], config["regex"])
with open(f"releases/{product_name}.json", "w") as f: endoflife.write_releases(product_name, releases)
f.write(json.dumps(releases, indent=2))
p_filter = sys.argv[1] if len(sys.argv) > 1 else None p_filter = sys.argv[1] if len(sys.argv) > 1 else None

10
src/rds.py
View File

@@ -1,4 +1,3 @@
import json
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
@@ -36,9 +35,8 @@ for db, url in dbs.items():
print(f"{version} : {date}") print(f"{version} : {date}")
releases[version] = date releases[version] = date
endoflife.write_releases(f"amazon-rds-{db.lower()}", dict(
# sort by date then version (desc)
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
))
print("::endgroup::") print("::endgroup::")
with open(f"releases/amazon-rds-{db.lower()}.json", "w") as f:
json.dump(dict(
# sort by date then version (desc)
sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True)
), f, indent=2)

10
src/rhel.py
View File

@@ -1,4 +1,3 @@
import json
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
@@ -7,11 +6,11 @@ URL = "https://access.redhat.com/articles/3078"
# https://regex101.com/r/877ibq/1 # https://regex101.com/r/877ibq/1
regex = r"RHEL (?P<major>\d)(\. ?(?P<minor>\d+))?(( Update (?P<minor2>\d))| GA)?" regex = r"RHEL (?P<major>\d)(\. ?(?P<minor>\d+))?(( Update (?P<minor2>\d))| GA)?"
versions = {}
print("::group::rhel") print("::group::rhel")
response = endoflife.fetch_url(URL) response = endoflife.fetch_url(URL)
soup = BeautifulSoup(response, features="html5lib") soup = BeautifulSoup(response, features="html5lib")
versions = {}
for tr in soup.findAll("tr"): for tr in soup.findAll("tr"):
td_list = tr.findAll("td") td_list = tr.findAll("td")
if len(td_list) > 0: if len(td_list) > 0:
@@ -25,7 +24,6 @@ for tr in soup.findAll("tr"):
date = td_list[1].get_text() date = td_list[1].get_text()
versions[version] = date versions[version] = date
print("%s: %s" % (version, date)) print("%s: %s" % (version, date))
print("::endgroup::")
with open("releases/redhat.json", "w") as f: endoflife.write_releases('redhat', versions)
f.write(json.dumps(versions, indent=2)) print("::endgroup::")

10
src/ros.py
View File

@@ -1,5 +1,4 @@
import datetime import datetime
import json
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import endoflife from common import endoflife
@@ -8,11 +7,11 @@ URL = "https://wiki.ros.org/Distributions"
# https://regex101.com/r/c1ribd/1 # https://regex101.com/r/c1ribd/1
regex = r"^ROS (?P<name>(\w| )+)" regex = r"^ROS (?P<name>(\w| )+)"
versions = {}
print("::group::ros") print("::group::ros")
response = endoflife.fetch_url(URL) response = endoflife.fetch_url(URL)
soup = BeautifulSoup(response, features="html5lib") soup = BeautifulSoup(response, features="html5lib")
versions = {}
for tr in soup.findAll("tr"): for tr in soup.findAll("tr"):
td_list = tr.findAll("td") td_list = tr.findAll("td")
if len(td_list) > 0: if len(td_list) > 0:
@@ -32,7 +31,6 @@ for tr in soup.findAll("tr"):
abs_date = date.strftime("%Y-%m-%d") abs_date = date.strftime("%Y-%m-%d")
versions[version] = abs_date versions[version] = abs_date
print("%s: %s" % (version, abs_date)) print("%s: %s" % (version, abs_date))
print("::endgroup::")
with open("releases/ros.json", "w") as f: endoflife.write_releases('ros', versions)
f.write(json.dumps(versions, indent=2)) print("::endgroup::")

5
src/unrealircd.py
View File

@@ -20,7 +20,6 @@ for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"):
if re.match(r"\d{4}-\d{2}-\d{2}", maybe_date): if re.match(r"\d{4}-\d{2}-\d{2}", maybe_date):
versions[maybe_version] = maybe_date versions[maybe_version] = maybe_date
print("%s: %s" % (maybe_version, maybe_date)) print("%s: %s" % (maybe_version, maybe_date))
print("::endgroup::")
with open("releases/unrealircd.json", "w") as f: endoflife.write_releases('unrealircd', versions)
f.write(json.dumps(versions, indent=2)) print("::endgroup::")