[py] Fix styling
This commit is contained in:
133
src/apple.py
133
src/apple.py
@@ -6,85 +6,92 @@ import re
|
||||
from html.parser import HTMLParser
|
||||
|
||||
URLS = [
|
||||
"https://support.apple.com/en-us/HT201222", # latest
|
||||
"https://support.apple.com/kb/HT213078", # 2018-2019
|
||||
"https://support.apple.com/kb/HT213077", # 2016-2017
|
||||
"https://support.apple.com/kb/HT209441", # 2015
|
||||
"https://support.apple.com/kb/HT205762", # 2014
|
||||
"https://support.apple.com/kb/HT205759", # 2013
|
||||
"https://support.apple.com/kb/HT204611", # 2011 to 2012
|
||||
"https://support.apple.com/kb/HT5165", # 2010
|
||||
"https://support.apple.com/kb/HT4218", # 2008-2009
|
||||
"https://support.apple.com/kb/HT1263", # 2005-2007
|
||||
"https://support.apple.com/en-us/HT201222", # latest
|
||||
"https://support.apple.com/kb/HT213078", # 2018-2019
|
||||
"https://support.apple.com/kb/HT213077", # 2016-2017
|
||||
"https://support.apple.com/kb/HT209441", # 2015
|
||||
"https://support.apple.com/kb/HT205762", # 2014
|
||||
"https://support.apple.com/kb/HT205759", # 2013
|
||||
"https://support.apple.com/kb/HT204611", # 2011 to 2012
|
||||
"https://support.apple.com/kb/HT5165", # 2010
|
||||
"https://support.apple.com/kb/HT4218", # 2008-2009
|
||||
"https://support.apple.com/kb/HT1263", # 2005-2007
|
||||
]
|
||||
|
||||
# If you are changing these, please
|
||||
# use https://www.toptal.com/developers/hastebin/mikahukube.txt as your corpus
|
||||
# to validate your changes
|
||||
CONFIG = {
|
||||
"macos": [
|
||||
# This covers Sierra and beyond
|
||||
r"macOS [\w ]+ (?P<version>\d{2}(?:\.\d+)+)",
|
||||
# This covers Mavericks - El Capitan
|
||||
r"OS X [\w ]+ v?(?P<version>\d{2}(?:\.\d+)+)",
|
||||
# This covers even older versions (OS X)
|
||||
r"^Mac OS X [\w ]+ v?(?P<version>\d{2}(?:\.\d+)+)",
|
||||
],
|
||||
"ios": [
|
||||
r"iOS (?P<version>\d+(?:)(?:\.\d+)+)",
|
||||
r"iPhone v?(?P<version>\d+(?:)(?:\.\d+)+)"
|
||||
],
|
||||
"ipados": [
|
||||
r"iPadOS (?P<version>\d+(?:)(?:\.\d+)+)"
|
||||
],
|
||||
"watchos": [
|
||||
r"watchOS (?P<version>\d+(?:)(?:\.\d+)+)"
|
||||
]
|
||||
"macos": [
|
||||
# This covers Sierra and beyond
|
||||
r"macOS [\w ]+ (?P<version>\d{2}(?:\.\d+)+)",
|
||||
# This covers Mavericks - El Capitan
|
||||
r"OS X [\w ]+ v?(?P<version>\d{2}(?:\.\d+)+)",
|
||||
# This covers even older versions (OS X)
|
||||
r"^Mac OS X [\w ]+ v?(?P<version>\d{2}(?:\.\d+)+)",
|
||||
],
|
||||
"ios": [
|
||||
r"iOS (?P<version>\d+(?:)(?:\.\d+)+)",
|
||||
r"iPhone v?(?P<version>\d+(?:)(?:\.\d+)+)",
|
||||
],
|
||||
"ipados": [r"iPadOS (?P<version>\d+(?:)(?:\.\d+)+)"],
|
||||
"watchos": [r"watchOS (?P<version>\d+(?:)(?:\.\d+)+)"],
|
||||
}
|
||||
|
||||
release_lists = {k: {} for k in CONFIG.keys()}
|
||||
print("::group::apple")
|
||||
|
||||
|
||||
def parse_date(input):
|
||||
d,m,y = input.strip().split(" ")
|
||||
m=m[0:3].lower()
|
||||
return datetime.datetime.strptime("%s %s %s" % (d,m,y), "%d %b %Y")
|
||||
d, m, y = input.strip().split(" ")
|
||||
m = m[0:3].lower()
|
||||
return datetime.datetime.strptime("%s %s %s" % (d, m, y), "%d %b %Y")
|
||||
|
||||
|
||||
for url in URLS:
|
||||
with urllib.request.urlopen(url, data=None, timeout=5) as response:
|
||||
soup = BeautifulSoup(response, features="html5lib")
|
||||
table = soup.find(id='tableWraper')
|
||||
for tr in reversed(table.findAll('tr')[1:]):
|
||||
td_list = tr.findAll('td')
|
||||
version_text = td_list[0].get_text()
|
||||
for key,regexes in CONFIG.items():
|
||||
for regex in regexes:
|
||||
matches = re.findall(regex, version_text, re.MULTILINE)
|
||||
if matches:
|
||||
for version in matches:
|
||||
abs_date = None
|
||||
try:
|
||||
abs_date = parse_date(td_list[2].get_text())
|
||||
print_date = abs_date.strftime("%Y-%m-%d")
|
||||
# Only update the date if we are adding first time
|
||||
# or if the date is lower
|
||||
if version not in release_lists[key]:
|
||||
release_lists[key][version] = abs_date
|
||||
print("%s-%s: %s" % (key, version, print_date))
|
||||
elif release_lists[key][version] < abs_date:
|
||||
print("%s-%s: %s [IGNORED]" % (key, version, print_date))
|
||||
elif release_lists[key][version] > abs_date:
|
||||
# This is a lower date, so we mark it with a bang
|
||||
print("%s-%s: %s [UPDATED]" % (key, version, print_date))
|
||||
release_lists[key][version] = abs_date
|
||||
except ValueError as e:
|
||||
print("%s-%s Failed to parse Date (%s)" % (key, version, td_list[2].get_text()))
|
||||
next
|
||||
with urllib.request.urlopen(url, data=None, timeout=5) as response:
|
||||
soup = BeautifulSoup(response, features="html5lib")
|
||||
table = soup.find(id="tableWraper")
|
||||
for tr in reversed(table.findAll("tr")[1:]):
|
||||
td_list = tr.findAll("td")
|
||||
version_text = td_list[0].get_text()
|
||||
for key, regexes in CONFIG.items():
|
||||
for regex in regexes:
|
||||
matches = re.findall(regex, version_text, re.MULTILINE)
|
||||
if matches:
|
||||
for version in matches:
|
||||
abs_date = None
|
||||
try:
|
||||
abs_date = parse_date(td_list[2].get_text())
|
||||
print_date = abs_date.strftime("%Y-%m-%d")
|
||||
# Only update the date if we are adding first time
|
||||
# or if the date is lower
|
||||
if version not in release_lists[key]:
|
||||
release_lists[key][version] = abs_date
|
||||
print("%s-%s: %s" % (key, version, print_date))
|
||||
elif release_lists[key][version] < abs_date:
|
||||
print(
|
||||
"%s-%s: %s [IGNORED]"
|
||||
% (key, version, print_date)
|
||||
)
|
||||
elif release_lists[key][version] > abs_date:
|
||||
# This is a lower date, so we mark it with a bang
|
||||
print(
|
||||
"%s-%s: %s [UPDATED]"
|
||||
% (key, version, print_date)
|
||||
)
|
||||
release_lists[key][version] = abs_date
|
||||
except ValueError as e:
|
||||
print(
|
||||
"%s-%s Failed to parse Date (%s)"
|
||||
% (key, version, td_list[2].get_text())
|
||||
)
|
||||
next
|
||||
|
||||
|
||||
for k in CONFIG.keys():
|
||||
with open("releases/%s.json" % k, 'w') as f:
|
||||
data = {v: d.strftime("%Y-%m-%d") for v,d in release_lists[k].items()}
|
||||
f.write(json.dumps(data, indent=2))
|
||||
with open("releases/%s.json" % k, "w") as f:
|
||||
data = {v: d.strftime("%Y-%m-%d") for v, d in release_lists[k].items()}
|
||||
f.write(json.dumps(data, indent=2))
|
||||
|
||||
print("::endgroup::")
|
||||
|
||||
@@ -10,19 +10,23 @@ from html.parser import HTMLParser
|
||||
from liquid import Template
|
||||
|
||||
# Same as used in Ruby (update.rb)
|
||||
DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
|
||||
DEFAULT_TAG_TEMPLATE = (
|
||||
"{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
|
||||
)
|
||||
|
||||
|
||||
def get_versions_from_headline(regex, headline, template):
|
||||
if not isinstance(regex, list):
|
||||
regex = [regex]
|
||||
for r in regex:
|
||||
matches = re.match(r.strip(), headline)
|
||||
if matches:
|
||||
match_data = matches.groupdict()
|
||||
version_string = template.render(**match_data)
|
||||
return version_string.split("\n")
|
||||
if not isinstance(regex, list):
|
||||
regex = [regex]
|
||||
for r in regex:
|
||||
matches = re.match(r.strip(), headline)
|
||||
if matches:
|
||||
match_data = matches.groupdict()
|
||||
version_string = template.render(**match_data)
|
||||
return version_string.split("\n")
|
||||
|
||||
return {}
|
||||
|
||||
return {}
|
||||
|
||||
def fetch_releases(distrowatch_id, regex, template):
|
||||
releases = {}
|
||||
@@ -34,34 +38,37 @@ def fetch_releases(distrowatch_id, regex, template):
|
||||
headline = table.select_one("td.NewsHeadline a[href]").get_text().strip()
|
||||
date = table.select_one("td.NewsDate").get_text()
|
||||
for v in get_versions_from_headline(regex, headline, l_template):
|
||||
print("%s: %s" % (v, date))
|
||||
releases[v] = date
|
||||
print("%s: %s" % (v, date))
|
||||
releases[v] = date
|
||||
return releases
|
||||
|
||||
|
||||
def update_releases(product_filter=None):
|
||||
for product_file in glob("website/products/*.md"):
|
||||
product_name = os.path.splitext(os.path.basename(product_file))[0]
|
||||
if product_filter and product_name != product_filter:
|
||||
continue
|
||||
with open(product_file, "r") as f:
|
||||
data = frontmatter.load(f)
|
||||
if "auto" in data:
|
||||
for config in data["auto"]:
|
||||
for key, d_id in config.items():
|
||||
if key == "distrowatch":
|
||||
update_product(product_name, config)
|
||||
for product_file in glob("website/products/*.md"):
|
||||
product_name = os.path.splitext(os.path.basename(product_file))[0]
|
||||
if product_filter and product_name != product_filter:
|
||||
continue
|
||||
with open(product_file, "r") as f:
|
||||
data = frontmatter.load(f)
|
||||
if "auto" in data:
|
||||
for config in data["auto"]:
|
||||
for key, d_id in config.items():
|
||||
if key == "distrowatch":
|
||||
update_product(product_name, config)
|
||||
|
||||
|
||||
def update_product(product_name, config):
|
||||
t = config.get("template", DEFAULT_TAG_TEMPLATE)
|
||||
if "regex" in config:
|
||||
print("::group::%s" % product_name)
|
||||
r = fetch_releases(config['distrowatch'], config["regex"], t)
|
||||
with open("releases/%s.json" % product_name, "w") as f:
|
||||
f.write(json.dumps(r, indent=2))
|
||||
print("::endgroup::")
|
||||
t = config.get("template", DEFAULT_TAG_TEMPLATE)
|
||||
if "regex" in config:
|
||||
print("::group::%s" % product_name)
|
||||
r = fetch_releases(config["distrowatch"], config["regex"], t)
|
||||
with open("releases/%s.json" % product_name, "w") as f:
|
||||
f.write(json.dumps(r, indent=2))
|
||||
print("::endgroup::")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1:
|
||||
update_releases(sys.argv[1])
|
||||
else:
|
||||
update_releases()
|
||||
if len(sys.argv) > 1:
|
||||
update_releases(sys.argv[1])
|
||||
else:
|
||||
update_releases()
|
||||
|
||||
@@ -3,19 +3,19 @@ import re
|
||||
import urllib.request
|
||||
|
||||
# https://regex101.com/r/1JCnFC/1
|
||||
REGEX = r'^(\d{4})\/(\d{2})\/(\d{2})\s+:\s+(\d+\.\d+\.\d.?)$'
|
||||
REGEX = r"^(\d{4})\/(\d{2})\/(\d{2})\s+:\s+(\d+\.\d+\.\d.?)$"
|
||||
|
||||
list = {}
|
||||
|
||||
for i in range(17, 27):
|
||||
url = "https://www.haproxy.org/download/%s/src/CHANGELOG" % (i/10)
|
||||
with urllib.request.urlopen(url) as response:
|
||||
for line in response:
|
||||
m = re.match(REGEX, line.decode('utf-8'))
|
||||
if m:
|
||||
year,month,date,version = m.groups()
|
||||
abs_date = "%s-%s-%s" % (year, month, date)
|
||||
list[version] = abs_date
|
||||
url = "https://www.haproxy.org/download/%s/src/CHANGELOG" % (i / 10)
|
||||
with urllib.request.urlopen(url) as response:
|
||||
for line in response:
|
||||
m = re.match(REGEX, line.decode("utf-8"))
|
||||
if m:
|
||||
year, month, date, version = m.groups()
|
||||
abs_date = "%s-%s-%s" % (year, month, date)
|
||||
list[version] = abs_date
|
||||
|
||||
with open('releases/haproxy.json', 'w') as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
with open("releases/haproxy.json", "w") as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
|
||||
68
src/maven.py
68
src/maven.py
@@ -6,48 +6,58 @@ import frontmatter
|
||||
import urllib.request
|
||||
import datetime
|
||||
|
||||
|
||||
def fetch_releases(package_identifier):
|
||||
groupId, artifactId = package_identifier.split("/")
|
||||
releases = {}
|
||||
start = 0
|
||||
while True:
|
||||
url = "https://search.maven.org/solrsearch/select?q=g:%s+AND+a:%s&core=gav&rows=100&wt=json&start=%s" % (groupId, artifactId, start)
|
||||
with urllib.request.urlopen(url, data=None, timeout=5) as response:
|
||||
data = json.load(response)
|
||||
for row in data['response']['docs']:
|
||||
date = datetime.datetime.utcfromtimestamp(row['timestamp'] / 1000)
|
||||
version = row['v']
|
||||
if not any(exception in version for exception in ['alpha', 'beta', 'nightly', 'rc', '-M']):
|
||||
abs_date = date.strftime("%Y-%m-%d")
|
||||
releases[version] = abs_date
|
||||
print("%s: %s" % (version, abs_date))
|
||||
start+=100
|
||||
if data['response']['numFound'] <= start:
|
||||
break
|
||||
url = (
|
||||
"https://search.maven.org/solrsearch/select?q=g:%s+AND+a:%s&core=gav&rows=100&wt=json&start=%s"
|
||||
% (groupId, artifactId, start)
|
||||
)
|
||||
with urllib.request.urlopen(url, data=None, timeout=5) as response:
|
||||
data = json.load(response)
|
||||
for row in data["response"]["docs"]:
|
||||
date = datetime.datetime.utcfromtimestamp(row["timestamp"] / 1000)
|
||||
version = row["v"]
|
||||
if not any(
|
||||
exception in version
|
||||
for exception in ["alpha", "beta", "nightly", "rc", "-M"]
|
||||
):
|
||||
abs_date = date.strftime("%Y-%m-%d")
|
||||
releases[version] = abs_date
|
||||
print("%s: %s" % (version, abs_date))
|
||||
start += 100
|
||||
if data["response"]["numFound"] <= start:
|
||||
break
|
||||
return releases
|
||||
|
||||
|
||||
def update_releases(product_filter=None):
|
||||
for product_file in glob("website/products/*.md"):
|
||||
product_name = os.path.splitext(os.path.basename(product_file))[0]
|
||||
if product_filter and product_name != product_filter:
|
||||
continue
|
||||
with open(product_file, "r") as f:
|
||||
data = frontmatter.load(f)
|
||||
if "auto" in data:
|
||||
for config in data["auto"]:
|
||||
for key, _ in config.items():
|
||||
if key == "maven":
|
||||
update_product(product_name, config)
|
||||
for product_file in glob("website/products/*.md"):
|
||||
product_name = os.path.splitext(os.path.basename(product_file))[0]
|
||||
if product_filter and product_name != product_filter:
|
||||
continue
|
||||
with open(product_file, "r") as f:
|
||||
data = frontmatter.load(f)
|
||||
if "auto" in data:
|
||||
for config in data["auto"]:
|
||||
for key, _ in config.items():
|
||||
if key == "maven":
|
||||
update_product(product_name, config)
|
||||
|
||||
|
||||
def update_product(product_name, config):
|
||||
print("::group::%s" % product_name)
|
||||
r = fetch_releases(config['maven'])
|
||||
r = fetch_releases(config["maven"])
|
||||
with open("releases/%s.json" % product_name, "w") as f:
|
||||
f.write(json.dumps(r, indent=2))
|
||||
print("::endgroup::")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1:
|
||||
update_releases(sys.argv[1])
|
||||
else:
|
||||
update_releases()
|
||||
if len(sys.argv) > 1:
|
||||
update_releases(sys.argv[1])
|
||||
else:
|
||||
update_releases()
|
||||
|
||||
@@ -6,36 +6,40 @@ from bs4 import BeautifulSoup
|
||||
URL = "https://www.paloaltonetworks.com/services/support/end-of-life-announcements/end-of-life-summary"
|
||||
|
||||
ID_MAPPING = {
|
||||
"pan-os-panorama": "pan-os",
|
||||
"globalprotect": "pan-gp",
|
||||
"traps-esm-and-cortex": "pan-xdr"
|
||||
"pan-os-panorama": "pan-os",
|
||||
"globalprotect": "pan-gp",
|
||||
"traps-esm-and-cortex": "pan-xdr",
|
||||
}
|
||||
|
||||
|
||||
def update_releases(html_id, file):
|
||||
list = {}
|
||||
with urllib.request.urlopen(URL, data=None, timeout=5) as response:
|
||||
soup = BeautifulSoup(response, features="html5lib")
|
||||
table = soup.find(id=html_id)
|
||||
for tr in table.findAll('tr')[3:]:
|
||||
td_list = tr.findAll('td')
|
||||
version = td_list[0].get_text().strip().lower().replace(" ", "-").replace("*","")
|
||||
if file == 'pan-xdr':
|
||||
if "xdr" not in version:
|
||||
continue
|
||||
version = version.removesuffix("-(cortex-xdr-agent)")
|
||||
version = version.removesuffix("-(vm-series-only)")
|
||||
version = version.removesuffix("-(panorama-only)")
|
||||
try:
|
||||
month,date,year = td_list[1].get_text().split('/')
|
||||
abs_date = f"{year}-{month:0>2}-{date:0>2}"
|
||||
except Exception as e:
|
||||
date = datetime.datetime.strptime(td_list[1].get_text(), "%B %d, %Y")
|
||||
abs_date = date.strftime("%Y-%m-%d")
|
||||
list = {}
|
||||
with urllib.request.urlopen(URL, data=None, timeout=5) as response:
|
||||
soup = BeautifulSoup(response, features="html5lib")
|
||||
table = soup.find(id=html_id)
|
||||
for tr in table.findAll("tr")[3:]:
|
||||
td_list = tr.findAll("td")
|
||||
version = (
|
||||
td_list[0].get_text().strip().lower().replace(" ", "-").replace("*", "")
|
||||
)
|
||||
if file == "pan-xdr":
|
||||
if "xdr" not in version:
|
||||
continue
|
||||
version = version.removesuffix("-(cortex-xdr-agent)")
|
||||
version = version.removesuffix("-(vm-series-only)")
|
||||
version = version.removesuffix("-(panorama-only)")
|
||||
try:
|
||||
month, date, year = td_list[1].get_text().split("/")
|
||||
abs_date = f"{year}-{month:0>2}-{date:0>2}"
|
||||
except Exception as e:
|
||||
date = datetime.datetime.strptime(td_list[1].get_text(), "%B %d, %Y")
|
||||
abs_date = date.strftime("%Y-%m-%d")
|
||||
|
||||
list[version] = abs_date
|
||||
list[version] = abs_date
|
||||
|
||||
with open("releases/%s.json" % file, "w") as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
|
||||
with open('releases/%s.json' % file, 'w') as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
|
||||
for html_id in ID_MAPPING:
|
||||
update_releases(html_id, ID_MAPPING[html_id])
|
||||
update_releases(html_id, ID_MAPPING[html_id])
|
||||
|
||||
78
src/pypi.py
78
src/pypi.py
@@ -11,55 +11,61 @@ from html.parser import HTMLParser
|
||||
from liquid import Template
|
||||
|
||||
# Same as used in Ruby (update.rb)
|
||||
DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
|
||||
REGEX = r'^(?:(\d+\.(?:\d+\.)*\d+))$'
|
||||
DEFAULT_TAG_TEMPLATE = (
|
||||
"{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
|
||||
)
|
||||
REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
|
||||
|
||||
|
||||
def fetch_releases(pypi_id, regex):
|
||||
releases = {}
|
||||
|
||||
if not isinstance(regex, list):
|
||||
regex = [regex]
|
||||
regex = [regex]
|
||||
|
||||
url = "https://pypi.org/pypi/%s/json" % pypi_id
|
||||
with urllib.request.urlopen(url, data=None, timeout=5) as response:
|
||||
data = json.loads(response.read().decode("utf-8"))
|
||||
for version in data['releases']:
|
||||
R = data['releases'][version]
|
||||
matches = False
|
||||
for r in regex:
|
||||
if re.match(r, version):
|
||||
matches = True
|
||||
if matches:
|
||||
d = datetime.fromisoformat(R[0]['upload_time']).strftime("%Y-%m-%d")
|
||||
releases[version] = d
|
||||
print("%s: %s" % (version, d))
|
||||
data = json.loads(response.read().decode("utf-8"))
|
||||
for version in data["releases"]:
|
||||
R = data["releases"][version]
|
||||
matches = False
|
||||
for r in regex:
|
||||
if re.match(r, version):
|
||||
matches = True
|
||||
if matches:
|
||||
d = datetime.fromisoformat(R[0]["upload_time"]).strftime("%Y-%m-%d")
|
||||
releases[version] = d
|
||||
print("%s: %s" % (version, d))
|
||||
|
||||
return releases
|
||||
|
||||
|
||||
def update_releases(product_filter=None):
|
||||
for product_file in glob("website/products/*.md"):
|
||||
product_name = os.path.splitext(os.path.basename(product_file))[0]
|
||||
if product_filter and product_name != product_filter:
|
||||
continue
|
||||
with open(product_file, "r") as f:
|
||||
data = frontmatter.load(f)
|
||||
if "auto" in data:
|
||||
for config in data["auto"]:
|
||||
for key, d_id in config.items():
|
||||
if key == "pypi":
|
||||
update_product(product_name, config)
|
||||
for product_file in glob("website/products/*.md"):
|
||||
product_name = os.path.splitext(os.path.basename(product_file))[0]
|
||||
if product_filter and product_name != product_filter:
|
||||
continue
|
||||
with open(product_file, "r") as f:
|
||||
data = frontmatter.load(f)
|
||||
if "auto" in data:
|
||||
for config in data["auto"]:
|
||||
for key, d_id in config.items():
|
||||
if key == "pypi":
|
||||
update_product(product_name, config)
|
||||
|
||||
|
||||
def update_product(product_name, config):
|
||||
if "pypi" in config:
|
||||
print("::group::%s" % product_name)
|
||||
config = config | {"regex": REGEX}
|
||||
r = fetch_releases(config['pypi'], config["regex"])
|
||||
with open("releases/%s.json" % product_name, "w") as f:
|
||||
f.write(json.dumps(r, indent=2))
|
||||
print("::endgroup::")
|
||||
if "pypi" in config:
|
||||
print("::group::%s" % product_name)
|
||||
config = config | {"regex": REGEX}
|
||||
r = fetch_releases(config["pypi"], config["regex"])
|
||||
with open("releases/%s.json" % product_name, "w") as f:
|
||||
f.write(json.dumps(r, indent=2))
|
||||
print("::endgroup::")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1:
|
||||
update_releases(sys.argv[1])
|
||||
else:
|
||||
update_releases()
|
||||
if len(sys.argv) > 1:
|
||||
update_releases(sys.argv[1])
|
||||
else:
|
||||
update_releases()
|
||||
|
||||
34
src/rhel.py
34
src/rhel.py
@@ -8,24 +8,22 @@ URL = "https://access.redhat.com/articles/3078"
|
||||
regex = r"RHEL (?P<major>\d)(\. ?(?P<minor>\d+))?(( Update (?P<minor2>\d))| GA)?"
|
||||
|
||||
list = {}
|
||||
headers = {
|
||||
'user-agent': 'mozilla'
|
||||
}
|
||||
req = urllib.request.Request(URL,headers=headers)
|
||||
headers = {"user-agent": "mozilla"}
|
||||
req = urllib.request.Request(URL, headers=headers)
|
||||
|
||||
with urllib.request.urlopen(req, timeout=5) as response:
|
||||
soup = BeautifulSoup(response, features="html5lib")
|
||||
for tr in soup.findAll('tr'):
|
||||
td_list = tr.findAll('td')
|
||||
if len(td_list)>0:
|
||||
version = td_list[0].get_text()
|
||||
m = re.match(regex, version.strip()).groupdict()
|
||||
version = m['major']
|
||||
if m['minor']:
|
||||
version += ".%s" % m['minor']
|
||||
if m['minor2']:
|
||||
version += ".%s" % m['minor2']
|
||||
list[version] = td_list[1].get_text()
|
||||
soup = BeautifulSoup(response, features="html5lib")
|
||||
for tr in soup.findAll("tr"):
|
||||
td_list = tr.findAll("td")
|
||||
if len(td_list) > 0:
|
||||
version = td_list[0].get_text()
|
||||
m = re.match(regex, version.strip()).groupdict()
|
||||
version = m["major"]
|
||||
if m["minor"]:
|
||||
version += ".%s" % m["minor"]
|
||||
if m["minor2"]:
|
||||
version += ".%s" % m["minor2"]
|
||||
list[version] = td_list[1].get_text()
|
||||
|
||||
with open('releases/redhat.json', 'w') as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
with open("releases/redhat.json", "w") as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
|
||||
40
src/ros.py
40
src/ros.py
@@ -11,24 +11,26 @@ regex = r"^ROS (?P<name>(\w| )+)"
|
||||
list = {}
|
||||
|
||||
with urllib.request.urlopen(URL, timeout=5) as response:
|
||||
soup = BeautifulSoup(response, features="html5lib")
|
||||
for tr in soup.findAll('tr'):
|
||||
td_list = tr.findAll('td')
|
||||
if len(td_list)>0:
|
||||
version = td_list[0].get_text()
|
||||
soup = BeautifulSoup(response, features="html5lib")
|
||||
for tr in soup.findAll("tr"):
|
||||
td_list = tr.findAll("td")
|
||||
if len(td_list) > 0:
|
||||
version = td_list[0].get_text()
|
||||
|
||||
m = re.match(regex, version.strip())
|
||||
if m:
|
||||
version = td_list[0].findAll('a')[0]['href'][1:]
|
||||
try:
|
||||
date = datetime.datetime.strptime(td_list[1].get_text().strip(), "%B %d, %Y")
|
||||
# The date is a suffix (May 23rd, 2020)
|
||||
except Exception as e:
|
||||
x = td_list[1].get_text().split(',')
|
||||
date = datetime.datetime.strptime(x[0][:-2] + x[1], "%B %d %Y")
|
||||
abs_date = date.strftime("%Y-%m-%d")
|
||||
list[version] = abs_date
|
||||
print("%s: %s" % (version, abs_date))
|
||||
m = re.match(regex, version.strip())
|
||||
if m:
|
||||
version = td_list[0].findAll("a")[0]["href"][1:]
|
||||
try:
|
||||
date = datetime.datetime.strptime(
|
||||
td_list[1].get_text().strip(), "%B %d, %Y"
|
||||
)
|
||||
# The date is a suffix (May 23rd, 2020)
|
||||
except Exception as e:
|
||||
x = td_list[1].get_text().split(",")
|
||||
date = datetime.datetime.strptime(x[0][:-2] + x[1], "%B %d %Y")
|
||||
abs_date = date.strftime("%Y-%m-%d")
|
||||
list[version] = abs_date
|
||||
print("%s: %s" % (version, abs_date))
|
||||
|
||||
with open('releases/ros.json', 'w') as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
with open("releases/ros.json", "w") as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
|
||||
@@ -4,21 +4,21 @@ import re
|
||||
import urllib.request
|
||||
|
||||
URL = "https://www.unrealircd.org/docwiki/index.php?title=History_of_UnrealIRCd_releases&action=raw"
|
||||
REGEX = r'^(?:(\d+\.(?:\d+\.)*\d+))$'
|
||||
REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
|
||||
|
||||
list = {}
|
||||
with urllib.request.urlopen(URL) as response:
|
||||
text = response.read()
|
||||
wikicode = mwparserfromhell.parse(text)
|
||||
for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"):
|
||||
items = tr.contents.filter_tags(matches=lambda node: node.tag == "td")
|
||||
if len(items) >=2:
|
||||
maybe_version = items[0].__strip__()
|
||||
if re.match(REGEX, maybe_version):
|
||||
maybe_date = items[1].__strip__()
|
||||
if re.match(r'\d{4}-\d{2}-\d{2}', maybe_date):
|
||||
list[maybe_version] = maybe_date
|
||||
text = response.read()
|
||||
wikicode = mwparserfromhell.parse(text)
|
||||
for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"):
|
||||
items = tr.contents.filter_tags(matches=lambda node: node.tag == "td")
|
||||
if len(items) >= 2:
|
||||
maybe_version = items[0].__strip__()
|
||||
if re.match(REGEX, maybe_version):
|
||||
maybe_date = items[1].__strip__()
|
||||
if re.match(r"\d{4}-\d{2}-\d{2}", maybe_date):
|
||||
list[maybe_version] = maybe_date
|
||||
|
||||
|
||||
with open('releases/unrealircd.json', 'w') as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
with open("releases/unrealircd.json", "w") as f:
|
||||
f.write(json.dumps(list, indent=2))
|
||||
|
||||
Reference in New Issue
Block a user