[py] Fix styling

This commit is contained in:
Nemo
2022-10-16 13:59:39 +05:30
committed by Nemo
parent 9a35436353
commit fac9620241
9 changed files with 283 additions and 249 deletions

View File

@@ -6,85 +6,92 @@ import re
from html.parser import HTMLParser
URLS = [
"https://support.apple.com/en-us/HT201222", # latest
"https://support.apple.com/kb/HT213078", # 2018-2019
"https://support.apple.com/kb/HT213077", # 2016-2017
"https://support.apple.com/kb/HT209441", # 2015
"https://support.apple.com/kb/HT205762", # 2014
"https://support.apple.com/kb/HT205759", # 2013
"https://support.apple.com/kb/HT204611", # 2011 to 2012
"https://support.apple.com/kb/HT5165", # 2010
"https://support.apple.com/kb/HT4218", # 2008-2009
"https://support.apple.com/kb/HT1263", # 2005-2007
"https://support.apple.com/en-us/HT201222", # latest
"https://support.apple.com/kb/HT213078", # 2018-2019
"https://support.apple.com/kb/HT213077", # 2016-2017
"https://support.apple.com/kb/HT209441", # 2015
"https://support.apple.com/kb/HT205762", # 2014
"https://support.apple.com/kb/HT205759", # 2013
"https://support.apple.com/kb/HT204611", # 2011 to 2012
"https://support.apple.com/kb/HT5165", # 2010
"https://support.apple.com/kb/HT4218", # 2008-2009
"https://support.apple.com/kb/HT1263", # 2005-2007
]
# If you are changing these, please
# use https://www.toptal.com/developers/hastebin/mikahukube.txt as your corpus
# to validate your changes
CONFIG = {
"macos": [
# This covers Sierra and beyond
r"macOS [\w ]+ (?P<version>\d{2}(?:\.\d+)+)",
# This covers Mavericks - El Capitan
r"OS X [\w ]+ v?(?P<version>\d{2}(?:\.\d+)+)",
# This covers even older versions (OS X)
r"^Mac OS X [\w ]+ v?(?P<version>\d{2}(?:\.\d+)+)",
],
"ios": [
r"iOS (?P<version>\d+(?:)(?:\.\d+)+)",
r"iPhone v?(?P<version>\d+(?:)(?:\.\d+)+)"
],
"ipados": [
r"iPadOS (?P<version>\d+(?:)(?:\.\d+)+)"
],
"watchos": [
r"watchOS (?P<version>\d+(?:)(?:\.\d+)+)"
]
"macos": [
# This covers Sierra and beyond
r"macOS [\w ]+ (?P<version>\d{2}(?:\.\d+)+)",
# This covers Mavericks - El Capitan
r"OS X [\w ]+ v?(?P<version>\d{2}(?:\.\d+)+)",
# This covers even older versions (OS X)
r"^Mac OS X [\w ]+ v?(?P<version>\d{2}(?:\.\d+)+)",
],
"ios": [
r"iOS (?P<version>\d+(?:)(?:\.\d+)+)",
r"iPhone v?(?P<version>\d+(?:)(?:\.\d+)+)",
],
"ipados": [r"iPadOS (?P<version>\d+(?:)(?:\.\d+)+)"],
"watchos": [r"watchOS (?P<version>\d+(?:)(?:\.\d+)+)"],
}
release_lists = {k: {} for k in CONFIG.keys()}
print("::group::apple")
def parse_date(input):
d,m,y = input.strip().split(" ")
m=m[0:3].lower()
return datetime.datetime.strptime("%s %s %s" % (d,m,y), "%d %b %Y")
d, m, y = input.strip().split(" ")
m = m[0:3].lower()
return datetime.datetime.strptime("%s %s %s" % (d, m, y), "%d %b %Y")
for url in URLS:
with urllib.request.urlopen(url, data=None, timeout=5) as response:
soup = BeautifulSoup(response, features="html5lib")
table = soup.find(id='tableWraper')
for tr in reversed(table.findAll('tr')[1:]):
td_list = tr.findAll('td')
version_text = td_list[0].get_text()
for key,regexes in CONFIG.items():
for regex in regexes:
matches = re.findall(regex, version_text, re.MULTILINE)
if matches:
for version in matches:
abs_date = None
try:
abs_date = parse_date(td_list[2].get_text())
print_date = abs_date.strftime("%Y-%m-%d")
# Only update the date if we are adding first time
# or if the date is lower
if version not in release_lists[key]:
release_lists[key][version] = abs_date
print("%s-%s: %s" % (key, version, print_date))
elif release_lists[key][version] < abs_date:
print("%s-%s: %s [IGNORED]" % (key, version, print_date))
elif release_lists[key][version] > abs_date:
# This is a lower date, so we mark it with a bang
print("%s-%s: %s [UPDATED]" % (key, version, print_date))
release_lists[key][version] = abs_date
except ValueError as e:
print("%s-%s Failed to parse Date (%s)" % (key, version, td_list[2].get_text()))
next
with urllib.request.urlopen(url, data=None, timeout=5) as response:
soup = BeautifulSoup(response, features="html5lib")
table = soup.find(id="tableWraper")
for tr in reversed(table.findAll("tr")[1:]):
td_list = tr.findAll("td")
version_text = td_list[0].get_text()
for key, regexes in CONFIG.items():
for regex in regexes:
matches = re.findall(regex, version_text, re.MULTILINE)
if matches:
for version in matches:
abs_date = None
try:
abs_date = parse_date(td_list[2].get_text())
print_date = abs_date.strftime("%Y-%m-%d")
# Only update the date if we are adding first time
# or if the date is lower
if version not in release_lists[key]:
release_lists[key][version] = abs_date
print("%s-%s: %s" % (key, version, print_date))
elif release_lists[key][version] < abs_date:
print(
"%s-%s: %s [IGNORED]"
% (key, version, print_date)
)
elif release_lists[key][version] > abs_date:
# This is a lower date, so we mark it with a bang
print(
"%s-%s: %s [UPDATED]"
% (key, version, print_date)
)
release_lists[key][version] = abs_date
except ValueError as e:
print(
"%s-%s Failed to parse Date (%s)"
% (key, version, td_list[2].get_text())
)
next
for k in CONFIG.keys():
with open("releases/%s.json" % k, 'w') as f:
data = {v: d.strftime("%Y-%m-%d") for v,d in release_lists[k].items()}
f.write(json.dumps(data, indent=2))
with open("releases/%s.json" % k, "w") as f:
data = {v: d.strftime("%Y-%m-%d") for v, d in release_lists[k].items()}
f.write(json.dumps(data, indent=2))
print("::endgroup::")

View File

@@ -10,19 +10,23 @@ from html.parser import HTMLParser
from liquid import Template
# Same as used in Ruby (update.rb)
DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
DEFAULT_TAG_TEMPLATE = (
"{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
)
def get_versions_from_headline(regex, headline, template):
if not isinstance(regex, list):
regex = [regex]
for r in regex:
matches = re.match(r.strip(), headline)
if matches:
match_data = matches.groupdict()
version_string = template.render(**match_data)
return version_string.split("\n")
if not isinstance(regex, list):
regex = [regex]
for r in regex:
matches = re.match(r.strip(), headline)
if matches:
match_data = matches.groupdict()
version_string = template.render(**match_data)
return version_string.split("\n")
return {}
return {}
def fetch_releases(distrowatch_id, regex, template):
releases = {}
@@ -34,34 +38,37 @@ def fetch_releases(distrowatch_id, regex, template):
headline = table.select_one("td.NewsHeadline a[href]").get_text().strip()
date = table.select_one("td.NewsDate").get_text()
for v in get_versions_from_headline(regex, headline, l_template):
print("%s: %s" % (v, date))
releases[v] = date
print("%s: %s" % (v, date))
releases[v] = date
return releases
def update_releases(product_filter=None):
for product_file in glob("website/products/*.md"):
product_name = os.path.splitext(os.path.basename(product_file))[0]
if product_filter and product_name != product_filter:
continue
with open(product_file, "r") as f:
data = frontmatter.load(f)
if "auto" in data:
for config in data["auto"]:
for key, d_id in config.items():
if key == "distrowatch":
update_product(product_name, config)
for product_file in glob("website/products/*.md"):
product_name = os.path.splitext(os.path.basename(product_file))[0]
if product_filter and product_name != product_filter:
continue
with open(product_file, "r") as f:
data = frontmatter.load(f)
if "auto" in data:
for config in data["auto"]:
for key, d_id in config.items():
if key == "distrowatch":
update_product(product_name, config)
def update_product(product_name, config):
t = config.get("template", DEFAULT_TAG_TEMPLATE)
if "regex" in config:
print("::group::%s" % product_name)
r = fetch_releases(config['distrowatch'], config["regex"], t)
with open("releases/%s.json" % product_name, "w") as f:
f.write(json.dumps(r, indent=2))
print("::endgroup::")
t = config.get("template", DEFAULT_TAG_TEMPLATE)
if "regex" in config:
print("::group::%s" % product_name)
r = fetch_releases(config["distrowatch"], config["regex"], t)
with open("releases/%s.json" % product_name, "w") as f:
f.write(json.dumps(r, indent=2))
print("::endgroup::")
if __name__ == "__main__":
if len(sys.argv) > 1:
update_releases(sys.argv[1])
else:
update_releases()
if len(sys.argv) > 1:
update_releases(sys.argv[1])
else:
update_releases()

View File

@@ -3,19 +3,19 @@ import re
import urllib.request
# https://regex101.com/r/1JCnFC/1
REGEX = r'^(\d{4})\/(\d{2})\/(\d{2})\s+:\s+(\d+\.\d+\.\d.?)$'
REGEX = r"^(\d{4})\/(\d{2})\/(\d{2})\s+:\s+(\d+\.\d+\.\d.?)$"
list = {}
for i in range(17, 27):
url = "https://www.haproxy.org/download/%s/src/CHANGELOG" % (i/10)
with urllib.request.urlopen(url) as response:
for line in response:
m = re.match(REGEX, line.decode('utf-8'))
if m:
year,month,date,version = m.groups()
abs_date = "%s-%s-%s" % (year, month, date)
list[version] = abs_date
url = "https://www.haproxy.org/download/%s/src/CHANGELOG" % (i / 10)
with urllib.request.urlopen(url) as response:
for line in response:
m = re.match(REGEX, line.decode("utf-8"))
if m:
year, month, date, version = m.groups()
abs_date = "%s-%s-%s" % (year, month, date)
list[version] = abs_date
with open('releases/haproxy.json', 'w') as f:
f.write(json.dumps(list, indent=2))
with open("releases/haproxy.json", "w") as f:
f.write(json.dumps(list, indent=2))

View File

@@ -6,48 +6,58 @@ import frontmatter
import urllib.request
import datetime
def fetch_releases(package_identifier):
groupId, artifactId = package_identifier.split("/")
releases = {}
start = 0
while True:
url = "https://search.maven.org/solrsearch/select?q=g:%s+AND+a:%s&core=gav&rows=100&wt=json&start=%s" % (groupId, artifactId, start)
with urllib.request.urlopen(url, data=None, timeout=5) as response:
data = json.load(response)
for row in data['response']['docs']:
date = datetime.datetime.utcfromtimestamp(row['timestamp'] / 1000)
version = row['v']
if not any(exception in version for exception in ['alpha', 'beta', 'nightly', 'rc', '-M']):
abs_date = date.strftime("%Y-%m-%d")
releases[version] = abs_date
print("%s: %s" % (version, abs_date))
start+=100
if data['response']['numFound'] <= start:
break
url = (
"https://search.maven.org/solrsearch/select?q=g:%s+AND+a:%s&core=gav&rows=100&wt=json&start=%s"
% (groupId, artifactId, start)
)
with urllib.request.urlopen(url, data=None, timeout=5) as response:
data = json.load(response)
for row in data["response"]["docs"]:
date = datetime.datetime.utcfromtimestamp(row["timestamp"] / 1000)
version = row["v"]
if not any(
exception in version
for exception in ["alpha", "beta", "nightly", "rc", "-M"]
):
abs_date = date.strftime("%Y-%m-%d")
releases[version] = abs_date
print("%s: %s" % (version, abs_date))
start += 100
if data["response"]["numFound"] <= start:
break
return releases
def update_releases(product_filter=None):
for product_file in glob("website/products/*.md"):
product_name = os.path.splitext(os.path.basename(product_file))[0]
if product_filter and product_name != product_filter:
continue
with open(product_file, "r") as f:
data = frontmatter.load(f)
if "auto" in data:
for config in data["auto"]:
for key, _ in config.items():
if key == "maven":
update_product(product_name, config)
for product_file in glob("website/products/*.md"):
product_name = os.path.splitext(os.path.basename(product_file))[0]
if product_filter and product_name != product_filter:
continue
with open(product_file, "r") as f:
data = frontmatter.load(f)
if "auto" in data:
for config in data["auto"]:
for key, _ in config.items():
if key == "maven":
update_product(product_name, config)
def update_product(product_name, config):
print("::group::%s" % product_name)
r = fetch_releases(config['maven'])
r = fetch_releases(config["maven"])
with open("releases/%s.json" % product_name, "w") as f:
f.write(json.dumps(r, indent=2))
print("::endgroup::")
if __name__ == "__main__":
if len(sys.argv) > 1:
update_releases(sys.argv[1])
else:
update_releases()
if len(sys.argv) > 1:
update_releases(sys.argv[1])
else:
update_releases()

View File

@@ -6,36 +6,40 @@ from bs4 import BeautifulSoup
URL = "https://www.paloaltonetworks.com/services/support/end-of-life-announcements/end-of-life-summary"
ID_MAPPING = {
"pan-os-panorama": "pan-os",
"globalprotect": "pan-gp",
"traps-esm-and-cortex": "pan-xdr"
"pan-os-panorama": "pan-os",
"globalprotect": "pan-gp",
"traps-esm-and-cortex": "pan-xdr",
}
def update_releases(html_id, file):
list = {}
with urllib.request.urlopen(URL, data=None, timeout=5) as response:
soup = BeautifulSoup(response, features="html5lib")
table = soup.find(id=html_id)
for tr in table.findAll('tr')[3:]:
td_list = tr.findAll('td')
version = td_list[0].get_text().strip().lower().replace(" ", "-").replace("*","")
if file == 'pan-xdr':
if "xdr" not in version:
continue
version = version.removesuffix("-(cortex-xdr-agent)")
version = version.removesuffix("-(vm-series-only)")
version = version.removesuffix("-(panorama-only)")
try:
month,date,year = td_list[1].get_text().split('/')
abs_date = f"{year}-{month:0>2}-{date:0>2}"
except Exception as e:
date = datetime.datetime.strptime(td_list[1].get_text(), "%B %d, %Y")
abs_date = date.strftime("%Y-%m-%d")
list = {}
with urllib.request.urlopen(URL, data=None, timeout=5) as response:
soup = BeautifulSoup(response, features="html5lib")
table = soup.find(id=html_id)
for tr in table.findAll("tr")[3:]:
td_list = tr.findAll("td")
version = (
td_list[0].get_text().strip().lower().replace(" ", "-").replace("*", "")
)
if file == "pan-xdr":
if "xdr" not in version:
continue
version = version.removesuffix("-(cortex-xdr-agent)")
version = version.removesuffix("-(vm-series-only)")
version = version.removesuffix("-(panorama-only)")
try:
month, date, year = td_list[1].get_text().split("/")
abs_date = f"{year}-{month:0>2}-{date:0>2}"
except Exception as e:
date = datetime.datetime.strptime(td_list[1].get_text(), "%B %d, %Y")
abs_date = date.strftime("%Y-%m-%d")
list[version] = abs_date
list[version] = abs_date
with open("releases/%s.json" % file, "w") as f:
f.write(json.dumps(list, indent=2))
with open('releases/%s.json' % file, 'w') as f:
f.write(json.dumps(list, indent=2))
for html_id in ID_MAPPING:
update_releases(html_id, ID_MAPPING[html_id])
update_releases(html_id, ID_MAPPING[html_id])

View File

@@ -11,55 +11,61 @@ from html.parser import HTMLParser
from liquid import Template
# Same as used in Ruby (update.rb)
DEFAULT_TAG_TEMPLATE = "{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
REGEX = r'^(?:(\d+\.(?:\d+\.)*\d+))$'
DEFAULT_TAG_TEMPLATE = (
"{{major}}{% if minor %}.{{minor}}{% if patch %}.{{patch}}{%endif%}{%endif%}"
)
REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
def fetch_releases(pypi_id, regex):
releases = {}
if not isinstance(regex, list):
regex = [regex]
regex = [regex]
url = "https://pypi.org/pypi/%s/json" % pypi_id
with urllib.request.urlopen(url, data=None, timeout=5) as response:
data = json.loads(response.read().decode("utf-8"))
for version in data['releases']:
R = data['releases'][version]
matches = False
for r in regex:
if re.match(r, version):
matches = True
if matches:
d = datetime.fromisoformat(R[0]['upload_time']).strftime("%Y-%m-%d")
releases[version] = d
print("%s: %s" % (version, d))
data = json.loads(response.read().decode("utf-8"))
for version in data["releases"]:
R = data["releases"][version]
matches = False
for r in regex:
if re.match(r, version):
matches = True
if matches:
d = datetime.fromisoformat(R[0]["upload_time"]).strftime("%Y-%m-%d")
releases[version] = d
print("%s: %s" % (version, d))
return releases
def update_releases(product_filter=None):
for product_file in glob("website/products/*.md"):
product_name = os.path.splitext(os.path.basename(product_file))[0]
if product_filter and product_name != product_filter:
continue
with open(product_file, "r") as f:
data = frontmatter.load(f)
if "auto" in data:
for config in data["auto"]:
for key, d_id in config.items():
if key == "pypi":
update_product(product_name, config)
for product_file in glob("website/products/*.md"):
product_name = os.path.splitext(os.path.basename(product_file))[0]
if product_filter and product_name != product_filter:
continue
with open(product_file, "r") as f:
data = frontmatter.load(f)
if "auto" in data:
for config in data["auto"]:
for key, d_id in config.items():
if key == "pypi":
update_product(product_name, config)
def update_product(product_name, config):
if "pypi" in config:
print("::group::%s" % product_name)
config = config | {"regex": REGEX}
r = fetch_releases(config['pypi'], config["regex"])
with open("releases/%s.json" % product_name, "w") as f:
f.write(json.dumps(r, indent=2))
print("::endgroup::")
if "pypi" in config:
print("::group::%s" % product_name)
config = config | {"regex": REGEX}
r = fetch_releases(config["pypi"], config["regex"])
with open("releases/%s.json" % product_name, "w") as f:
f.write(json.dumps(r, indent=2))
print("::endgroup::")
if __name__ == "__main__":
if len(sys.argv) > 1:
update_releases(sys.argv[1])
else:
update_releases()
if len(sys.argv) > 1:
update_releases(sys.argv[1])
else:
update_releases()

View File

@@ -8,24 +8,22 @@ URL = "https://access.redhat.com/articles/3078"
regex = r"RHEL (?P<major>\d)(\. ?(?P<minor>\d+))?(( Update (?P<minor2>\d))| GA)?"
list = {}
headers = {
'user-agent': 'mozilla'
}
req = urllib.request.Request(URL,headers=headers)
headers = {"user-agent": "mozilla"}
req = urllib.request.Request(URL, headers=headers)
with urllib.request.urlopen(req, timeout=5) as response:
soup = BeautifulSoup(response, features="html5lib")
for tr in soup.findAll('tr'):
td_list = tr.findAll('td')
if len(td_list)>0:
version = td_list[0].get_text()
m = re.match(regex, version.strip()).groupdict()
version = m['major']
if m['minor']:
version += ".%s" % m['minor']
if m['minor2']:
version += ".%s" % m['minor2']
list[version] = td_list[1].get_text()
soup = BeautifulSoup(response, features="html5lib")
for tr in soup.findAll("tr"):
td_list = tr.findAll("td")
if len(td_list) > 0:
version = td_list[0].get_text()
m = re.match(regex, version.strip()).groupdict()
version = m["major"]
if m["minor"]:
version += ".%s" % m["minor"]
if m["minor2"]:
version += ".%s" % m["minor2"]
list[version] = td_list[1].get_text()
with open('releases/redhat.json', 'w') as f:
f.write(json.dumps(list, indent=2))
with open("releases/redhat.json", "w") as f:
f.write(json.dumps(list, indent=2))

View File

@@ -11,24 +11,26 @@ regex = r"^ROS (?P<name>(\w| )+)"
list = {}
with urllib.request.urlopen(URL, timeout=5) as response:
soup = BeautifulSoup(response, features="html5lib")
for tr in soup.findAll('tr'):
td_list = tr.findAll('td')
if len(td_list)>0:
version = td_list[0].get_text()
soup = BeautifulSoup(response, features="html5lib")
for tr in soup.findAll("tr"):
td_list = tr.findAll("td")
if len(td_list) > 0:
version = td_list[0].get_text()
m = re.match(regex, version.strip())
if m:
version = td_list[0].findAll('a')[0]['href'][1:]
try:
date = datetime.datetime.strptime(td_list[1].get_text().strip(), "%B %d, %Y")
# The date is a suffix (May 23rd, 2020)
except Exception as e:
x = td_list[1].get_text().split(',')
date = datetime.datetime.strptime(x[0][:-2] + x[1], "%B %d %Y")
abs_date = date.strftime("%Y-%m-%d")
list[version] = abs_date
print("%s: %s" % (version, abs_date))
m = re.match(regex, version.strip())
if m:
version = td_list[0].findAll("a")[0]["href"][1:]
try:
date = datetime.datetime.strptime(
td_list[1].get_text().strip(), "%B %d, %Y"
)
# The date is a suffix (May 23rd, 2020)
except Exception as e:
x = td_list[1].get_text().split(",")
date = datetime.datetime.strptime(x[0][:-2] + x[1], "%B %d %Y")
abs_date = date.strftime("%Y-%m-%d")
list[version] = abs_date
print("%s: %s" % (version, abs_date))
with open('releases/ros.json', 'w') as f:
f.write(json.dumps(list, indent=2))
with open("releases/ros.json", "w") as f:
f.write(json.dumps(list, indent=2))

View File

@@ -4,21 +4,21 @@ import re
import urllib.request
URL = "https://www.unrealircd.org/docwiki/index.php?title=History_of_UnrealIRCd_releases&action=raw"
REGEX = r'^(?:(\d+\.(?:\d+\.)*\d+))$'
REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
list = {}
with urllib.request.urlopen(URL) as response:
text = response.read()
wikicode = mwparserfromhell.parse(text)
for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"):
items = tr.contents.filter_tags(matches=lambda node: node.tag == "td")
if len(items) >=2:
maybe_version = items[0].__strip__()
if re.match(REGEX, maybe_version):
maybe_date = items[1].__strip__()
if re.match(r'\d{4}-\d{2}-\d{2}', maybe_date):
list[maybe_version] = maybe_date
text = response.read()
wikicode = mwparserfromhell.parse(text)
for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"):
items = tr.contents.filter_tags(matches=lambda node: node.tag == "td")
if len(items) >= 2:
maybe_version = items[0].__strip__()
if re.match(REGEX, maybe_version):
maybe_date = items[1].__strip__()
if re.match(r"\d{4}-\d{2}-\d{2}", maybe_date):
list[maybe_version] = maybe_date
with open('releases/unrealircd.json', 'w') as f:
f.write(json.dumps(list, indent=2))
with open("releases/unrealircd.json", "w") as f:
f.write(json.dumps(list, indent=2))