Simplify date parsing (#195)

Create common functions parse_date, parse_month_year_date and parse_datetime.

Those functions support trying multiple formats, and come with default formats lists that support most of the date format encountered so far.

Notable change: year-month dates are now set to the end of month (impacted couchbase-server and ibm-aix).
This commit is contained in:
Marc Wrobel
2023-11-26 21:01:35 +01:00
committed by GitHub
parent 1e65a048b0
commit 0d17306872
24 changed files with 133 additions and 168 deletions

View File

@@ -1,22 +1,22 @@
{ {
"7.2.3": "2023-11-15", "7.2.3": "2023-11-30",
"7.1.6": "2023-11-15", "7.1.6": "2023-11-30",
"7.2.2": "2023-09-15", "7.2.2": "2023-09-30",
"7.2.1": "2023-09-15", "7.2.1": "2023-09-30",
"7.1.5": "2023-08-15", "7.1.5": "2023-08-31",
"7.2.0": "2023-06-01", "7.2.0": "2023-06-01",
"7.1.4": "2023-03-15", "7.1.4": "2023-03-31",
"7.0.5": "2022-12-15", "7.0.5": "2022-12-31",
"7.1.3": "2022-11-15", "7.1.3": "2022-11-30",
"7.1.2": "2022-10-15", "7.1.2": "2022-10-31",
"7.1.1": "2022-07-15", "7.1.1": "2022-07-31",
"7.0.4": "2022-06-15", "7.0.4": "2022-06-30",
"7.1.0": "2022-05-15", "7.1.0": "2022-05-31",
"6.6.5": "2022-01-15", "6.6.5": "2022-01-31",
"7.0.3": "2021-12-15", "7.0.3": "2021-12-31",
"7.0.2": "2021-10-15", "7.0.2": "2021-10-31",
"7.0.1": "2021-09-15", "7.0.1": "2021-09-30",
"7.0.0": "2021-07-15", "7.0.0": "2021-07-31",
"6.6.0": "2020-08-12", "6.6.0": "2020-08-12",
"6.0.1": "2019-02-15", "6.0.1": "2019-02-15",
"6.0.0": "2018-10-31" "6.0.0": "2018-10-31"

View File

@@ -1,12 +1,12 @@
{ {
"7.3.2": "2023-11-01", "7.3.2": "2023-11-30",
"7.3.1": "2022-12-01", "7.3.1": "2022-12-31",
"7.3.0": "2021-12-01", "7.3.0": "2021-12-31",
"7.2.5": "2020-11-01", "7.2.5": "2020-11-30",
"7.2.4": "2019-11-01", "7.2.4": "2019-11-30",
"7.2.3": "2018-09-01", "7.2.3": "2018-09-30",
"7.2.2": "2017-10-01", "7.2.2": "2017-10-31",
"7.1.5": "2017-10-01", "7.1.5": "2017-10-31",
"7.2.1": "2016-11-01", "7.2.1": "2016-11-30",
"7.2.0": "2015-12-01" "7.2.0": "2015-12-31"
} }

View File

@@ -1,7 +1,7 @@
import re import re
from xml.dom.minidom import parseString from xml.dom.minidom import parseString
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
"""Fetch versions with their dates from the RSS feed of """Fetch versions with their dates from the RSS feed of
https://docs.aws.amazon.com/neptune/latest/userguide/engine-releases.html. https://docs.aws.amazon.com/neptune/latest/userguide/engine-releases.html.
@@ -22,7 +22,7 @@ for item in rss.getElementsByTagName("item"):
matches = re.match(REGEX, title) matches = re.match(REGEX, title)
if matches: if matches:
version = matches['version'] version = matches['version']
date = datetime.strptime(pubDate, "%a, %d %b %Y %H:%M:%S %Z").strftime("%Y-%m-%d") date = dates.parse_datetime(pubDate).strftime("%Y-%m-%d")
versions[version] = date versions[version] = date
print(f"{version}: {date}") print(f"{version}: {date}")

View File

@@ -1,6 +1,6 @@
import re import re
from datetime import datetime
from pathlib import Path from pathlib import Path
from common import dates
from common import endoflife from common import endoflife
from common.git import Git from common.git import Git
@@ -18,13 +18,8 @@ REPO_URL = "https://github.com/apache/httpd.git"
def parse(date: str) -> str: def parse(date: str) -> str:
date = date.replace("Feburary", "February") date = date.replace("Feburary", "February")
for format in ["%B %d, %Y", "%B %d, %Y", "%b %d, %Y", "%b. %d, %Y"]: date = date.replace(". ", " ")
try: return dates.parse_date(date).strftime("%Y-%m-%d")
return datetime.strptime(date, format).strftime("%Y-%m-%d")
except ValueError:
pass
raise ValueError(f"Unknown date format for '{date}'")
def fetch_versions_from_file(release_notes_file: Path, versions: dict): def fetch_versions_from_file(release_notes_file: Path, versions: dict):

View File

@@ -1,6 +1,6 @@
import datetime
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
URLS = [ URLS = [
@@ -46,10 +46,9 @@ CONFIG = {
} }
def parse_date(s): def parse_date(date_str):
d, m, y = s.strip().split(" ") date_str = date_str.replace("Sept", "Sep")
m = m[0:3].lower() # reduce months to 3 letters, such as "Sept" to "Sep", so it can be parsed return dates.parse_date(date_str)
return datetime.datetime.strptime(f"{d} {m} {y}", "%d %b %Y")
print("::group::apple") print("::group::apple")

View File

@@ -1,22 +1,14 @@
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
from requests_html import HTMLSession from requests_html import HTMLSession
URL = "https://jfrog.com/help/r/jfrog-release-information/artifactory-end-of-life" URL = "https://jfrog.com/help/r/jfrog-release-information/artifactory-end-of-life"
PRODUCT = "artifactory" PRODUCT = "artifactory"
def parse_date(text): def parse_date(date_str):
text = text.replace("Sept", "Sep").replace("_", "-") date_str = date_str.replace("Sept", "Sep").replace("_", "-")
date_formats = ['%d-%b-%Y', '%d-%B-%Y'] return dates.parse_date(date_str).strftime("%Y-%m-%d")
for date_format in date_formats:
try:
return datetime.strptime(text, date_format).strftime("%Y-%m-%d")
except ValueError:
pass
raise ValueError("Cannot parse '" + text + "' with formats " + str(date_formats))
def fetch_releases(): def fetch_releases():

View File

@@ -1,8 +1,8 @@
import re import re
import sys import sys
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime, timezone
from liquid import Template from liquid import Template
"""Fetch versions with their dates from a cgit repository, such as """Fetch versions with their dates from a cgit repository, such as
@@ -22,15 +22,6 @@ DEFAULT_VERSION_REGEX = (
) )
# Parse date with format 2023-05-01 08:32:34 +0900 and convert to UTC
def parse_date(d):
return (
datetime.strptime(d, "%Y-%m-%d %H:%M:%S %z")
.astimezone(timezone.utc)
.strftime("%Y-%m-%d")
)
def make_bs_request(url): def make_bs_request(url):
response = endoflife.fetch_url(url + '/refs/tags') response = endoflife.fetch_url(url + '/refs/tags')
return BeautifulSoup(response, features="html5lib") return BeautifulSoup(response, features="html5lib")
@@ -54,7 +45,7 @@ def fetch_releases(url, regex, template):
if matches: if matches:
match_data = matches.groupdict() match_data = matches.groupdict()
version_string = l_template.render(**match_data) version_string = l_template.render(**match_data)
date = parse_date(datetime_text) date = dates.parse_datetime(datetime_text).strftime("%Y-%m-%d")
print(f"{version_string} : {date}") print(f"{version_string} : {date}")
releases[version_string] = date releases[version_string] = date

51
src/common/dates.py Normal file
View File

@@ -0,0 +1,51 @@
from datetime import datetime, timezone
import calendar
def parse_date(text, formats=frozenset([
"%B %d, %Y", # January 1, 2020
"%b %d, %Y", # Jan 1, 2020
"%B %d %Y", # January 1 2020
"%b %d %Y", # Jan 1 2020
"%d %B %Y", # 1 January 2020
"%d %b %Y", # 1 Jan 2020
"%d-%b-%Y", # 1-Jan-2020
"%d-%B-%Y", # 1-January-2020
])) -> datetime:
"""Parse a given text representing a date using a list of formats.
"""
return parse_datetime(text, formats, to_utc=False)
def parse_month_year_date(text, formats=frozenset([
"%B %Y", # January 2020
"%b %Y", # Jan 2020
])) -> datetime:
"""Parse a given text representing a partial date using a list of formats,
adjusting it to the last day of the month.
"""
date = parse_datetime(text, formats, to_utc=False)
_, last_day = calendar.monthrange(date.year, date.month)
return date.replace(day=last_day)
def parse_datetime(text, formats=frozenset([
"%Y-%m-%d %H:%M:%S", # 2023-05-01 08:32:34
"%Y-%m-%dT%H:%M:%S", # 2023-05-01T08:32:34
"%Y-%m-%d %H:%M:%S %z", # 2023-05-01 08:32:34 +0900
"%a, %d %b %Y %H:%M:%S %Z", # Wed, 01 Jan 2020 00:00:00 GMT
"%Y-%m-%dT%H:%M:%S%z", # 2023-05-01T08:32:34+0900
]), to_utc=True) -> datetime:
"""Parse a given text representing a datetime using a list of formats,
optionally converting it to UTC.
"""
text = text.strip()
for fmt in formats:
try:
date = datetime.strptime(text, fmt)
date = date.astimezone(timezone.utc) if to_utc else date
return date
except ValueError:
pass
raise ValueError(f"'{text}' could not be parsed as a date with any of the formats: {str(formats)}")

View File

@@ -1,6 +1,6 @@
from requests_html import HTMLSession from requests_html import HTMLSession
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
"""Fetch Confluence versions with their dates from the Atlassian Website. """Fetch Confluence versions with their dates from the Atlassian Website.
@@ -11,13 +11,7 @@ because the page needs JavaScript to render correctly.
PRODUCT = 'confluence' PRODUCT = 'confluence'
URL = 'https://www.atlassian.com/software/confluence/download-archives' URL = 'https://www.atlassian.com/software/confluence/download-archives'
def parse_date(text):
return datetime.strptime(text, "%d-%b-%Y").strftime("%Y-%m-%d")
print(f"::group::{PRODUCT}") print(f"::group::{PRODUCT}")
session = HTMLSession() session = HTMLSession()
r = session.get(URL) r = session.get(URL)
r.html.render(sleep=1, scrolldown=3) r.html.render(sleep=1, scrolldown=3)
@@ -25,7 +19,8 @@ r.html.render(sleep=1, scrolldown=3)
versions = {} versions = {}
for version_block in r.html.find('.versions-list'): for version_block in r.html.find('.versions-list'):
version = version_block.find('a.product-versions', first=True).attrs['data-version'] version = version_block.find('a.product-versions', first=True).attrs['data-version']
date = parse_date(version_block.find('.release-date', first=True).text) date_text = version_block.find('.release-date', first=True).text
date = dates.parse_date(date_text).strftime('%Y-%m-%d')
print(f"{version}: {date}") print(f"{version}: {date}")
versions[version] = date versions[version] = date

View File

@@ -1,10 +1,9 @@
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
URL = "https://cloud.google.com/container-optimized-os/docs/release-notes/" URL = "https://cloud.google.com/container-optimized-os/docs/release-notes/"
DATE_FORMAT = '%b %d, %Y'
REGEX = r"^(cos-\d+-\d+-\d+-\d+)" REGEX = r"^(cos-\d+-\d+-\d+-\d+)"
@@ -20,11 +19,10 @@ def fetch_milestones(milestones):
return endoflife.fetch_urls(urls) return endoflife.fetch_urls(urls)
def parse_date(d): def parse_date(date_str):
# If the date begins with a >3 letter month name, trim it to just 3 letters date_str = date_str.strip().replace('Date: ', '')
# Strip out the Date: section from the start date_str = re.sub(r'Sep[a-zA-Z]+', 'Sep', date_str)
d = re.sub(r'(?:Date\: )?(\w{3})(?:\w{1,})? (\d{1,2}), (\d{4})', r'\1 \2, \3', d) return dates.parse_date(date_str).strftime('%Y-%m-%d')
return datetime.strptime(d, DATE_FORMAT).strftime('%Y-%m-%d')
def find_versions(text): def find_versions(text):

View File

@@ -1,7 +1,7 @@
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
"""Fetch versions with their dates from docs.couchbase.com. """Fetch versions with their dates from docs.couchbase.com.
@@ -46,7 +46,7 @@ for response in endoflife.fetch_urls(minor_version_urls):
m = re.match(REGEX, versionAndDate) m = re.match(REGEX, versionAndDate)
if m: if m:
version = f"{m['version']}.0" if len(m['version'].split('.')) == 2 else m['version'] version = f"{m['version']}.0" if len(m['version'].split('.')) == 2 else m['version']
date = datetime.strptime(m['date'], "%B %Y").strftime("%Y-%m-15") date = dates.parse_month_year_date(m['date']).strftime("%Y-%m-%d")
versions[version] = date versions[version] = date
print(f"{version}: {date}") print(f"{version}: {date}")

View File

@@ -1,8 +1,8 @@
import re import re
import urllib.parse import urllib.parse
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
"""Fetch Firefox versions with their dates from https://www.mozilla.org/""" """Fetch Firefox versions with their dates from https://www.mozilla.org/"""
@@ -12,13 +12,7 @@ PRODUCT = "firefox"
def format_date(text: str) -> str: def format_date(text: str) -> str:
text = text.replace(')', '') text = text.replace(')', '')
formats = ["%b %d, %Y", "%B %d, %Y"] return dates.parse_date(text).strftime("%Y-%m-%d")
for f in formats:
try:
return datetime.strptime(text, f).strftime("%Y-%m-%d")
except ValueError:
pass
return ""
print(f"::group::{PRODUCT}") print(f"::group::{PRODUCT}")

View File

@@ -1,7 +1,7 @@
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
# https://regex101.com/r/zPxBqT/1 # https://regex101.com/r/zPxBqT/1
REGEX = r"\d.\d+\.\d+-gke\.\d+" REGEX = r"\d.\d+\.\d+-gke\.\d+"
@@ -22,7 +22,7 @@ def parse_soup_for_versions(soup):
# h2 contains the date, which we parse # h2 contains the date, which we parse
for h2 in section.find_all('h2'): for h2 in section.find_all('h2'):
date = h2.get('data-text') date = h2.get('data-text')
date = datetime.strptime(date, '%B %d, %Y').strftime('%Y-%m-%d') date = dates.parse_date(date).strftime("%Y-%m-%d")
# The div next to the h2 contains the notes about changes made # The div next to the h2 contains the notes about changes made
# on that date # on that date
next_div = h2.find_next('div') next_div = h2.find_next('div')

View File

@@ -1,14 +1,11 @@
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
URL = "https://www.graalvm.org/release-calendar/" URL = "https://www.graalvm.org/release-calendar/"
# https://regex101.com/r/877ibq/1 # https://regex101.com/r/877ibq/1
regex = r"RHEL (?P<major>\d)(\. ?(?P<minor>\d+))?(( Update (?P<minor2>\d))| GA)?" regex = r"RHEL (?P<major>\d)(\. ?(?P<minor>\d+))?(( Update (?P<minor2>\d))| GA)?"
def parse_date(text):
return datetime.strptime(text, "%B %d, %Y").strftime("%Y-%m-%d")
def split_versions(text): def split_versions(text):
# GraalVM for JDK versions has to be prefixed as their release cycle collide # GraalVM for JDK versions has to be prefixed as their release cycle collide
# with older GraalVM release cycles. Example: GraalVM for JDK 20 and 20.0. # with older GraalVM release cycles. Example: GraalVM for JDK 20 and 20.0.
@@ -21,7 +18,7 @@ soup = BeautifulSoup(response, features="html5lib")
versions = {} versions = {}
for tr in soup.findAll("table")[1].find("tbody").findAll("tr"): for tr in soup.findAll("table")[1].find("tbody").findAll("tr"):
td_list = tr.findAll("td") td_list = tr.findAll("td")
date = parse_date(td_list[0].get_text()) date = dates.parse_date(td_list[0].get_text()).strftime("%Y-%m-%d")
for version in split_versions(td_list[2].get_text()): for version in split_versions(td_list[2].get_text()):
versions[version] = date versions[version] = date

View File

@@ -1,17 +1,10 @@
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
PRODUCT = "ibm-aix" PRODUCT = "ibm-aix"
URL = "https://www.ibm.com/support/pages/aix-support-lifecycle-information" URL = "https://www.ibm.com/support/pages/aix-support-lifecycle-information"
# Convert date from e.g. "November 2022" format to "2022-11-01"
def convert_date(date_str):
return datetime.strptime(date_str, "%B %Y").strftime("%Y-%m-%d")
def fetch_releases(): def fetch_releases():
response = endoflife.fetch_url(URL) response = endoflife.fetch_url(URL)
soup = BeautifulSoup(response, features="html5lib") soup = BeautifulSoup(response, features="html5lib")
@@ -23,7 +16,7 @@ def fetch_releases():
for row in release_table.find_all("tr")[1:]: for row in release_table.find_all("tr")[1:]:
cells = row.find_all("td") cells = row.find_all("td")
version = cells[0].text.strip("AIX ").replace(' TL', '.') version = cells[0].text.strip("AIX ").replace(' TL', '.')
date = convert_date(cells[1].text) date = dates.parse_month_year_date(cells[1].text).strftime("%Y-%m-%d")
print(f"{version} : {date}") print(f"{version} : {date}")
releases[version] = date releases[version] = date

View File

@@ -1,6 +1,6 @@
from requests_html import HTMLSession from requests_html import HTMLSession
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
"""Fetch Jira versions with their dates from the Atlassian Website. """Fetch Jira versions with their dates from the Atlassian Website.
@@ -11,13 +11,7 @@ because the page needs JavaScript to render correctly.
PRODUCT = 'jira' PRODUCT = 'jira'
URL = 'https://www.atlassian.com/software/jira/update' URL = 'https://www.atlassian.com/software/jira/update'
def parse_date(text):
return datetime.strptime(text, "%d-%b-%Y").strftime("%Y-%m-%d")
print(f"::group::{PRODUCT}") print(f"::group::{PRODUCT}")
session = HTMLSession() session = HTMLSession()
r = session.get(URL) r = session.get(URL)
r.html.render(sleep=1, scrolldown=3) r.html.render(sleep=1, scrolldown=3)
@@ -25,7 +19,8 @@ r.html.render(sleep=1, scrolldown=3)
versions = {} versions = {}
for version_block in r.html.find('.versions-list'): for version_block in r.html.find('.versions-list'):
version = version_block.find('a.product-versions', first=True).attrs['data-version'] version = version_block.find('a.product-versions', first=True).attrs['data-version']
date = parse_date(version_block.find('.release-date', first=True).text) date_text = version_block.find('.release-date', first=True).text
date = dates.parse_date(date_text).strftime('%Y-%m-%d')
print(f"{version}: {date}") print(f"{version}: {date}")
versions[version] = date versions[version] = date

View File

@@ -1,8 +1,7 @@
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime, timezone
from xml.dom.minidom import parseString from xml.dom.minidom import parseString
"""Fetch Looker versions with their dates from the Google Cloud release notes RSS feed. """Fetch Looker versions with their dates from the Google Cloud release notes RSS feed.
@@ -14,16 +13,13 @@ ANNOUNCEMENT_PATTERN = re.compile(r"includes\s+the\s+following\s+changes", re.IG
VERSION_PATTERN = re.compile(r"Looker\s+(?P<version>\d+\.\d+)", re.IGNORECASE) VERSION_PATTERN = re.compile(r"Looker\s+(?P<version>\d+\.\d+)", re.IGNORECASE)
def parse_date(date_str):
return datetime.fromisoformat(date_str).astimezone(timezone.utc).strftime("%Y-%m-%d")
print(f"::group::{PRODUCT}") print(f"::group::{PRODUCT}")
versions = {} versions = {}
response = endoflife.fetch_url(URL) response = endoflife.fetch_url(URL)
rss = parseString(response) rss = parseString(response)
for item in rss.getElementsByTagName("entry"): for item in rss.getElementsByTagName("entry"):
date = parse_date(item.getElementsByTagName("updated")[0].firstChild.nodeValue) date = dates.parse_datetime(item.getElementsByTagName("updated")[0].firstChild.nodeValue).strftime("%Y-%m-%d")
content = item.getElementsByTagName("content")[0].firstChild.nodeValue content = item.getElementsByTagName("content")[0].firstChild.nodeValue
soup = BeautifulSoup(content, features="html5lib") soup = BeautifulSoup(content, features="html5lib")

View File

@@ -1,26 +1,16 @@
import json import json
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
PHP_MAJOR_VERSIONS = [4, 5, 7, 8] PHP_MAJOR_VERSIONS = [4, 5, 7, 8]
# Date format is 03 Nov 2022
# With some versions using 03 November 2022 instead
# we return it as YYYY-MM-DD
def parse_date(date_str):
try:
return datetime.strptime(date_str, "%d %b %Y").strftime("%Y-%m-%d")
except ValueError:
return datetime.strptime(date_str, "%d %B %Y").strftime("%Y-%m-%d")
def fetch_versions(major_version): def fetch_versions(major_version):
url = f"https://www.php.net/releases/index.php?json&max=-1&version={major_version}" url = f"https://www.php.net/releases/index.php?json&max=-1&version={major_version}"
response = endoflife.fetch_url(url) response = endoflife.fetch_url(url)
data = json.loads(response) data = json.loads(response)
for v in data: for v in data:
data[v] = parse_date(data[v]["date"]) data[v] = dates.parse_date(data[v]["date"]).strftime("%Y-%m-%d")
print(f"{v}: {data[v]}") print(f"{v}: {data[v]}")
return data return data

View File

@@ -1,6 +1,6 @@
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
URL = "https://docs.plesk.com/release-notes/obsidian/change-log" URL = "https://docs.plesk.com/release-notes/obsidian/change-log"
PRODUCT = "plesk" PRODUCT = "plesk"
@@ -27,7 +27,7 @@ def fetch_releases():
version = version.replace(' Update ', '.').replace('Plesk Obsidian ', '') version = version.replace(' Update ', '.').replace('Plesk Obsidian ', '')
if ' ' in version: if ' ' in version:
continue continue
date = datetime.strptime(release.p.text.strip(), '%d %B %Y').strftime("%Y-%m-%d") date = dates.parse_date(release.p.text).strftime("%Y-%m-%d")
result[version] = date result[version] = date
print(f"{version}: {date}") print(f"{version}: {date}")

View File

@@ -1,8 +1,8 @@
import json import json
import re import re
import sys import sys
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
METHOD = "pypi" METHOD = "pypi"
DEFAULT_TAG_TEMPLATE = ( # Same as used in Ruby (update.rb) DEFAULT_TAG_TEMPLATE = ( # Same as used in Ruby (update.rb)
@@ -27,7 +27,7 @@ def fetch_releases(pypi_id, regex):
if re.match(r, version): if re.match(r, version):
matches = True matches = True
if matches and R: if matches and R:
d = datetime.fromisoformat(R[0]["upload_time"]).strftime("%Y-%m-%d") d = dates.parse_datetime(R[0]["upload_time"], to_utc=False).strftime("%Y-%m-%d")
releases[version] = d releases[version] = d
print(f"{version}: {d}") print(f"{version}: {d}")

View File

@@ -1,7 +1,7 @@
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
VERSION_REGEX = r"(?P<v>\d+(?:\.\d+)*)" # https://regex101.com/r/BY1vwV/1 VERSION_REGEX = r"(?P<v>\d+(?:\.\d+)*)" # https://regex101.com/r/BY1vwV/1
DBS = { DBS = {
@@ -9,14 +9,6 @@ DBS = {
"postgresql": "https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-release-calendar.html", "postgresql": "https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-release-calendar.html",
} }
def parse_date(d):
try:
return datetime.strptime(d, "%d %B %Y").strftime("%Y-%m-%d")
except ValueError:
return None
for db, url in DBS.items(): for db, url in DBS.items():
print(f"::group::{db}") print(f"::group::{db}")
versions = {} versions = {}
@@ -33,7 +25,7 @@ for db, url in DBS.items():
if len(columns) > 3: if len(columns) > 3:
m = re.search(VERSION_REGEX, columns[0].text.strip(), flags=re.IGNORECASE) m = re.search(VERSION_REGEX, columns[0].text.strip(), flags=re.IGNORECASE)
if m: if m:
date = parse_date(columns[2].text.strip()) date = dates.parse_date(columns[2].text).strftime("%Y-%m-%d")
if date: if date:
version = m.group("v") version = m.group("v")
print(f"{version} : {date}") print(f"{version} : {date}")

View File

@@ -1,16 +1,15 @@
import re import re
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
URL = "https://raw.githubusercontent.com/rocky-linux/wiki.rockylinux.org/development/docs/include/releng/version_table.md" URL = "https://raw.githubusercontent.com/rocky-linux/wiki.rockylinux.org/development/docs/include/releng/version_table.md"
REGEX = r"^(\d+\.\d+)$" REGEX = r"^(\d+\.\d+)$"
def parse_date(date_str): def parse_date(date_str):
date_str = date_str.replace(',', '').strip() date_str = date_str.replace(',', '').strip()
try: return dates.parse_date(date_str).strftime("%Y-%m-%d")
return datetime.strptime(date_str, "%B %d %Y").strftime("%Y-%m-%d")
except ValueError:
return datetime.strptime(date_str, "%b %d %Y").strftime("%Y-%m-%d")
def parse_markdown_table(table_text): def parse_markdown_table(table_text):
lines = table_text.strip().split('\n') lines = table_text.strip().split('\n')
@@ -26,6 +25,7 @@ def parse_markdown_table(table_text):
return versions return versions
print("::group::rockylinux") print("::group::rockylinux")
response = endoflife.fetch_url(URL) response = endoflife.fetch_url(URL)
versions = parse_markdown_table(response) versions = parse_markdown_table(response)

View File

@@ -1,19 +1,10 @@
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
PRODUCT = "sles" PRODUCT = "sles"
URL = "https://www.suse.com/lifecycle" URL = "https://www.suse.com/lifecycle"
DATE_FORMAT = "%d %b %Y"
# Convert date from e.g. "16 Jul 2018" to "2018-07-16"
def convert_date(date_str):
# If the date begins with a >3 letter month name, trim it to just 3 letters
# Strip out the Date: section from the start
d = re.sub(r'(\d{1,2}) (\w{3})(?:\w{1,4})? (\d{4})', r'\1 \2 \3', date_str)
return datetime.strptime(d, DATE_FORMAT).strftime('%Y-%m-%d')
def strip_version(version_str): def strip_version(version_str):
@@ -47,7 +38,7 @@ def fetch_releases():
version = strip_version(cells[0].text) version = strip_version(cells[0].text)
try: try:
release_date = convert_date(cells[1].text) release_date = dates.parse_date(cells[1].text).strftime("%Y-%m-%d")
versions[version] = release_date versions[version] = release_date
print(f"{version}: {release_date}") print(f"{version}: {release_date}")
except ValueError as e: except ValueError as e:

View File

@@ -1,7 +1,7 @@
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common import dates
from common import endoflife from common import endoflife
from datetime import datetime
PRODUCT = "splunk" PRODUCT = "splunk"
URL = "https://docs.splunk.com/Documentation/Splunk" URL = "https://docs.splunk.com/Documentation/Splunk"
@@ -9,10 +9,6 @@ RELNOTES_URL_TEMPLATE = "https://docs.splunk.com/Documentation/Splunk/{version}/
PATTERN = r"Splunk Enterprise (?P<version>\d+\.\d+(?:\.\d+)*) was (?:first )?released on (?P<date>\w+\s\d\d?,\s\d{4})\." PATTERN = r"Splunk Enterprise (?P<version>\d+\.\d+(?:\.\d+)*) was (?:first )?released on (?P<date>\w+\s\d\d?,\s\d{4})\."
def convert_date(date: str) -> str:
return datetime.strptime(date, "%B %d, %Y").strftime("%Y-%m-%d")
def get_latest_minor_versions(versions): def get_latest_minor_versions(versions):
versions_split = [version.split('.') for version in versions] versions_split = [version.split('.') for version in versions]
@@ -55,7 +51,7 @@ latest_minor_versions_urls = [RELNOTES_URL_TEMPLATE.format(version=v) for v in l
for response in endoflife.fetch_urls(latest_minor_versions_urls): for response in endoflife.fetch_urls(latest_minor_versions_urls):
for (version, date_str) in re.findall(PATTERN, response.text, re.MULTILINE): for (version, date_str) in re.findall(PATTERN, response.text, re.MULTILINE):
version = f"{version}.0" if len(version.split(".")) == 2 else version # convert x.y to x.y.0 version = f"{version}.0" if len(version.split(".")) == 2 else version # convert x.y to x.y.0
date = convert_date(date_str) date = dates.parse_date(date_str).strftime("%Y-%m-%d")
versions[version] = date versions[version] = date
print(f"{version}: {date}") print(f"{version}: {date}")