diff --git a/src/apple.py b/src/apple.py index 28d5e375..2f80f2d1 100644 --- a/src/apple.py +++ b/src/apple.py @@ -16,9 +16,9 @@ URLS = [ "https://support.apple.com/kb/HT1263", # 2005-2007 ] -# If you are changing these, please -# use https://gist.githubusercontent.com/captn3m0/e7cb1f4fc3c07a5da0296ebda2b33e15/raw/5747e42ad611ec9ffdb7a2d1c0e3946bb87ab6d7/apple.txt as your corpus -# to validate your changes +# If you are changing these, please use +# https://gist.githubusercontent.com/captn3m0/e7cb1f4fc3c07a5da0296ebda2b33e15/raw/5747e42ad611ec9ffdb7a2d1c0e3946bb87ab6d7/apple.txt +# as your corpus to validate your changes CONFIG = { "macos": [ # This covers Sierra and beyond diff --git a/src/cos.py b/src/cos.py index 9e1eea30..0a14dd65 100644 --- a/src/cos.py +++ b/src/cos.py @@ -3,6 +3,7 @@ from bs4 import BeautifulSoup from common import endoflife from datetime import datetime +DATE_FORMAT = '%b %d, %Y' REGEX = r"^(cos-\d+-\d+-\d+-\d+)" @@ -16,34 +17,35 @@ def fetch_all_milestones(): def fetch_milestone(channel): - url = "https://cloud.google.com/container-optimized-os/docs/release-notes/m{}".format(channel) + url = f"https://cloud.google.com/container-optimized-os/docs/release-notes/m{channel}" # Retry as Google Docs often returns SSL errors. response = endoflife.fetch_url(url, retry_count=10) return BeautifulSoup(response, features="html5lib") +def parse_date(d): + # If the date begins with a >3 letter month name, trim it to just 3 letters + # Strip out the Date: section from the start + d = re.sub(r'(?:Date\: )?(\w{3})(?:\w{1,4})? (\d{1,2}), (\d{4})', r'\1 \2, \3', d) + return datetime.strptime(d, DATE_FORMAT).strftime('%Y-%m-%d') + + def parse_soup_for_versions(soup): """Takes soup, and returns a dictionary of versions and their release dates """ versions = {} for article in soup.find_all('article', class_='devsite-article'): - def parse_date(d): - # If the date begins with a >3 letter month name, trim it to just 3 letters - # Strip out the Date: section from the start - d = re.sub(r'(?:Date\: )?(\w{3})(?:\w{1,4})? (\d{1,2}), (\d{4})', r'\1 \2, \3', d) - return datetime.strptime(d, date_format).strftime('%Y-%m-%d') # h2 contains the date, which we parse for heading in article.find_all(['h2', 'h3']): version = heading.get('data-text') m = re.match(REGEX, version) if m: version = m.group(1) - date_format = '%b %d, %Y' try: - # The first row is the header, so we pick the first td in the second row + # 1st row is the header, so pick the first td in the 2nd row d = heading.find_next('tr').find_next('tr').find_next('td').text - except: - # In some older releases, it is mentioned as Date: [Date] in the text + except AttributeError: + # In some older releases, it is mentioned as Date: [Date] d = heading.find_next('i').text try: date = parse_date(d) @@ -51,7 +53,7 @@ def parse_soup_for_versions(soup): d = heading.find_previous('h2').get('data-text') date = parse_date(d) versions[version] = date - print("%s: %s" % (version, date)) + print(f"{version}: {date}") return versions diff --git a/src/debian.py b/src/debian.py index 5e91471d..57df3c4f 100644 --- a/src/debian.py +++ b/src/debian.py @@ -30,6 +30,7 @@ def clone_repository(): ret_code = call(f"git {git_opts} pull --depth 1 origin master", shell=True) exit(-ret_code) if ret_code < 0 else None + def extract_major_releases(releases): child = subprocess.Popen( f"grep -RhE -A 1 'Debian [0-9]+.+ released' {REPO_DIR}/english/News " @@ -48,8 +49,8 @@ def extract_major_releases(releases): is_release_line = False else: date = line - print(f"{version}: {date}") releases[version] = date + print(f"{version}: {date}") is_release_line = True @@ -75,11 +76,11 @@ def extract_point_releases(releases): print(f"::group::{PRODUCT}") clone_repository() -releases = {} -extract_major_releases(releases) -extract_point_releases(releases) +all_releases = {} +extract_major_releases(all_releases) +extract_point_releases(all_releases) endoflife.write_releases(PRODUCT, dict( # sort by date then version (desc) - sorted(releases.items(), key=lambda x: (x[1], x[0]), reverse=True) + sorted(all_releases.items(), key=lambda x: (x[1], x[0]), reverse=True) )) print("::endgroup::") diff --git a/src/distrowatch.py b/src/distrowatch.py index 18361fbc..0a4c32ae 100644 --- a/src/distrowatch.py +++ b/src/distrowatch.py @@ -33,7 +33,7 @@ def fetch_releases(distrowatch_id, regex, template): headline = table.select_one("td.NewsHeadline a[href]").get_text().strip() date = table.select_one("td.NewsDate").get_text() for v in get_versions_from_headline(regex, headline, l_template): - print("%s: %s" % (v, date)) + print(f"{v}: {date}") releases[v] = date return releases @@ -52,6 +52,6 @@ def update_product(product_name, configs): p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product, configs in endoflife.list_products(METHOD, p_filter).items(): - print("::group::%s" % product) + print(f"::group::{product}") update_product(product, configs) print("::endgroup::") diff --git a/src/eks.py b/src/eks.py index f4bcd867..9dd5e7c0 100644 --- a/src/eks.py +++ b/src/eks.py @@ -24,9 +24,9 @@ def parse_platforms_page(): d = datetime.strptime(date, "%B %d, %Y").strftime("%Y-%m-%d") k8s_version = ".".join(data[0].text.split(".")[:-1]) eks_version = data[1].text.replace(".", "-") - version = "%s-%s" % (k8s_version, eks_version) + version = f"{k8s_version}-{eks_version}" all_versions[version] = d - print("%s: %s" % (version, d)) + print(f"{version}: {d}") print("::endgroup::") return all_versions diff --git a/src/firefox.py b/src/firefox.py index a98f57ef..0cdf07fc 100644 --- a/src/firefox.py +++ b/src/firefox.py @@ -6,21 +6,25 @@ from common import endoflife from datetime import datetime from typing import Tuple -"""Fetch Firefox versions with their dates from https://www.mozilla.org/en-US/firefox/releases/""" +"""Fetch Firefox versions with their dates from https://www.mozilla.org/""" + URL = "https://www.mozilla.org/en-US/firefox/releases/" PRODUCT = "firefox" DATE_REGEX = r"(January|Jan|February|Feb|March|Mar|April|Apr|May|June|Jun|July|Jul|August|Aug|September|Sept|October|Oct|November|Nov|December|Dec)\s+\d{1,2}(st|nd|rd|th)?,\s+\d{4}" VERSION_REGEX = r"\d+(\.\d+)*" -class UnsupportedReleasePageError(Exception): - "Raised when a firefox release page is not supported" + +class UnsupportedPageError(Exception): + """Raised when a firefox release page is not supported""" pass + class InvalidPageVariantError(Exception): - "Raised when an invalid variant is passed to get_version_and_date" + """Raised when an invalid variant is passed to get_version_and_date""" pass + def format_date(unformatted_date: str) -> str: """ Format date from July 11, 2002 to 2002-07-11 """ date = re.sub(r'(\d)(st|nd|rd|th)', r'\1', unformatted_date) @@ -32,7 +36,8 @@ def format_date(unformatted_date: str) -> str: pass return "" -def get_version_and_date_varant_1(soup: BeautifulSoup) -> Tuple[str, str]: + +def get_version_and_date_variant_1(soup: BeautifulSoup) -> Tuple[str, str]: """ Version matching for firefox versions >= 28.0 (usually) """ # get version version = soup.find("div", class_="c-release-version").get_text() @@ -41,7 +46,8 @@ def get_version_and_date_varant_1(soup: BeautifulSoup) -> Tuple[str, str]: unformatted_date = soup.find("p", class_="c-release-date").get_text() date = format_date(unformatted_date) - return (version, date) + return version, date + def get_version_and_date_variant_2(soup: BeautifulSoup) -> Tuple[str, str]: """ Version matching for firefox versions >= 10.0 (usually) """ @@ -60,7 +66,8 @@ def get_version_and_date_variant_2(soup: BeautifulSoup) -> Tuple[str, str]: unformatted_date = unformatted_date_match.group() date = format_date(unformatted_date) - return (version, date) + return version, date + def get_version_and_date_variant_3(soup: BeautifulSoup) -> Tuple[str, str]: """ Version matching for firefox versions >= 3.0 (usually) """ @@ -79,22 +86,27 @@ def get_version_and_date_variant_3(soup: BeautifulSoup) -> Tuple[str, str]: unformatted_date = unformatted_date_match.group() date = format_date(unformatted_date) - return (version, date) + return version, date + def get_version_and_date(release_page: str, release_version: str) -> Tuple[str, str]: """ Get version and date from the given release page """ major = int(release_version.split(".")[0]) - # firefox release pages for versions <3.0 don't include release dates so we + # firefox release pages for versions <3.0 don't include release dates, so we # can't match these versions for now. # example: https://www.mozilla.org/en-US/firefox/2.0/releasenotes/ if major < 3: - raise UnsupportedReleasePageError("Unsupported release page: %s" % release_page) + raise UnsupportedPageError(f"Unsupported release page: {release_page}") - # Firefox release pages come in 3 different variants. Unforunately, there is no - # consistent way to determine which variant a page is (say, by version number), so - # we have to try each variant until we find one that works. - functions = [get_version_and_date_varant_1, get_version_and_date_variant_2, get_version_and_date_variant_3] + # Firefox release pages come in 3 different variants. Unfortunately, there + # is no consistent way to determine which variant a page is (say, by version + # number), so we have to try each variant until we find one that works. + functions = [ + get_version_and_date_variant_1, + get_version_and_date_variant_2, + get_version_and_date_variant_3 + ] soup = make_bs_request(release_page) for function in functions: @@ -103,13 +115,15 @@ def get_version_and_date(release_page: str, release_version: str) -> Tuple[str, except (InvalidPageVariantError, AttributeError, IndexError): pass - raise UnsupportedReleasePageError("Unable to find version and date for %s" % release_page) + raise UnsupportedPageError(f"Unable to find version and date from {release_page}") + def make_bs_request(url: str) -> BeautifulSoup: # requests to www.mozilla.org often time out, retry in case of failures response = endoflife.fetch_url(url, timeout=10, retry_count=5) return BeautifulSoup(response, features="html5lib") + def fetch_releases(): releases = {} soup = make_bs_request(URL) @@ -126,13 +140,14 @@ def fetch_releases(): for future in concurrent.futures.as_completed(future_to_url): try: (version, date) = future.result() - print("%s: %s" % (version, date)) + print(f"{version}: {date}") releases[version] = date - except UnsupportedReleasePageError: - print("Unsupported release page: %s" % future_to_url[future]) + except UnsupportedPageError: + print(f"Unsupported release page: {future_to_url[future]}") return releases + print(f"::group::{PRODUCT}") releases = fetch_releases() endoflife.write_releases(PRODUCT, dict( diff --git a/src/github-releases.py b/src/github-releases.py index f1763823..1daba3bb 100644 --- a/src/github-releases.py +++ b/src/github-releases.py @@ -43,9 +43,10 @@ query($endCursor: String) { def fetch_releases(repo_id, regex): - """Returns this repository releases using https://docs.github.com/en/rest/releases/releases#list-releases. - Only the first page is fetched: there are rate limit rules in place on the GitHub API, and the most recent - releases are sufficient. + """Returns this repository releases using + https://docs.github.com/en/rest/releases/releases#list-releases. Only the + first page is fetched: there are rate limit rules in place on the GitHub + API, and the most recent releases are sufficient. """ releases = {} regex = [regex] if not isinstance(regex, list) else regex @@ -59,7 +60,7 @@ def fetch_releases(repo_id, regex): version = match.group(1) date = raw_date.split("T")[0] releases[version] = date - print("%s: %s" % (version, date)) + print(f"{version}: {date}") return releases @@ -76,6 +77,6 @@ def update_product(product_name, configs): p_filter = sys.argv[1] if len(sys.argv) > 1 else None for product, configs in endoflife.list_products(METHOD, p_filter).items(): - print("::group::%s" % product) + print(f"::group::{product}") update_product(product, configs) print("::endgroup::") diff --git a/src/gke.py b/src/gke.py index e917b0ca..6e8b6eb0 100644 --- a/src/gke.py +++ b/src/gke.py @@ -5,42 +5,44 @@ from datetime import datetime # https://regex101.com/r/zPxBqT/1 REGEX = r"\d.\d+\.\d+-gke\.\d+" +CHANNELS = ['nochannel', 'stable', 'regular', 'rapid'] + def fetch_channel(channel): - url = "https://cloud.google.com/kubernetes-engine/docs/release-notes-{}".format(channel) + url = f"https://cloud.google.com/kubernetes-engine/docs/release-notes-{channel}" response = endoflife.fetch_url(url) return BeautifulSoup(response, features="html5lib") -""" -Takes soup, and returns a dictionary of versions and their release dates -""" + def parse_soup_for_versions(soup): - """ Parse the soup """ + """Takes soup, and returns a dictionary of versions and their release dates + """ versions = {} for section in soup.find_all('section', class_='releases'): # h2 contains the date, which we parse for h2 in section.find_all('h2'): date = h2.get('data-text') date = datetime.strptime(date, '%B %d, %Y').strftime('%Y-%m-%d') - # The div next to the h2 contains the notes about changes made on that date + # The div next to the h2 contains the notes about changes made + # on that date next_div = h2.find_next('div') # New releases are noted in a nested list, so we look for that # and parse it using the version regex for li in next_div.find_all('li'): - # If the
  • text contains with "versions are now available:", get the