Refactor HTTP common code (#207)

- move to common/http.py,
- make fetch_url return a Response.

src/amazon-neptune.py

@@ -1,7 +1,8 @@
 import re
-from xml.dom.minidom import parseString
+from common import http
 from common import dates
 from common import endoflife
+from xml.dom.minidom import parseString
 
 """Fetch versions with their dates from the RSS feed of
 https://docs.aws.amazon.com/neptune/latest/userguide/engine-releases.html.
@@ -14,8 +15,8 @@ URL = "https://docs.aws.amazon.com/neptune/latest/userguide/rssupdates.rss"
 print(f"::group::{PRODUCT}")
 versions = {}
 
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-rss = parseString(response)
+rss = parseString(response.text)
 for item in rss.getElementsByTagName("item"):
     title = item.getElementsByTagName("title")[0].firstChild.nodeValue
     pubDate = item.getElementsByTagName("pubDate")[0].firstChild.nodeValue

src/apple.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -12,7 +13,6 @@ URLS = [
     "https://support.apple.com/kb/HT205759",  # 2013
     "https://support.apple.com/kb/HT204611",  # 2011 to 2012
     # Apple still links to the following articles, but they are 404:
-    # Disabled, too much timed out.
     "http://web.archive.org/web/20230404214605_/https://support.apple.com/en-us/HT5165",  # 2010
     "http://web.archive.org/web/20230327200842_/https://support.apple.com/en-us/HT4218",  # 2008-2009
     "http://web.archive.org/web/20230204234533_/https://support.apple.com/en-us/HT1263",  # 2005-2007
@@ -54,7 +54,7 @@ def parse_date(date_str):
 print("::group::apple")
 versions_by_product = {k: {} for k in CONFIG.keys()}
 
-for response in endoflife.fetch_urls(URLS):
+for response in http.fetch_urls(URLS):
     soup = BeautifulSoup(response.text, features="html5lib")
     versions_table = soup.find(id="tableWraper")
     versions_table = versions_table if versions_table else soup.find('table', class_="gb-table")

src/aws-lambda.py

@@ -1,4 +1,5 @@
 from bs4 import BeautifulSoup
+from common import http
 from common import endoflife
 from datetime import datetime
 
@@ -37,10 +38,9 @@ except FileNotFoundError:
     releases_data = {}
     print(f"{PRODUCT} file not found, real release dates will not be used.")
 
-response = endoflife.fetch_url(URL)
-soup = BeautifulSoup(response, features="html5lib")
-
 versions = {}
+response = http.fetch_url(URL)
+soup = BeautifulSoup(response.text, features="html5lib")
 for row in soup.find_all("tr"):
     cells = row.find_all("td")
     if len(cells) == 6:  # Supported Runtimes

src/cgit.py

@@ -1,6 +1,7 @@
 import re
 import sys
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 from liquid import Template
@@ -14,15 +15,12 @@ do not support partial clone so we cannot.
 
 METHOD = 'cgit'
 
-def make_bs_request(url):
-    response = endoflife.fetch_url(url + '/refs/tags')
-    return BeautifulSoup(response, features="html5lib")
-
 
 def fetch_releases(url, regex, template):
     releases = {}
 
-    soup = make_bs_request(url)
+    response = http.fetch_url(url + '/refs/tags')
+    soup = BeautifulSoup(response.text, features="html5lib")
     l_template = Template(template)
 
     for table in soup.find_all("table", class_="list"):

src/coldfusion.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -28,7 +29,7 @@ REGEX = r"[r|R]elease [d|D]ate[,|:]? (.*?)\).*?Build Number: (.*?)$"
 print(f"::group::{PRODUCT}")
 versions = RELEASE_DATES | {}
 
-for response in endoflife.fetch_urls(URLS):
+for response in http.fetch_urls(URLS):
     soup = BeautifulSoup(response.text, features="html5lib")
     for p in soup.findAll("div", class_="text"):
         text = p.get_text().strip().replace('\xa0', ' ')

src/common/endoflife.py

@@ -1,16 +1,7 @@
 import json
 import frontmatter
-from concurrent.futures import as_completed
 from glob import glob
 from os import path
-from requests import Response
-from requests.adapters import HTTPAdapter
-from requests.exceptions import ChunkedEncodingError
-from requests_futures.sessions import FuturesSession
-from urllib3.util import Retry
-
-# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent.
-USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0'
 
 # Handle versions having at least 2 digits (ex. 1.2) and at most 4 digits (ex. 1.2.3.4), with an optional leading "v".
 # Major version must be >= 1.
@@ -49,28 +40,6 @@ def list_products(method, products_filter=None, pathname="website/products") ->
 
 
 # Keep the default timeout high enough to avoid errors with web.archive.org.
-def fetch_urls(urls, data=None, headers=None, max_retries=10, backoff_factor=0.5, timeout=30) -> list[Response]:
-    try:
-        with FuturesSession() as session:
-            adapter = HTTPAdapter(max_retries=Retry(total=max_retries, backoff_factor=backoff_factor))
-            session.mount('http://', adapter)
-            session.mount('https://', adapter)
-
-            headers = {'User-Agent': USER_AGENT} | ({} if headers is None else headers)
-            futures = [session.get(url, headers=headers, data=data, timeout=timeout, stream=None) for url in urls]
-            return [future.result() for future in as_completed(futures)]
-    except ChunkedEncodingError as e:  # See https://github.com/psf/requests/issues/4771#issue-354077499
-        next_max_retries = max_retries - 1
-        if next_max_retries == 0:
-            raise e  # So that the function does not get stuck in an infinite loop.
-        else:
-            # We could wait a bit before retrying, but it's not clear if it would help.
-            print(f"Got ChunkedEncodingError while fetching {urls} ({e}), retrying (remaining retries = {next_max_retries}).")
-            return fetch_urls(urls, data, headers, next_max_retries, backoff_factor, timeout)
-
-
-def fetch_url(url, data=None, headers=None, max_retries=5, backoff_factor=0.5, timeout=30) -> str:
-    return fetch_urls([url], data, headers, max_retries, backoff_factor, timeout)[0].text
 
 
 def write_releases(product, releases, pathname="releases") -> None:

src/common/http.py

@@ -0,0 +1,33 @@
+from concurrent.futures import as_completed
+from requests import Response
+from requests.adapters import HTTPAdapter
+from requests.exceptions import ChunkedEncodingError
+from requests_futures.sessions import FuturesSession
+from urllib3.util import Retry
+
+# See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent.
+USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0'
+
+
+def fetch_urls(urls, data=None, headers=None, max_retries=10, backoff_factor=0.5, timeout=30) -> list[Response]:
+    try:
+        with FuturesSession() as session:
+            adapter = HTTPAdapter(max_retries=Retry(total=max_retries, backoff_factor=backoff_factor))
+            session.mount('http://', adapter)
+            session.mount('https://', adapter)
+
+            headers = {'User-Agent': USER_AGENT} | ({} if headers is None else headers)
+            futures = [session.get(url, headers=headers, data=data, timeout=timeout, stream=None) for url in urls]
+            return [future.result() for future in as_completed(futures)]
+    except ChunkedEncodingError as e:  # See https://github.com/psf/requests/issues/4771#issue-354077499
+        next_max_retries = max_retries - 1
+        if next_max_retries == 0:
+            raise e  # So that the function does not get stuck in an infinite loop.
+        else:
+            # We could wait a bit before retrying, but it's not clear if it would help.
+            print(f"Got ChunkedEncodingError while fetching {urls} ({e}), retrying (remaining retries = {next_max_retries}).")
+            return fetch_urls(urls, data, headers, next_max_retries, backoff_factor, timeout)
+
+
+def fetch_url(url, data=None, headers=None, max_retries=5, backoff_factor=0.5, timeout=30) -> Response:
+    return fetch_urls([url], data, headers, max_retries, backoff_factor, timeout)[0]

src/cos.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -8,15 +9,15 @@ REGEX = r"^(cos-\d+-\d+-\d+-\d+)"
 
 
 def list_milestones():
-    response = endoflife.fetch_url(URL)
+    response = http.fetch_url(URL)
-    soup = BeautifulSoup(response, features="html5lib")
+    soup = BeautifulSoup(response.text, features="html5lib")
     milestones = soup.find_all('td', string=re.compile(r'COS \d+ LTS'))
     return [m.text.split(' ')[1] for m in milestones]
 
 
 def fetch_milestones(milestones):
     urls = [f"{URL}m{channel}" for channel in milestones]
-    return endoflife.fetch_urls(urls)
+    return http.fetch_urls(urls)
 
 
 def parse_date(date_str):

src/couchbase-server.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -29,8 +30,8 @@ FIXED_VERSIONS = {
 print(f"::group::{PRODUCT}")
 versions = {}
 
-response = endoflife.fetch_url(f"{URLS}/current/install/install-intro.html")
+response = http.fetch_url(f"{URLS}/current/install/install-intro.html")
-soup = BeautifulSoup(response, features="html5lib")
+soup = BeautifulSoup(response.text, features="html5lib")
 
 minor_versions = [options.attrs["value"] for options in soup.find(class_="version_list").find_all("option")]
 
@@ -39,7 +40,7 @@ for minor in minor_versions:
     versions[minor + '.0'] = 'N/A'
 
 minor_version_urls = [f"{URLS}/{minor}/release-notes/relnotes.html" for minor in minor_versions]
-for response in endoflife.fetch_urls(minor_version_urls):
+for response in http.fetch_urls(minor_version_urls):
     soup = BeautifulSoup(response.text, features="html5lib")
     for title in soup.find_all("h2"):
         versionAndDate = title.get_text().strip()

src/distrowatch.py

@@ -1,6 +1,7 @@
 import re
 import sys
 from bs4 import BeautifulSoup
+from common import http
 from common import endoflife
 from liquid import Template
 
@@ -27,8 +28,8 @@ def fetch_releases(distrowatch_id, regex, template):
     releases = {}
     l_template = Template(template)
     url = f"https://distrowatch.com/index.php?distribution={distrowatch_id}"
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    soup = BeautifulSoup(response, features="html5lib")
+    soup = BeautifulSoup(response.text, features="html5lib")
     for table in soup.select("td.News1>table.News"):
         headline = table.select_one("td.NewsHeadline a[href]").get_text().strip()
         date = table.select_one("td.NewsDate").get_text()

src/docker_hub.py

@@ -1,6 +1,6 @@
-import json
 import re
 import sys
+from common import http
 from common import endoflife
 
 METHOD = "docker_hub"
@@ -10,8 +10,8 @@ def fetch_releases(url, regex, releases):
     if not isinstance(regex, list):
         regex = [regex]
 
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    data = json.loads(response)
+    data = response.json()
     for result in data["results"]:
         version = result["name"]
 

src/eks.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -22,8 +23,8 @@ def parse_platforms_pages():
     all_versions = {}
     print("::group::eks")
     for url in URLS:
-        response = endoflife.fetch_url(url)
+        response = http.fetch_url(url)
-        soup = BeautifulSoup(response, features="html5lib")
+        soup = BeautifulSoup(response.text, features="html5lib")
         for tr in soup.select("#main-col-body")[0].findAll("tr"):
             td = tr.find("td")
             if td and re.match(endoflife.DEFAULT_VERSION_REGEX, td.text.strip()):

src/firefox.py

@@ -1,6 +1,7 @@
 import re
 import urllib.parse
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -18,11 +19,11 @@ def format_date(text: str) -> str:
 print(f"::group::{PRODUCT}")
 versions = {}
 
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-ff_releases = BeautifulSoup(response, features="html5lib").find_all("ol", class_="c-release-list")
+ff_releases = BeautifulSoup(response.text, features="html5lib").find_all("ol", class_="c-release-list")
 urls = [urllib.parse.urljoin(URL, p.get("href")) for p in ff_releases[0].find_all("a")]
 
-for response in endoflife.fetch_urls(urls):
+for response in http.fetch_urls(urls):
     soup = BeautifulSoup(response.text, features="html5lib")
 
     version = response.request.url.split("/")[-3]

src/gke.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -10,8 +11,8 @@ CHANNELS = ['nochannel', 'stable', 'regular', 'rapid']
 
 def fetch_channel(channel):
     url = f"https://cloud.google.com/kubernetes-engine/docs/release-notes-{channel}"
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    return BeautifulSoup(response, features="html5lib")
+    return BeautifulSoup(response.text, features="html5lib")
 
 
 def parse_soup_for_versions(soup):

src/graalvm.py

@@ -1,4 +1,5 @@
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -12,8 +13,8 @@ def split_versions(text):
     return text.replace("GraalVM for JDK ", "jdk-").split(", ")
 
 print("::group::graalvm")
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-soup = BeautifulSoup(response, features="html5lib")
+soup = BeautifulSoup(response.text, features="html5lib")
 
 versions = {}
 for tr in soup.findAll("table")[1].find("tbody").findAll("tr"):

src/haproxy.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import endoflife
 
 """Fetch HAProxy versions with their dates from https://www.haproxy.org/.
@@ -14,8 +15,8 @@ VERSION_REGEX = r"^(\d{4})\/(\d{2})\/(\d{2})\s+:\s+(\d+\.\d+\.\d.?)$"
 def fetch_cycles():
     cycles = []
 
-    response = endoflife.fetch_url('https://www.haproxy.org/download/')
+    response = http.fetch_url('https://www.haproxy.org/download/')
-    soup = BeautifulSoup(response, features="html5lib")
+    soup = BeautifulSoup(response.text, features="html5lib")
     for link in soup.select("a"):
         m = re.match(CYCLE_REGEX, link.attrs["href"])
         if m:
@@ -32,7 +33,7 @@ def fetch_releases(cycles):
     releases = {}
 
     urls = [f"https://www.haproxy.org/download/{cycle}/src/CHANGELOG" for cycle in cycles]
-    for response in endoflife.fetch_urls(urls):
+    for response in http.fetch_urls(urls):
         for line in response.text.split('\n'):
             m = re.match(VERSION_REGEX, line)
             if m:

src/ibm-aix.py

@@ -1,4 +1,5 @@
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -6,8 +7,8 @@ PRODUCT = "ibm-aix"
 URL = "https://www.ibm.com/support/pages/aix-support-lifecycle-information"
 
 def fetch_releases():
-    response = endoflife.fetch_url(URL)
+    response = http.fetch_url(URL)
-    soup = BeautifulSoup(response, features="html5lib")
+    soup = BeautifulSoup(response.text, features="html5lib")
 
     releases = {}
     # for all release tables

src/looker.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 from xml.dom.minidom import parseString
@@ -16,8 +17,8 @@ VERSION_PATTERN = re.compile(r"Looker\s+(?P<version>\d+\.\d+)", re.IGNORECASE)
 print(f"::group::{PRODUCT}")
 versions = {}
 
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-rss = parseString(response)
+rss = parseString(response.text)
 for item in rss.getElementsByTagName("entry"):
     date = dates.parse_datetime(item.getElementsByTagName("updated")[0].firstChild.nodeValue).strftime("%Y-%m-%d")
     content = item.getElementsByTagName("content")[0].firstChild.nodeValue

src/maven.py

@@ -1,7 +1,7 @@
 import datetime
-import json
 import re
 import sys
+from common import http
 from common import endoflife
 
 METHOD = "maven"
@@ -17,8 +17,8 @@ def valid_version(version):
 
 def fetch_json(group_id, artifact_id, start):
     url = f"https://search.maven.org/solrsearch/select?q=g:{group_id}+AND+a:{artifact_id}&core=gav&rows=100&wt=json&start={start}"
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    return json.loads(response)
+    return response.json()
 
 
 def fetch_releases(package_identifier):

src/npm.py

@@ -1,6 +1,6 @@
-import json
 import re
 import sys
+from common import http
 from common import endoflife
 
 METHOD = "npm"
@@ -13,8 +13,8 @@ def fetch_releases(npm_id, regex):
         regex = [regex]
 
     url = f"https://registry.npmjs.org/{npm_id}"
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    data = json.loads(response)
+    data = response.json()
     for version in data["time"]:
         matches = False
         for r in regex:

src/nutanix.py

@@ -1,4 +1,4 @@
-import json
+from common import http
 from common import endoflife
 
 """Fetch Nutanix products versions with their dates from https://portal.nutanix.com/api/v1.
@@ -17,8 +17,8 @@ def fetch_releases(product_code):
     versions = {}
     url = BASE_URL + product_code
     print(url)
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    data = json.loads(response)
+    data = response.json()
 
     for version_data in data["contents"]:
         if 'GENERAL_AVAILABILITY' in version_data:

src/palo-alto-networks.py

@@ -1,6 +1,7 @@
 import datetime
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import endoflife
 
 URL = "https://www.paloaltonetworks.com/services/support/end-of-life-announcements/end-of-life-summary"
@@ -15,8 +16,8 @@ def update_releases(html_identifier, file):
     versions = {}
 
     print(f"::group::{html_identifier}")
-    response = endoflife.fetch_url(URL)
+    response = http.fetch_url(URL)
-    soup = BeautifulSoup(response, features="html5lib")
+    soup = BeautifulSoup(response.text, features="html5lib")
 
     table = soup.find(id=html_identifier)
     for tr in table.findAll("tr")[3:]:

src/php.py

@@ -1,4 +1,4 @@
-import json
+from common import http
 from common import dates
 from common import endoflife
 
@@ -7,8 +7,8 @@ PHP_MAJOR_VERSIONS = [4, 5, 7, 8]
 
 def fetch_versions(major_version):
     url = f"https://www.php.net/releases/index.php?json&max=-1&version={major_version}"
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    data = json.loads(response)
+    data = response.json()
     for v in data:
         data[v] = dates.parse_date(data[v]["date"]).strftime("%Y-%m-%d")
         print(f"{v}: {data[v]}")

src/plesk.py

@@ -1,4 +1,5 @@
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -7,8 +8,8 @@ PRODUCT = "plesk"
 
 
 def make_bs_request(url):
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    return BeautifulSoup(response, features="html5lib")
+    return BeautifulSoup(response.text, features="html5lib")
 
 
 # Only 18.0.20.3 and later will be picked up :

src/pypi.py

@@ -1,6 +1,6 @@
-import json
 import re
 import sys
+from common import http
 from common import dates
 from common import endoflife
 
@@ -14,8 +14,8 @@ def fetch_releases(pypi_id, regex):
         regex = [regex]
 
     url = f"https://pypi.org/pypi/{pypi_id}/json"
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    data = json.loads(response)
+    data = response.json()
     for version in data["releases"]:
         R = data["releases"][version]
         matches = False

src/rds.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -13,8 +14,8 @@ for db, url in DBS.items():
     print(f"::group::{db}")
     versions = {}
 
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    soup = BeautifulSoup(response, features="html5lib")
+    soup = BeautifulSoup(response.text, features="html5lib")
 
     for table in soup.find_all("table"):
         for row in table.find_all("tr"):

src/redhat-satellite.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import endoflife
 
 """Fetch versions with their dates from access.redhat.com.
@@ -13,8 +14,8 @@ URL = "https://access.redhat.com/articles/1365633"
 regex = r"^Satellite (?P<version>\d+\.\d+\.\d+([.-]\d+)?) ([Uu]pdate|[Rr]elease)$"
 
 print("::group::redhat-satellite")
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-soup = BeautifulSoup(response, features="html5lib")
+soup = BeautifulSoup(response.text, features="html5lib")
 
 versions = {}
 for table in soup.findAll("tbody"):

src/rhel.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import endoflife
 
 URL = "https://access.redhat.com/articles/3078"
@@ -7,8 +8,8 @@ URL = "https://access.redhat.com/articles/3078"
 regex = r"RHEL (?P<major>\d)(\. ?(?P<minor>\d+))?(( Update (?P<minor2>\d))| GA)?"
 
 print("::group::rhel")
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-soup = BeautifulSoup(response, features="html5lib")
+soup = BeautifulSoup(response.text, features="html5lib")
 
 versions = {}
 for tr in soup.findAll("tr"):

src/rockylinux.py

@@ -1,4 +1,5 @@
 import re
+from common import http
 from common import dates
 from common import endoflife
 
@@ -26,7 +27,7 @@ def parse_markdown_table(table_text):
 
 
 print("::group::rockylinux")
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-versions = parse_markdown_table(response)
+versions = parse_markdown_table(response.text)
 endoflife.write_releases('rockylinux', versions)
 print("::endgroup::")

src/ros.py

@@ -1,6 +1,7 @@
 import datetime
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import endoflife
 
 URL = "https://wiki.ros.org/Distributions"
@@ -8,8 +9,8 @@ URL = "https://wiki.ros.org/Distributions"
 regex = r"^ROS (?P<name>(\w| )+)"
 
 print("::group::ros")
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-soup = BeautifulSoup(response, features="html5lib")
+soup = BeautifulSoup(response.text, features="html5lib")
 
 versions = {}
 for tr in soup.findAll("tr"):

src/sles.py

@@ -1,4 +1,5 @@
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -11,8 +12,8 @@ def strip_version(version_str):
 
 
 def fetch_releases():
-    response = endoflife.fetch_url(URL)
+    response = http.fetch_url(URL)
-    soup = BeautifulSoup(response, features="html5lib")
+    soup = BeautifulSoup(response.text, features="html5lib")
     products_table = soup.find("tbody", id="productSupportLifecycle")
     # Get rows with SLES products
     sles_header_rows = products_table.find_all("tr", class_="row", attrs={"data-productfilter": "SUSE Linux Enterprise Server"})

src/splunk.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -35,8 +36,8 @@ def get_latest_minor_versions(versions):
 
 print(f"::group::{PRODUCT}")
 versions = dict()
-main = endoflife.fetch_url(URL)
+main = http.fetch_url(URL)
-soup = BeautifulSoup(main, features="html5lib")
+soup = BeautifulSoup(main.text, features="html5lib")
 
 all_versions = list(map(
     lambda option: option.attrs['value'],
@@ -48,7 +49,7 @@ all_versions = list(map(
 latest_minor_versions = get_latest_minor_versions(all_versions)
 latest_minor_versions_urls = [RELNOTES_URL_TEMPLATE.format(version=v) for v in latest_minor_versions]
 
-for response in endoflife.fetch_urls(latest_minor_versions_urls):
+for response in http.fetch_urls(latest_minor_versions_urls):
     for (version, date_str) in re.findall(PATTERN, response.text, re.MULTILINE):
         version = f"{version}.0" if len(version.split(".")) == 2 else version  # convert x.y to x.y.0
         date = dates.parse_date(date_str).strftime("%Y-%m-%d")

src/typo3.py

@@ -1,4 +1,5 @@
 import json
+from common import http
 from common import endoflife
 
 PRODUCT = "typo3"
@@ -7,8 +8,8 @@ URL = "https://get.typo3.org/api/v1/release/"
 print(f"::group::{PRODUCT}")
 versions = {}
 
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-data = json.loads(response)
+data = json.loads(response.text)
 for v in data:
     if v['type'] != 'development':
         date = v["date"][0:10]

src/unity.py

@@ -1,4 +1,5 @@
 from bs4 import BeautifulSoup
+from common import http
 from common import endoflife
 
 # Fetches the Unity LTS releases from the Unity website. Non-LTS releases are not listed there,
@@ -13,8 +14,8 @@ URL = 'https://unity.com/releases/editor/qa/lts-releases'
 
 def fetch_releases(releases, url) -> str:
     print(url)
-    response = endoflife.fetch_url(url)
+    response = http.fetch_url(url)
-    soup = BeautifulSoup(response, features="html5lib")
+    soup = BeautifulSoup(response.text, features="html5lib")
 
     for release in soup.find_all('div', class_='component-releases-item__show__inner-header'):
         version = release.find('h4').find('span').text

src/unrealircd.py

@@ -1,12 +1,13 @@
 import mwparserfromhell
 import re
+from common import http
 from common import endoflife
 
 URL = "https://www.unrealircd.org/docwiki/index.php?title=History_of_UnrealIRCd_releases&action=raw"
 
 print("::group::unrealircd")
-response = endoflife.fetch_url(URL)
+response = http.fetch_url(URL)
-wikicode = mwparserfromhell.parse(response)
+wikicode = mwparserfromhell.parse(response.text)
 
 versions = {}
 for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"):

src/visualstudio.py

@@ -1,5 +1,6 @@
 import re
 from bs4 import BeautifulSoup
+from common import http
 from common import dates
 from common import endoflife
 
@@ -16,7 +17,7 @@ URLS = [
 print(f"::group::{PRODUCT}")
 versions = {}
 
-for response in endoflife.fetch_urls(URLS):
+for response in http.fetch_urls(URLS):
     soup = BeautifulSoup(response.text, features="html5lib")
     for table in soup.find_all("table"):
         headers = [th.get_text().strip().lower() for th in table.find_all("th")]