Refactor HTTP URL fetching scripts

This creates a common function to fetch HTTP URLs, with enhanced capabilities (retry, use of a known User-Agent).
It makes scripts that need those capabilities simpler, while improving other scripts.

This commit also fixes some scripts that did not log properly (cos.py, eks.py, haproxy.py, palo-alto-networks.py, rhel.py, ros.py, unrealircd.py).

src/eks.py

@@ -1,10 +1,10 @@
-import urllib.request
 import datetime
+import json
 import markdown
 import re
-import json
-from datetime import datetime
 from bs4 import BeautifulSoup
+from common import endoflife
+from datetime import datetime
 
 URL = "https://raw.githubusercontent.com/awsdocs/amazon-eks-user-guide/master/doc_source/platform-versions.md"
 REGEX = r"^(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)$"
@@ -12,20 +12,23 @@ REGEX = r"^(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)$"
 
 def parse_platforms_page():
     all_versions = {}
-    with urllib.request.urlopen(URL, data=None, timeout=5) as contents:
-        html = markdown.markdown(contents.read().decode("utf-8"), extensions=["tables"])
-        soup = BeautifulSoup(html, features="html5lib")
-        for tr in soup.findAll("tr"):
-            td = tr.find("td")
-            if td and re.match(REGEX, td.text):
-                data = tr.findAll("td")
-                date = data[-1].text
-                if len(date) > 0:
-                    d = datetime.strptime(date, "%B %d, %Y").strftime("%Y-%m-%d")
-                    k8s_version = ".".join(data[0].text.split(".")[:-1])
-                    eks_version = data[1].text.replace(".", "-")
-                    version = "%s-%s" % (k8s_version, eks_version)
-                    all_versions[version] = d
+    print("::group::eks")
+    response = endoflife.fetch_url(URL)
+    html = markdown.markdown(response, extensions=["tables"])
+    soup = BeautifulSoup(html, features="html5lib")
+    for tr in soup.findAll("tr"):
+        td = tr.find("td")
+        if td and re.match(REGEX, td.text):
+            data = tr.findAll("td")
+            date = data[-1].text
+            if len(date) > 0:
+                d = datetime.strptime(date, "%B %d, %Y").strftime("%Y-%m-%d")
+                k8s_version = ".".join(data[0].text.split(".")[:-1])
+                eks_version = data[1].text.replace(".", "-")
+                version = "%s-%s" % (k8s_version, eks_version)
+                all_versions[version] = d
+                print("%s: %s" % (version, d))
+    print("::endgroup::")
     return all_versions