Refactor HTTP URL fetching scripts

This creates a common function to fetch HTTP URLs, with enhanced capabilities (retry, use of a known User-Agent).
It makes scripts that need those capabilities simpler, while improving other scripts.

This commit also fixes some scripts that did not log properly (cos.py, eks.py, haproxy.py, palo-alto-networks.py, rhel.py, ros.py, unrealircd.py).

src/ros.py

@@ -1,8 +1,8 @@
-import json
-import urllib.request
 import datetime
-from bs4 import BeautifulSoup
+import json
 import re
+from bs4 import BeautifulSoup
+from common import endoflife
 
 URL = "https://wiki.ros.org/Distributions"
 # https://regex101.com/r/c1ribd/1
@@ -10,27 +10,29 @@ regex = r"^ROS (?P<name>(\w| )+)"
 
 versions = {}
 
-with urllib.request.urlopen(URL, timeout=5) as response:
-    soup = BeautifulSoup(response, features="html5lib")
-    for tr in soup.findAll("tr"):
-        td_list = tr.findAll("td")
-        if len(td_list) > 0:
-            version = td_list[0].get_text()
+print("::group::ros")
+response = endoflife.fetch_url(URL)
+soup = BeautifulSoup(response, features="html5lib")
+for tr in soup.findAll("tr"):
+    td_list = tr.findAll("td")
+    if len(td_list) > 0:
+        version = td_list[0].get_text()
 
-            m = re.match(regex, version.strip())
-            if m:
-                version = td_list[0].findAll("a")[0]["href"][1:]
-                try:
-                    date = datetime.datetime.strptime(
-                        td_list[1].get_text().strip(), "%B %d, %Y"
-                    )
-                # The date is a suffix (May 23rd, 2020)
-                except Exception as e:
-                    x = td_list[1].get_text().split(",")
-                    date = datetime.datetime.strptime(x[0][:-2] + x[1], "%B %d %Y")
-                abs_date = date.strftime("%Y-%m-%d")
-                versions[version] = abs_date
-                print("%s: %s" % (version, abs_date))
+        m = re.match(regex, version.strip())
+        if m:
+            version = td_list[0].findAll("a")[0]["href"][1:]
+            try:
+                date = datetime.datetime.strptime(
+                    td_list[1].get_text().strip(), "%B %d, %Y"
+                )
+            # The date is a suffix (May 23rd, 2020)
+            except Exception as e:
+                x = td_list[1].get_text().split(",")
+                date = datetime.datetime.strptime(x[0][:-2] + x[1], "%B %d %Y")
+            abs_date = date.strftime("%Y-%m-%d")
+            versions[version] = abs_date
+            print("%s: %s" % (version, abs_date))
+print("::endgroup::")
 
 with open("releases/ros.json", "w") as f:
     f.write(json.dumps(versions, indent=2))