Refactor HTTP URL fetching scripts

This creates a common function to fetch HTTP URLs, with enhanced capabilities (retry, use of a known User-Agent).
It makes scripts that need those capabilities simpler, while improving other scripts.

This commit also fixes some scripts that did not log properly (cos.py, eks.py, haproxy.py, palo-alto-networks.py, rhel.py, ros.py, unrealircd.py).

src/unrealircd.py

@@ -1,24 +1,26 @@
-import mwparserfromhell
 import json
+import mwparserfromhell
 import re
-import urllib.request
+from common import endoflife
 
 URL = "https://www.unrealircd.org/docwiki/index.php?title=History_of_UnrealIRCd_releases&action=raw"
 REGEX = r"^(?:(\d+\.(?:\d+\.)*\d+))$"
 
-versions = {}
-with urllib.request.urlopen(URL) as response:
-    text = response.read()
-    wikicode = mwparserfromhell.parse(text)
-    for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"):
-        items = tr.contents.filter_tags(matches=lambda node: node.tag == "td")
-        if len(items) >= 2:
-            maybe_version = items[0].__strip__()
-            if re.match(REGEX, maybe_version):
-                maybe_date = items[1].__strip__()
-                if re.match(r"\d{4}-\d{2}-\d{2}", maybe_date):
-                    versions[maybe_version] = maybe_date
+print("::group::unrealircd")
+response = endoflife.fetch_url(URL)
+wikicode = mwparserfromhell.parse(response)
 
+versions = {}
+for tr in wikicode.ifilter_tags(matches=lambda node: node.tag == "tr"):
+    items = tr.contents.filter_tags(matches=lambda node: node.tag == "td")
+    if len(items) >= 2:
+        maybe_version = items[0].__strip__()
+        if re.match(REGEX, maybe_version):
+            maybe_date = items[1].__strip__()
+            if re.match(r"\d{4}-\d{2}-\d{2}", maybe_date):
+                versions[maybe_version] = maybe_date
+                print("%s: %s" % (maybe_version, maybe_date))
+print("::endgroup::")
 
 with open("releases/unrealircd.json", "w") as f:
     f.write(json.dumps(versions, indent=2))