Refactor HTTP URL fetching scripts

This creates a common function to fetch HTTP URLs, with enhanced capabilities (retry, use of a known User-Agent).
It makes scripts that need those capabilities simpler, while improving other scripts.

This commit also fixes some scripts that did not log properly (cos.py, eks.py, haproxy.py, palo-alto-networks.py, rhel.py, ros.py, unrealircd.py).
This commit is contained in:
Marc Wrobel
2023-05-14 09:35:28 +02:00
parent 5176abd4d4
commit a16d9090d3
19 changed files with 295 additions and 311 deletions

View File

@@ -1,8 +1,7 @@
import sys
import json
import urllib.request
import datetime
import json
import re
import sys
from common import endoflife
METHOD = "maven"
@@ -19,20 +18,8 @@ def valid_version(version):
def fetch_json(group_id, artifact_id, start):
url = f"https://search.maven.org/solrsearch/select?q=g:{group_id}+AND+a:{artifact_id}&core=gav&rows=100&wt=json&start={start}"
last_exception = None
# search.maven.org often time out lately, retry the request in case of failures.
for i in range(0, 5):
try:
with urllib.request.urlopen(url, data=None, timeout=5) as response:
return json.load(response)
except Exception as e:
last_exception = e
message = getattr(e, 'message', repr(e)) # https://stackoverflow.com/a/45532289/374236
print(f"Error while requesting {url} ({message}), retrying ({i})...")
continue
raise last_exception
response = endoflife.fetch_url(url, retry_count=5)
return json.loads(response)
def fetch_releases(package_identifier):