diff --git a/src/common/endoflife.py b/src/common/endoflife.py index 1cc61816..30da1414 100644 --- a/src/common/endoflife.py +++ b/src/common/endoflife.py @@ -31,7 +31,8 @@ def list_products(method, products_filter=None, pathname="website/products"): return products_with_method -def fetch_url(url, retry_count=5, timeout=10, data=None, headers=None, encoding='utf-8'): +# Keep the default timeout high enough to avoid errors with web.archive.org. +def fetch_url(url, retry_count=5, timeout=30, data=None, headers=None, encoding='utf-8'): last_exception = None headers = {'User-Agent': USER_AGENT} | {} if headers is None else headers diff --git a/src/firefox.py b/src/firefox.py index 04617ed4..4ca347e4 100644 --- a/src/firefox.py +++ b/src/firefox.py @@ -129,8 +129,7 @@ def get_version_and_date(release_page: str, release_version: str) -> Tuple[str, def make_bs_request(url: str) -> BeautifulSoup: - # requests to www.mozilla.org often time out, retry in case of failures - response = endoflife.fetch_url(url, timeout=10, retry_count=5) + response = endoflife.fetch_url(url) return BeautifulSoup(response, features="html5lib") diff --git a/src/maven.py b/src/maven.py index 3d11ba72..c76bb7ef 100644 --- a/src/maven.py +++ b/src/maven.py @@ -18,7 +18,7 @@ def valid_version(version): def fetch_json(group_id, artifact_id, start): url = f"https://search.maven.org/solrsearch/select?q=g:{group_id}+AND+a:{artifact_id}&core=gav&rows=100&wt=json&start={start}" - response = endoflife.fetch_url(url, retry_count=5) + response = endoflife.fetch_url(url) return json.loads(response)