From 94e04f3b4f48da1cac1dcafc1e5e263dba9eb3eb Mon Sep 17 00:00:00 2001 From: Marc Wrobel Date: Mon, 11 Sep 2023 22:25:39 +0200 Subject: [PATCH] Increase the default timeout value Recently links to web.archive.org were added for various products. Such links are very long to load. This increases the default timeout value so that such links do not make the update fail. --- src/common/endoflife.py | 3 ++- src/firefox.py | 3 +-- src/maven.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/endoflife.py b/src/common/endoflife.py index 1cc61816..30da1414 100644 --- a/src/common/endoflife.py +++ b/src/common/endoflife.py @@ -31,7 +31,8 @@ def list_products(method, products_filter=None, pathname="website/products"): return products_with_method -def fetch_url(url, retry_count=5, timeout=10, data=None, headers=None, encoding='utf-8'): +# Keep the default timeout high enough to avoid errors with web.archive.org. +def fetch_url(url, retry_count=5, timeout=30, data=None, headers=None, encoding='utf-8'): last_exception = None headers = {'User-Agent': USER_AGENT} | {} if headers is None else headers diff --git a/src/firefox.py b/src/firefox.py index 04617ed4..4ca347e4 100644 --- a/src/firefox.py +++ b/src/firefox.py @@ -129,8 +129,7 @@ def get_version_and_date(release_page: str, release_version: str) -> Tuple[str, def make_bs_request(url: str) -> BeautifulSoup: - # requests to www.mozilla.org often time out, retry in case of failures - response = endoflife.fetch_url(url, timeout=10, retry_count=5) + response = endoflife.fetch_url(url) return BeautifulSoup(response, features="html5lib") diff --git a/src/maven.py b/src/maven.py index 3d11ba72..c76bb7ef 100644 --- a/src/maven.py +++ b/src/maven.py @@ -18,7 +18,7 @@ def valid_version(version): def fetch_json(group_id, artifact_id, start): url = f"https://search.maven.org/solrsearch/select?q=g:{group_id}+AND+a:{artifact_id}&core=gav&rows=100&wt=json&start={start}" - response = endoflife.fetch_url(url, retry_count=5) + response = endoflife.fetch_url(url) return json.loads(response)