From 7c047202b310baa71d291b7bd96f85c1d20fef5f Mon Sep 17 00:00:00 2001 From: Marc Wrobel Date: Sun, 8 Jan 2023 13:58:26 +0100 Subject: [PATCH] [firefox] Retry on timeout --- src/firefox.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/firefox.py b/src/firefox.py index a95717a5..a3c59ddc 100644 --- a/src/firefox.py +++ b/src/firefox.py @@ -108,10 +108,21 @@ def get_version_and_date(release_page: str, release_version: str) -> Tuple[str, def make_bs_request(url: str) -> BeautifulSoup: """ Make a request to the given url and return a BeautifulSoup object """ + last_exception = None headers = {"user-agent": "mozilla"} - req = urllib.request.Request(url, headers=headers) - res = urllib.request.urlopen(req, timeout=5) - return BeautifulSoup(res.read(), features="html5lib") + + # requests to www.mozilla.org often time out, retry in case of failures + for i in range(0, 5): + try: + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req, timeout=5) as response: + return BeautifulSoup(response.read(), features="html5lib") + except TimeoutError as e: + last_exception = e + print(f"Request to {url} timed out, retrying ({i})...") + continue + + raise last_exception def fetch_releases(): releases = {}