retry HTTP requests for more exception classes
This commit is contained in:
@@ -64,9 +64,12 @@ class Extractor():
|
|||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
response = self.session.request(method, url, **kwargs)
|
response = self.session.request(method, url, **kwargs)
|
||||||
except (requests.ConnectionError, requests.Timeout) as exc:
|
except (requests.exceptions.ConnectionError,
|
||||||
|
requests.exceptions.Timeout,
|
||||||
|
requests.exceptions.ChunkedEncodingError,
|
||||||
|
requests.exceptions.ContentDecodingError) as exc:
|
||||||
msg = exc
|
msg = exc
|
||||||
except requests.exceptions.RequestException as exc:
|
except (requests.exceptions.RequestException) as exc:
|
||||||
raise exception.HttpError(exc)
|
raise exception.HttpError(exc)
|
||||||
else:
|
else:
|
||||||
code = response.status_code
|
code = response.status_code
|
||||||
@@ -79,11 +82,11 @@ class Extractor():
|
|||||||
if code < 500 and code != 429:
|
if code < 500 and code != 429:
|
||||||
break
|
break
|
||||||
|
|
||||||
self.log.debug("%s (%d/%d)", msg, tries + 1, retries)
|
tries += 1
|
||||||
|
self.log.debug("%s (%d/%d)", msg, tries, retries)
|
||||||
if tries >= retries:
|
if tries >= retries:
|
||||||
break
|
break
|
||||||
time.sleep(2 ** tries)
|
time.sleep(2 ** tries)
|
||||||
tries += 1
|
|
||||||
|
|
||||||
raise exception.HttpError(msg)
|
raise exception.HttpError(msg)
|
||||||
|
|
||||||
|
|||||||
@@ -60,7 +60,10 @@ class Job():
|
|||||||
res = str(exc) or "resource (gallery/image/user)"
|
res = str(exc) or "resource (gallery/image/user)"
|
||||||
log.error("The %s at '%s' does not exist", res, self.url)
|
log.error("The %s at '%s' does not exist", res, self.url)
|
||||||
except exception.HttpError as exc:
|
except exception.HttpError as exc:
|
||||||
log.error("HTTP request failed: %s", exc)
|
err = exc.args[0]
|
||||||
|
if isinstance(err, Exception):
|
||||||
|
err = "{}: {}".format(err.__class__.__name__, err)
|
||||||
|
log.error("HTTP request failed: %s", err)
|
||||||
except exception.FormatError as exc:
|
except exception.FormatError as exc:
|
||||||
err, obj = exc.args
|
err, obj = exc.args
|
||||||
log.error("Applying %s format string failed: %s: %s",
|
log.error("Applying %s format string failed: %s: %s",
|
||||||
|
|||||||
Reference in New Issue
Block a user