[skeb] fix '429 Too Many Requests' errors (#5766)

Introduce '_handle_429' method to make it easier for Extractors to react
to 429 errors regardless of 'sleep-429' settings.
This commit is contained in:
Mike Fährmann
2024-06-20 22:52:28 +02:00
parent 60b4541199
commit 11421cf940
2 changed files with 17 additions and 11 deletions

View File

@@ -43,6 +43,7 @@ class Extractor():
browser = None
request_interval = 0.0
request_interval_min = 0.0
request_interval_429 = 60.0
request_timestamp = 0.0
def __init__(self, match):
@@ -203,7 +204,9 @@ class Extractor():
self.log.warning("Cloudflare CAPTCHA")
break
if code == 429 and self._interval_429:
if code == 429 and self._handle_429(response):
continue
elif code == 429 and self._interval_429:
pass
elif code not in retry_codes and code < 500:
break
@@ -231,6 +234,8 @@ class Extractor():
raise exception.HttpError(msg, response)
_handle_429 = util.false
def wait(self, seconds=None, until=None, adjust=1.0,
reason="rate limit"):
now = time.time()
@@ -324,7 +329,7 @@ class Extractor():
self.request_interval_min,
)
self._interval_429 = util.build_duration_func(
self.config("sleep-429", 60),
self.config("sleep-429", self.request_interval_429),
)
if self._retries < 0: