github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implement linear/exponential backoff for 'sleep-429'

Browse Source
This commit is contained in:
Mike Fährmann
2026-02-12 19:23:29 +01:00
parent eb4e44401b
commit 0f41f343f4
5 changed files with 124 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
gallery_dl/extractor/common.py
View File

@@ -256,8 +256,8 @@ class Extractor():
s = self._interval()
if seconds < s:
seconds = s
if code == 429 and self._interval_429:
s = self._interval_429()
if code == 429 and self._interval_429 is not None:
s = self._interval_429(tries)
if seconds < s:
seconds = s
self.wait(seconds=seconds, reason="429 Too Many Requests")
@@ -418,9 +418,17 @@ class Extractor():
self.config("sleep-request", self.request_interval),
self.request_interval_min,
)
self._interval_429 = util.build_duration_func(
self.config("sleep-429", self.request_interval_429),
)
_interval_429 = self.config("sleep-429")
if _interval_429 is None:
_interval_429 = self.request_interval_429
try:
self._interval_429 = util.build_duration_func_ex(_interval_429)
except Exception as exc:
self.log.error("Invalid 'sleep-429' value '%s' (%s: %s)",
_interval_429, exc.__class__.__name__, exc)
self._interval_429 = util.build_duration_func_ex(
self.request_interval_429)
if self._retries < 0:
self._retries = float("inf")