From e93cfa3348d1547ca818511639a211e109df9960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 23 Jan 2026 19:54:28 +0100 Subject: [PATCH] [twitter] implement '"ratelimit": "abort:N"' (#5251 #8864) --- docs/configuration.rst | 3 +++ gallery_dl/extractor/twitter.py | 26 ++++++++++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 90d8d80f..1960fd8b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -6472,6 +6472,9 @@ Description ``"abort"`` Raise an error and stop extraction + ``"abort:N"`` + Raise an error and stop extraction + after waiting ``N`` times until rate limit reset ``"wait"`` Wait until rate limit reset ``"wait:N"`` diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 1e570403..31b431b8 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -2246,12 +2246,26 @@ class TwitterAPI(): rl = self.extractor.config("ratelimit") if rl == "abort": raise exception.AbortExtraction("Rate limit exceeded") - elif rl and isinstance(rl, str) and rl.startswith("wait:"): - until = None - seconds = text.parse_float(rl.partition(":")[2]) or 60.0 - else: - until = response.headers.get("x-rate-limit-reset") - seconds = None if until else 60.0 + + until = response.headers.get("x-rate-limit-reset") + seconds = None if until else 60.0 + + if rl and isinstance(rl, str) and ":" in rl: + rl, _, num = rl.partition(":") + if rl == "abort": + amt = getattr(self, "_ratelimit_amt", 1) + num = text.parse_int(num) + msg = f"Rate limit exceeded ({amt}/{num})" + if amt >= num: + raise exception.AbortExtraction(msg) + self.log.warning(msg) + self._ratelimit_amt = amt + 1 + elif rl == "wait": + until = None + seconds = text.parse_float(num) or 60.0 + else: + self.log.warning("Unsupported 'ratelimit' setting '%s'", rl) + self.extractor.wait(until=until, seconds=seconds) def _process_tombstone(self, entry, tombstone):