From 162d4269ec3e39df283557ec6e2a758296660314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Thu, 6 Jun 2024 01:18:08 +0200 Subject: [PATCH] [twitter] extend 'ratelimit' option (#5532) allow waiting for a set amount of seconds --- docs/configuration.rst | 9 +++++---- gallery_dl/extractor/twitter.py | 13 +++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 35856e37..e6ccd411 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -3987,6 +3987,7 @@ Description * ``"abort"``: Raise an error and stop extraction * ``"wait"``: Wait until rate limit reset + * ``"wait:N"``: Wait for ``N`` seconds extractor.twitter.relogin @@ -3996,10 +3997,10 @@ Type Default ``true`` Description - | When receiving a "Could not authenticate you" error while logged in with - `username & passeword `__, - | refresh the current login session and - try to continue from where it left off. + When receiving a "Could not authenticate you" error while logged in with + `username & passeword `__, + refresh the current login session and + try to continue from where it left off. extractor.twitter.locked diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index ff77828c..81f489bc 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -1709,11 +1709,16 @@ class TwitterAPI(): variables["cursor"] = cursor def _handle_ratelimit(self, response): - if self.extractor.config("ratelimit") == "abort": + rl = self.extractor.config("ratelimit") + if rl == "abort": raise exception.StopExtraction("Rate limit exceeded") - - until = response.headers.get("x-rate-limit-reset") - self.extractor.wait(until=until, seconds=None if until else 60) + elif rl and isinstance(rl, str) and rl.startswith("wait:"): + until = None + seconds = text.parse_float(rl.partition(":")[2]) or 60.0 + else: + until = response.headers.get("x-rate-limit-reset") + seconds = None if until else 60.0 + self.extractor.wait(until=until, seconds=seconds) def _process_tombstone(self, entry, tombstone): text = (tombstone.get("richText") or tombstone["text"])["text"]