From 1bf9f52c997fa919eeb94d8a28c70afc2e42a18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Tue, 4 Jul 2023 18:17:32 +0200 Subject: [PATCH] [twitter] add 'ratelimit' option (#4251) --- docs/configuration.rst | 13 +++++++++++++ gallery_dl/extractor/twitter.py | 3 +++ 2 files changed, 16 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 0a2085e9..2608d2ab 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -3183,6 +3183,19 @@ Description a quoted (original) Tweet when it sees the Tweet which quotes it. +extractor.twitter.ratelimit +--------------------------- +Type + ``string`` +Default + ``"wait"`` +Description + Selects how to handle exceeding the API rate limit. + + * ``"abort"``: Raise an error and stop extraction + * ``"wait"``: Wait until rate limit reset + + extractor.twitter.replies ------------------------- Type diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index b3064cc9..c0b4ab61 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -1439,6 +1439,9 @@ class TwitterAPI(): if response.status_code == 429: # rate limit exceeded + if self.extractor.config("ratelimit") == "abort": + raise exception.StopExtraction("Rate limit exceeded") + until = response.headers.get("x-rate-limit-reset") seconds = None if until else 60 self.extractor.wait(until=until, seconds=seconds)