[twitter] extend 'ratelimit' option (#5532)

allow waiting for a set amount of seconds
This commit is contained in:
Mike Fährmann
2024-06-06 01:18:08 +02:00
parent 2cab87c5b6
commit 162d4269ec
2 changed files with 14 additions and 8 deletions

View File

@@ -3987,6 +3987,7 @@ Description
* ``"abort"``: Raise an error and stop extraction * ``"abort"``: Raise an error and stop extraction
* ``"wait"``: Wait until rate limit reset * ``"wait"``: Wait until rate limit reset
* ``"wait:N"``: Wait for ``N`` seconds
extractor.twitter.relogin extractor.twitter.relogin
@@ -3996,10 +3997,10 @@ Type
Default Default
``true`` ``true``
Description Description
| When receiving a "Could not authenticate you" error while logged in with When receiving a "Could not authenticate you" error while logged in with
`username & passeword <extractor.*.username & .password_>`__, `username & passeword <extractor.*.username & .password_>`__,
| refresh the current login session and refresh the current login session and
try to continue from where it left off. try to continue from where it left off.
extractor.twitter.locked extractor.twitter.locked

View File

@@ -1709,11 +1709,16 @@ class TwitterAPI():
variables["cursor"] = cursor variables["cursor"] = cursor
def _handle_ratelimit(self, response): def _handle_ratelimit(self, response):
if self.extractor.config("ratelimit") == "abort": rl = self.extractor.config("ratelimit")
if rl == "abort":
raise exception.StopExtraction("Rate limit exceeded") raise exception.StopExtraction("Rate limit exceeded")
elif rl and isinstance(rl, str) and rl.startswith("wait:"):
until = response.headers.get("x-rate-limit-reset") until = None
self.extractor.wait(until=until, seconds=None if until else 60) seconds = text.parse_float(rl.partition(":")[2]) or 60.0
else:
until = response.headers.get("x-rate-limit-reset")
seconds = None if until else 60.0
self.extractor.wait(until=until, seconds=seconds)
def _process_tombstone(self, entry, tombstone): def _process_tombstone(self, entry, tombstone):
text = (tombstone.get("richText") or tombstone["text"])["text"] text = (tombstone.get("richText") or tombstone["text"])["text"]