[twitter] extend 'ratelimit' option (#5532)
allow waiting for a set amount of seconds
This commit is contained in:
@@ -3987,6 +3987,7 @@ Description
|
|||||||
|
|
||||||
* ``"abort"``: Raise an error and stop extraction
|
* ``"abort"``: Raise an error and stop extraction
|
||||||
* ``"wait"``: Wait until rate limit reset
|
* ``"wait"``: Wait until rate limit reset
|
||||||
|
* ``"wait:N"``: Wait for ``N`` seconds
|
||||||
|
|
||||||
|
|
||||||
extractor.twitter.relogin
|
extractor.twitter.relogin
|
||||||
@@ -3996,10 +3997,10 @@ Type
|
|||||||
Default
|
Default
|
||||||
``true``
|
``true``
|
||||||
Description
|
Description
|
||||||
| When receiving a "Could not authenticate you" error while logged in with
|
When receiving a "Could not authenticate you" error while logged in with
|
||||||
`username & passeword <extractor.*.username & .password_>`__,
|
`username & passeword <extractor.*.username & .password_>`__,
|
||||||
| refresh the current login session and
|
refresh the current login session and
|
||||||
try to continue from where it left off.
|
try to continue from where it left off.
|
||||||
|
|
||||||
|
|
||||||
extractor.twitter.locked
|
extractor.twitter.locked
|
||||||
|
|||||||
@@ -1709,11 +1709,16 @@ class TwitterAPI():
|
|||||||
variables["cursor"] = cursor
|
variables["cursor"] = cursor
|
||||||
|
|
||||||
def _handle_ratelimit(self, response):
|
def _handle_ratelimit(self, response):
|
||||||
if self.extractor.config("ratelimit") == "abort":
|
rl = self.extractor.config("ratelimit")
|
||||||
|
if rl == "abort":
|
||||||
raise exception.StopExtraction("Rate limit exceeded")
|
raise exception.StopExtraction("Rate limit exceeded")
|
||||||
|
elif rl and isinstance(rl, str) and rl.startswith("wait:"):
|
||||||
until = response.headers.get("x-rate-limit-reset")
|
until = None
|
||||||
self.extractor.wait(until=until, seconds=None if until else 60)
|
seconds = text.parse_float(rl.partition(":")[2]) or 60.0
|
||||||
|
else:
|
||||||
|
until = response.headers.get("x-rate-limit-reset")
|
||||||
|
seconds = None if until else 60.0
|
||||||
|
self.extractor.wait(until=until, seconds=seconds)
|
||||||
|
|
||||||
def _process_tombstone(self, entry, tombstone):
|
def _process_tombstone(self, entry, tombstone):
|
||||||
text = (tombstone.get("richText") or tombstone["text"])["text"]
|
text = (tombstone.get("richText") or tombstone["text"])["text"]
|
||||||
|
|||||||
Reference in New Issue
Block a user