[twitter] handle API rate limits (#526)

gallery_dl/cache.py

@@ -37,7 +37,7 @@ class CacheDecorator():
     def update(self, key, value):
         self.cache[key] = value
 
-    def invalidate(self, key):
+    def invalidate(self, key=""):
         try:
             del self.cache[key]
         except KeyError:

gallery_dl/extractor/twitter.py

@@ -57,6 +57,8 @@ class TwitterExtractor(Extractor):
                         self.root, data["tweet_id"])
                 else:
                     url = self._video_from_tweet(data["tweet_id"])
+                    if not url:
+                        continue
                     ext = text.ext_from_url(url)
                     if ext == "m3u8":
                         url = "ytdl:" + url
@@ -183,19 +185,28 @@ class TwitterExtractor(Extractor):
         if self.logged_in:
             headers["x-twitter-auth-type"] = "OAuth2Session"
         else:
-            token = self._guest_token(headers)
+            token = _guest_token(self, headers)
             cookies = {"gt": token}
             headers["x-guest-token"] = token
 
-        data = self.request(url, cookies=cookies, headers=headers).json()
+        response = self.request(
-        return data["track"]["playbackUrl"]
+            url, cookies=cookies, headers=headers, fatal=None)
 
-    @memcache()
+        if response.status_code == 429 or \
-    def _guest_token(self, headers):
+                response.headers.get("x-rate-limit-remaining") == "0":
-        return self.request(
+            if self.logged_in:
-            "https://api.twitter.com/1.1/guest/activate.json",
+                reset = response.headers.get("x-rate-limit-reset")
-            method="POST", headers=headers,
+                self.wait(until=reset, reason="rate limit reset")
-        ).json().get("guest_token")
+            else:
+                _guest_token.invalidate()
+            return self._video_from_tweet(tweet_id)
+
+        elif response.status_code >= 400:
+            self.log.warning("Unable to fetch video data for %s ('%s %s')",
+                             tweet_id, response.status_code, response.reason)
+            return None
+
+        return response.json()["track"]["playbackUrl"]
 
     def _tweets_from_api(self, url, max_position=None):
         params = {
@@ -357,3 +368,11 @@ class TwitterTweetExtractor(TwitterExtractor):
         end = page.index('class="js-tweet-stats-container')
         beg = page.rindex('<div class="tweet ', 0, end)
         return (page[beg:end],)
+
+
+@memcache()
+def _guest_token(extr, headers):
+    return extr.request(
+        "https://api.twitter.com/1.1/guest/activate.json",
+        method="POST", headers=headers,
+    ).json().get("guest_token")