improve Extractor.wait()

- allow 'until' to be a datetime object
- do "time calculations" with UTC timestamps
- set a default 'reason'

gallery_dl/extractor/common.py

@@ -122,23 +122,33 @@ class Extractor():
 
         raise exception.HttpError(msg)
 
-    def wait(self, *, seconds=None, until=None, reason=None, adjust=1):
-        now = datetime.datetime.now()
+    def wait(self, *, seconds=None, until=None, adjust=1.0,
+             reason="rate limit reset"):
+        now = time.time()
 
         if seconds:
             seconds = float(seconds)
-            until = now + datetime.timedelta(seconds=seconds)
+            until = now + seconds
         elif until:
-            until = datetime.datetime.fromtimestamp(float(until))
-            seconds = (until - now).total_seconds()
+            if isinstance(until, datetime.datetime):
+                # convert to UTC timestamp
+                epoch = datetime.datetime(1970, 1, 1)
+                until = (until - epoch) / datetime.timedelta(0, 1)
+            else:
+                until = float(until)
+            seconds = until - now
         else:
             raise ValueError("Either 'seconds' or 'until' is required")
 
+        seconds += adjust
+        if seconds <= 0.0:
+            return
+
         if reason:
-            t = until.time()
+            t = datetime.datetime.fromtimestamp(until).time()
             isotime = "{:02}:{:02}:{:02}".format(t.hour, t.minute, t.second)
             self.log.info("Waiting until %s for %s.", isotime, reason)
-        time.sleep(seconds + adjust)
+        time.sleep(seconds)
 
     def _get_auth_info(self):
         """Return authentication information as (username, password) tuple"""

gallery_dl/extractor/deviantart.py

@@ -284,7 +284,7 @@ class DeviantartExtractor(Extractor):
                     b"Request blocked." not in response.content:
                 DeviantartExtractor._last_request = time.time()
                 return response
-            self.wait(seconds=180, reason="rate limit reset")
+            self.wait(seconds=180)
 
 
 class DeviantartUserExtractor(DeviantartExtractor):

gallery_dl/extractor/reddit.py

@@ -313,8 +313,7 @@ class RedditAPI():
 
         remaining = response.headers.get("x-ratelimit-remaining")
         if remaining and float(remaining) < 2:
-            reset = response.headers["x-ratelimit-reset"]
-            self.extractor.wait(seconds=reset, reason="rate limit reset")
+            self.extractor.wait(seconds=response.headers["x-ratelimit-reset"])
             return self._call(endpoint, params)
 
         data = response.json()

gallery_dl/extractor/tumblr.py

@@ -418,7 +418,7 @@ class TumblrAPI(oauth.OAuth1API):
             reset = response.headers.get("x-ratelimit-perhour-reset")
             if reset:
                 self.log.info("Hourly API rate limit exceeded")
-                self.extractor.wait(seconds=reset, reason="rate limit reset")
+                self.extractor.wait(seconds=reset)
                 return self._call(blog, endpoint, params)
 
         raise exception.StopExtraction(data)

gallery_dl/extractor/twitter.py

@@ -224,8 +224,7 @@ class TwitterExtractor(Extractor):
         if response.status_code == 429 or \
                 response.headers.get("x-rate-limit-remaining") == "0":
             if self.logged_in:
-                reset = response.headers.get("x-rate-limit-reset")
-                self.wait(until=reset, reason="rate limit reset")
+                self.wait(until=response.headers.get("x-rate-limit-reset"))
             else:
                 _guest_token.invalidate()
             return self._video_from_tweet(tweet_id)