From d02f7c1118a8a98415befa241f995b39ede38ed4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sun, 5 Apr 2020 21:23:05 +0200
Subject: [PATCH] improve Extractor.wait()

- allow 'until' to be a datetime object
- do "time calculations" with UTC timestamps
- set a default 'reason'
---
 gallery_dl/extractor/common.py     | 24 +++++++++++++++++-------
 gallery_dl/extractor/deviantart.py |  2 +-
 gallery_dl/extractor/reddit.py     |  3 +--
 gallery_dl/extractor/tumblr.py     |  2 +-
 gallery_dl/extractor/twitter.py    |  3 +--
 5 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py
index 19ee182e..8986c997 100644
--- a/gallery_dl/extractor/common.py
+++ b/gallery_dl/extractor/common.py
@@ -122,23 +122,33 @@ class Extractor():
 
         raise exception.HttpError(msg)
 
-    def wait(self, *, seconds=None, until=None, reason=None, adjust=1):
-        now = datetime.datetime.now()
+    def wait(self, *, seconds=None, until=None, adjust=1.0,
+             reason="rate limit reset"):
+        now = time.time()
 
         if seconds:
             seconds = float(seconds)
-            until = now + datetime.timedelta(seconds=seconds)
+            until = now + seconds
         elif until:
-            until = datetime.datetime.fromtimestamp(float(until))
-            seconds = (until - now).total_seconds()
+            if isinstance(until, datetime.datetime):
+                # convert to UTC timestamp
+                epoch = datetime.datetime(1970, 1, 1)
+                until = (until - epoch) / datetime.timedelta(0, 1)
+            else:
+                until = float(until)
+            seconds = until - now
         else:
             raise ValueError("Either 'seconds' or 'until' is required")
 
+        seconds += adjust
+        if seconds <= 0.0:
+            return
+
         if reason:
-            t = until.time()
+            t = datetime.datetime.fromtimestamp(until).time()
             isotime = "{:02}:{:02}:{:02}".format(t.hour, t.minute, t.second)
             self.log.info("Waiting until %s for %s.", isotime, reason)
-        time.sleep(seconds + adjust)
+        time.sleep(seconds)
 
     def _get_auth_info(self):
         """Return authentication information as (username, password) tuple"""
diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index 897d9f70..0dd8f490 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -284,7 +284,7 @@ class DeviantartExtractor(Extractor):
                     b"Request blocked." not in response.content:
                 DeviantartExtractor._last_request = time.time()
                 return response
-            self.wait(seconds=180, reason="rate limit reset")
+            self.wait(seconds=180)
 
 
 class DeviantartUserExtractor(DeviantartExtractor):
diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py
index a312c1c2..d0232cce 100644
--- a/gallery_dl/extractor/reddit.py
+++ b/gallery_dl/extractor/reddit.py
@@ -313,8 +313,7 @@ class RedditAPI():
 
         remaining = response.headers.get("x-ratelimit-remaining")
         if remaining and float(remaining) < 2:
-            reset = response.headers["x-ratelimit-reset"]
-            self.extractor.wait(seconds=reset, reason="rate limit reset")
+            self.extractor.wait(seconds=response.headers["x-ratelimit-reset"])
             return self._call(endpoint, params)
 
         data = response.json()
diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py
index 0505fa94..7e99823c 100644
--- a/gallery_dl/extractor/tumblr.py
+++ b/gallery_dl/extractor/tumblr.py
@@ -418,7 +418,7 @@ class TumblrAPI(oauth.OAuth1API):
             reset = response.headers.get("x-ratelimit-perhour-reset")
             if reset:
                 self.log.info("Hourly API rate limit exceeded")
-                self.extractor.wait(seconds=reset, reason="rate limit reset")
+                self.extractor.wait(seconds=reset)
                 return self._call(blog, endpoint, params)
 
         raise exception.StopExtraction(data)
diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py
index cbb075c0..03ce3dd4 100644
--- a/gallery_dl/extractor/twitter.py
+++ b/gallery_dl/extractor/twitter.py
@@ -224,8 +224,7 @@ class TwitterExtractor(Extractor):
         if response.status_code == 429 or \
                 response.headers.get("x-rate-limit-remaining") == "0":
             if self.logged_in:
-                reset = response.headers.get("x-rate-limit-reset")
-                self.wait(until=reset, reason="rate limit reset")
+                self.wait(until=response.headers.get("x-rate-limit-reset"))
             else:
                 _guest_token.invalidate()
             return self._video_from_tweet(tweet_id)