reduce wait time growth rate from exponential to linear
Waiting for 2**N seconds after each error grows too fast. Simply waiting N seconds seems far more reasonable.
This commit is contained in:
@@ -723,9 +723,6 @@ extractor.deviantart.wait-min
|
|||||||
Type ``integer``
|
Type ``integer``
|
||||||
Default ``0``
|
Default ``0``
|
||||||
Description Minimum wait time in seconds before API requests.
|
Description Minimum wait time in seconds before API requests.
|
||||||
|
|
||||||
Note: This value will internally be rounded up
|
|
||||||
to the next power of 2.
|
|
||||||
=========== =====
|
=========== =====
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -87,7 +87,7 @@ class HttpDownloader(DownloaderBase):
|
|||||||
self.log.warning("%s (%s/%s)", msg, tries, self.retries+1)
|
self.log.warning("%s (%s/%s)", msg, tries, self.retries+1)
|
||||||
if tries > self.retries:
|
if tries > self.retries:
|
||||||
return False
|
return False
|
||||||
time.sleep(min(2 ** (tries-1), 1800))
|
time.sleep(tries)
|
||||||
tries += 1
|
tries += 1
|
||||||
|
|
||||||
headers = {}
|
headers = {}
|
||||||
|
|||||||
@@ -123,7 +123,7 @@ class Extractor():
|
|||||||
self.log.debug("%s (%s/%s)", msg, tries, retries+1)
|
self.log.debug("%s (%s/%s)", msg, tries, retries+1)
|
||||||
if tries > retries:
|
if tries > retries:
|
||||||
break
|
break
|
||||||
time.sleep(min(2 ** (tries-1), 1800))
|
time.sleep(tries)
|
||||||
tries += 1
|
tries += 1
|
||||||
|
|
||||||
raise exception.HttpError(msg, response)
|
raise exception.HttpError(msg, response)
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ from ..cache import cache, memcache
|
|||||||
import collections
|
import collections
|
||||||
import itertools
|
import itertools
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import math
|
|
||||||
import time
|
import time
|
||||||
import re
|
import re
|
||||||
|
|
||||||
@@ -837,8 +836,7 @@ class DeviantartOAuthAPI():
|
|||||||
self.log = extractor.log
|
self.log = extractor.log
|
||||||
self.headers = {}
|
self.headers = {}
|
||||||
|
|
||||||
delay = extractor.config("wait-min", 0)
|
self.delay = extractor.config("wait-min", 0)
|
||||||
self.delay = math.ceil(math.log2(delay)) if delay >= 1 else -1
|
|
||||||
self.delay_min = max(2, self.delay)
|
self.delay_min = max(2, self.delay)
|
||||||
|
|
||||||
self.mature = extractor.config("mature", "true")
|
self.mature = extractor.config("mature", "true")
|
||||||
@@ -993,8 +991,8 @@ class DeviantartOAuthAPI():
|
|||||||
"""Call an API endpoint"""
|
"""Call an API endpoint"""
|
||||||
url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint
|
url = "https://www.deviantart.com/api/v1/oauth2/" + endpoint
|
||||||
while True:
|
while True:
|
||||||
if self.delay >= 0:
|
if self.delay:
|
||||||
time.sleep(2 ** self.delay)
|
time.sleep(self.delay)
|
||||||
|
|
||||||
self.authenticate(None if public else self.refresh_token_key)
|
self.authenticate(None if public else self.refresh_token_key)
|
||||||
response = self.extractor.request(
|
response = self.extractor.request(
|
||||||
@@ -1015,9 +1013,9 @@ class DeviantartOAuthAPI():
|
|||||||
msg = "API responded with {} {}".format(
|
msg = "API responded with {} {}".format(
|
||||||
status, response.reason)
|
status, response.reason)
|
||||||
if status == 429:
|
if status == 429:
|
||||||
if self.delay < 9:
|
if self.delay < 30:
|
||||||
self.delay += 1
|
self.delay += 1
|
||||||
self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay)
|
self.log.warning("%s. Using %ds delay.", msg, self.delay)
|
||||||
else:
|
else:
|
||||||
self.log.error(msg)
|
self.log.error(msg)
|
||||||
return data
|
return data
|
||||||
|
|||||||
Reference in New Issue
Block a user