allow specifying a minimum/maximum for 'sleep-*' options (#1835)

for example '"sleep-request": [5.0, 10.0]' to wait between 5 and 10
seconds between each HTTP request
This commit is contained in:
Mike Fährmann
2021-09-14 17:40:05 +02:00
parent bd845303ad
commit c9e6693530
4 changed files with 57 additions and 18 deletions

View File

@@ -54,13 +54,13 @@ class Extractor():
self._retries = self.config("retries", 4)
self._timeout = self.config("timeout", 30)
self._verify = self.config("verify", True)
self.request_interval = self.config(
"sleep-request", self.request_interval)
self._interval = util.build_duration_func(
self.config("sleep-request", self.request_interval),
self.request_interval_min,
)
if self._retries < 0:
self._retries = float("inf")
if self.request_interval < self.request_interval_min:
self.request_interval = self.request_interval_min
self._init_session()
self._init_cookies()
@@ -114,8 +114,8 @@ class Extractor():
response = None
tries = 1
if self.request_interval:
seconds = (self.request_interval -
if self._interval:
seconds = (self._interval() -
(time.time() - Extractor.request_timestamp))
if seconds > 0.0:
self.log.debug("Sleeping for %.5s seconds", seconds)