adjust value resolution for retries/timeout/verify options
This change introduces 'extractor.*.retries/timeout/verify' options as a general way to set these values for all HTTP requests. 'downloader.http.retries/timeout/verify' is a way to override these options for file downloads only and will fall back to 'extractor.*.…* values if they haven't been explicitly set. Also: downloader classes now take an extractor object as first argument instead of a requests.session.
This commit is contained in:
@@ -37,12 +37,9 @@ class Extractor():
|
||||
self._set_headers()
|
||||
self._set_cookies()
|
||||
self._set_proxies()
|
||||
self._retries = config.interpolate(
|
||||
("downloader", "http", "retries"), 5)
|
||||
self._timeout = config.interpolate(
|
||||
("downloader", "http", "timeout"), 30)
|
||||
self._verify = config.interpolate(
|
||||
("downloader", "http", "verify"), True)
|
||||
self._retries = self.config("retries", 5)
|
||||
self._timeout = self.config("timeout", 30)
|
||||
self._verify = self.config("verify", True)
|
||||
|
||||
def __iter__(self):
|
||||
return self.items()
|
||||
|
||||
Reference in New Issue
Block a user