add 'sleep-extractor' option (closes #964)
(would have been nice if this were possible without code duplication)
This commit is contained in:
@@ -208,6 +208,16 @@ Description Number of seconds to sleep before each download.
|
|||||||
=========== =====
|
=========== =====
|
||||||
|
|
||||||
|
|
||||||
|
extractor.*.sleep-extractor
|
||||||
|
---------------------------
|
||||||
|
=========== =====
|
||||||
|
Type ``float``
|
||||||
|
Default ``0``
|
||||||
|
Description Number of seconds to sleep before handling an input URL,
|
||||||
|
i.e. before starting a new extractor.
|
||||||
|
=========== =====
|
||||||
|
|
||||||
|
|
||||||
extractor.*.username & .password
|
extractor.*.username & .password
|
||||||
--------------------------------
|
--------------------------------
|
||||||
=========== =====
|
=========== =====
|
||||||
|
|||||||
@@ -59,6 +59,9 @@ class Job():
|
|||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""Execute or run the job"""
|
"""Execute or run the job"""
|
||||||
|
sleep = self.extractor.config("sleep-extractor")
|
||||||
|
if sleep:
|
||||||
|
time.sleep(sleep)
|
||||||
try:
|
try:
|
||||||
log = self.extractor.log
|
log = self.extractor.log
|
||||||
for msg in self.extractor:
|
for msg in self.extractor:
|
||||||
@@ -586,6 +589,10 @@ class DataJob(Job):
|
|||||||
self.filter = (lambda x: x) if private else util.filter_dict
|
self.filter = (lambda x: x) if private else util.filter_dict
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
sleep = self.extractor.config("sleep-extractor")
|
||||||
|
if sleep:
|
||||||
|
time.sleep(sleep)
|
||||||
|
|
||||||
# collect data
|
# collect data
|
||||||
try:
|
try:
|
||||||
for msg in self.extractor:
|
for msg in self.extractor:
|
||||||
|
|||||||
Reference in New Issue
Block a user