fix issues with 'Extractor.finalize()'
- prevent crash in InstagramUserExtractor (#4359) - call it at the end of every DownloadJob - add it to tests
This commit is contained in:
@@ -36,7 +36,6 @@ class Extractor():
|
|||||||
browser = None
|
browser = None
|
||||||
root = ""
|
root = ""
|
||||||
test = None
|
test = None
|
||||||
finalize = None
|
|
||||||
request_interval = 0.0
|
request_interval = 0.0
|
||||||
request_interval_min = 0.0
|
request_interval_min = 0.0
|
||||||
request_timestamp = 0.0
|
request_timestamp = 0.0
|
||||||
@@ -66,6 +65,9 @@ class Extractor():
|
|||||||
self._init()
|
self._init()
|
||||||
self.initialize = util.noop
|
self.initialize = util.noop
|
||||||
|
|
||||||
|
def finalize(self):
|
||||||
|
pass
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
yield Message.Version, 1
|
yield Message.Version, 1
|
||||||
|
|
||||||
|
|||||||
@@ -405,6 +405,9 @@ class InstagramUserExtractor(InstagramExtractor):
|
|||||||
def initialize(self):
|
def initialize(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def finalize(self):
|
||||||
|
pass
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
base = "{}/{}/".format(self.root, self.item)
|
base = "{}/{}/".format(self.root, self.item)
|
||||||
stories = "{}/stories/{}/".format(self.root, self.item)
|
stories = "{}/stories/{}/".format(self.root, self.item)
|
||||||
|
|||||||
@@ -140,8 +140,7 @@ class Job():
|
|||||||
log.info("No results for %s", extractor.url)
|
log.info("No results for %s", extractor.url)
|
||||||
finally:
|
finally:
|
||||||
self.handle_finalize()
|
self.handle_finalize()
|
||||||
if extractor.finalize:
|
extractor.finalize()
|
||||||
extractor.finalize()
|
|
||||||
|
|
||||||
return self.status
|
return self.status
|
||||||
|
|
||||||
|
|||||||
@@ -138,6 +138,7 @@ class TestExtractorModule(unittest.TestCase):
|
|||||||
for test in cls._get_tests():
|
for test in cls._get_tests():
|
||||||
extr = cls.from_url(test[0])
|
extr = cls.from_url(test[0])
|
||||||
extr.initialize()
|
extr.initialize()
|
||||||
|
extr.finalize()
|
||||||
break
|
break
|
||||||
|
|
||||||
def test_docstrings(self):
|
def test_docstrings(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user