fix (sub)category-transfer for DownloadJob instances (#41)
... and extend "parent" parameters to TestJob- and DataJob-classes as well.
This commit is contained in:
@@ -135,7 +135,7 @@ class DownloadJob(Job):
|
|||||||
"""Download images into appropriate directory/filename locations"""
|
"""Download images into appropriate directory/filename locations"""
|
||||||
|
|
||||||
def __init__(self, url, parent=None):
|
def __init__(self, url, parent=None):
|
||||||
Job.__init__(self, url)
|
Job.__init__(self, url, parent)
|
||||||
self.pathfmt = util.PathFormat(self.extractor)
|
self.pathfmt = util.PathFormat(self.extractor)
|
||||||
self.downloaders = {}
|
self.downloaders = {}
|
||||||
self.out = output.select()
|
self.out = output.select()
|
||||||
@@ -270,8 +270,8 @@ class TestJob(DownloadJob):
|
|||||||
"""Update SHA1 hash"""
|
"""Update SHA1 hash"""
|
||||||
self.hashobj.update(content)
|
self.hashobj.update(content)
|
||||||
|
|
||||||
def __init__(self, url, content=False):
|
def __init__(self, url, parent=None, content=False):
|
||||||
DownloadJob.__init__(self, url)
|
DownloadJob.__init__(self, url, parent)
|
||||||
self.content = content
|
self.content = content
|
||||||
self.urllist = []
|
self.urllist = []
|
||||||
self.hash_url = hashlib.sha1()
|
self.hash_url = hashlib.sha1()
|
||||||
@@ -316,8 +316,8 @@ class TestJob(DownloadJob):
|
|||||||
class DataJob(Job):
|
class DataJob(Job):
|
||||||
"""Collect extractor results and dump them"""
|
"""Collect extractor results and dump them"""
|
||||||
|
|
||||||
def __init__(self, url, file=sys.stdout):
|
def __init__(self, url, parent=None, file=sys.stdout):
|
||||||
Job.__init__(self, url)
|
Job.__init__(self, url, parent)
|
||||||
self.file = file
|
self.file = file
|
||||||
self.data = []
|
self.data = []
|
||||||
self.ensure_ascii = config.get(("output", "ascii"), True)
|
self.ensure_ascii = config.get(("output", "ascii"), True)
|
||||||
|
|||||||
@@ -30,4 +30,4 @@ for urls, extr in tests:
|
|||||||
name = "%s-%s-%d.json" % (extr.category, extr.subcategory, i)
|
name = "%s-%s-%d.json" % (extr.category, extr.subcategory, i)
|
||||||
print(name)
|
print(name)
|
||||||
with open(os.path.join(path, name), "w") as outfile:
|
with open(os.path.join(path, name), "w") as outfile:
|
||||||
job.DataJob(url, outfile).run()
|
job.DataJob(url, file=outfile).run()
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ class TestExtractors(unittest.TestCase):
|
|||||||
|
|
||||||
def _run_test(self, extr, url, result):
|
def _run_test(self, extr, url, result):
|
||||||
content = "content" in result if result else False
|
content = "content" in result if result else False
|
||||||
tjob = job.TestJob(url, content)
|
tjob = job.TestJob(url, content=content)
|
||||||
self.assertEqual(extr, tjob.extractor.__class__)
|
self.assertEqual(extr, tjob.extractor.__class__)
|
||||||
if not result:
|
if not result:
|
||||||
return
|
return
|
||||||
|
|||||||
Reference in New Issue
Block a user