[tests:job] add tests for DataJob 'resolve'
This commit is contained in:
@@ -432,6 +432,38 @@ class TestDataJob(TestJob):
|
|||||||
tjob = self.jobclass(extr)
|
tjob = self.jobclass(extr)
|
||||||
tjob._init()
|
tjob._init()
|
||||||
|
|
||||||
|
def test_resolve(self):
|
||||||
|
extr = TestExtractorParent.from_url("test:parent:3")
|
||||||
|
tjob = self.jobclass(extr, file=None, resolve=0)
|
||||||
|
tjob.run()
|
||||||
|
self.assertEqual(len(tjob.data_urls), 3)
|
||||||
|
for url in tjob.data_urls:
|
||||||
|
self.assertEqual(url, "test:parent:2")
|
||||||
|
|
||||||
|
extr = TestExtractorParent.from_url("test:parent:3")
|
||||||
|
tjob = self.jobclass(extr, file=None, resolve=1)
|
||||||
|
tjob.run()
|
||||||
|
self.assertEqual(len(tjob.data_urls), 9)
|
||||||
|
for url in tjob.data_urls:
|
||||||
|
self.assertEqual(url, "test:parent:1")
|
||||||
|
|
||||||
|
extr = TestExtractorParent.from_url("test:parent")
|
||||||
|
tjob = self.jobclass(extr, file=None, resolve=64)
|
||||||
|
tjob.run()
|
||||||
|
self.assertEqual(len(tjob.data_urls), 9)
|
||||||
|
for url in tjob.data_urls:
|
||||||
|
self.assertRegex(url, r"^https://example.org/\d\.jpg$")
|
||||||
|
|
||||||
|
extr = TestExtractorParent.from_url("test:parent:1")
|
||||||
|
tjob = self.jobclass(extr, file=None, resolve=64)
|
||||||
|
tjob.run()
|
||||||
|
self.assertEqual(len(tjob.data_urls), 27)
|
||||||
|
|
||||||
|
extr = TestExtractorParent.from_url("test:parent:2")
|
||||||
|
tjob = self.jobclass(extr, file=None, resolve=64)
|
||||||
|
tjob.run()
|
||||||
|
self.assertEqual(len(tjob.data_urls), 81)
|
||||||
|
|
||||||
|
|
||||||
class TestExtractor(Extractor):
|
class TestExtractor(Extractor):
|
||||||
category = "test_category"
|
category = "test_category"
|
||||||
@@ -469,16 +501,22 @@ class TestExtractor(Extractor):
|
|||||||
class TestExtractorParent(Extractor):
|
class TestExtractorParent(Extractor):
|
||||||
category = "test_category"
|
category = "test_category"
|
||||||
subcategory = "test_subcategory_parent"
|
subcategory = "test_subcategory_parent"
|
||||||
pattern = r"test:parent"
|
pattern = r"test:parent(:\d+)?"
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
url = "test:child"
|
level = self.groups[0]
|
||||||
|
if level in {None, ":0"}:
|
||||||
|
url = "test:child"
|
||||||
|
extr = TestExtractor
|
||||||
|
else:
|
||||||
|
url = f"test:parent:{int(level[1:])-1}"
|
||||||
|
extr = TestExtractorParent
|
||||||
|
|
||||||
for i in range(11, 14):
|
for i in range(11, 14):
|
||||||
yield Message.Queue, url, {
|
yield Message.Queue, url, {
|
||||||
"num" : i,
|
"num" : i,
|
||||||
"tags": ["abc", "def"],
|
"tags": ["abc", "def"],
|
||||||
"_extractor": TestExtractor,
|
"_extractor": extr,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user