improve 'extractor.request'

- add 'fatal' argument
- improve internal logic and flow
- raise known exception on error
- update exception hierarchy
This commit is contained in:
Mike Fährmann
2017-08-05 16:11:46 +02:00
parent dcd573806e
commit 915a0137de
12 changed files with 75 additions and 62 deletions

View File

@@ -62,7 +62,7 @@ class HentaifoundryUserExtractor(Extractor):
def get_job_metadata(self):
"""Collect metadata for extractor-job"""
url = self.url_base + self.artist + "?enterAgree=1"
response = self.session.get(url)
response = self.request(url, fatal=False)
if response.status_code == 404:
raise exception.NotFoundError("user")
page = response.text
@@ -150,7 +150,7 @@ class HentaifoundryImageExtractor(Extractor):
"""Collect metadata for an image"""
url = "https://www.hentai-foundry.com/pictures/user/{}/{}".format(
self.artist, self.index)
response = self.session.get(url + "?enterAgree=1")
response = self.request(url + "?enterAgree=1", fatal=False)
if response.status_code == 404:
raise exception.NotFoundError("image")
extr = text.extract