share extractor and downloader sessions

There was never any "good" reason for the strict separation
between extractors and downloaders. This change allows for
reduced resource usage (probably unnoticeable) and less lines
of code at the "cost" of tighter coupling.
This commit is contained in:
Mike Fährmann
2017-06-30 19:38:14 +02:00
parent 4414aefe97
commit 58e95a7487
9 changed files with 13 additions and 62 deletions

View File

@@ -50,9 +50,8 @@ class ExhentaiGalleryExtractor(Extractor):
def items(self):
self.login()
self.setup_headers()
yield Message.Version, 1
yield Message.Headers, self.setup_headers()
yield Message.Cookies, self.session.cookies
url = "{}/g/{}/{}/".format(self.root, self.gid, self.token)
response = self.session.get(url)
@@ -76,14 +75,9 @@ class ExhentaiGalleryExtractor(Extractor):
"""Initialize headers"""
self.session.headers.update({
"User-Agent": "Mozilla/5.0",
"Accept": "text/html,application/xhtml+xml,"
"application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Referer": self.root + "/",
})
headers = self.session.headers.copy()
headers["Accept"] = "image/png,image/*;q=0.8,*/*;q=0.5"
return headers
def get_job_metadata(self, page):
"""Collect metadata for extractor-job"""