share extractor and downloader sessions

There was never any "good" reason for the strict separation
between extractors and downloaders. This change allows for
reduced resource usage (probably unnoticeable) and less lines
of code at the "cost" of tighter coupling.
This commit is contained in:
Mike Fährmann
2017-06-30 19:38:14 +02:00
parent 4414aefe97
commit 58e95a7487
9 changed files with 13 additions and 62 deletions

View File

@@ -6,10 +6,9 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Downloader module for http:// and https:// urls"""
"""Downloader module for http:// and https:// URLs"""
import time
import requests
import requests.exceptions as rexcepts
import mimetypes
import logging
@@ -24,9 +23,9 @@ class Downloader(BasicDownloader):
retries = config.interpolate(("downloader", "http", "retries",), 5)
timeout = config.interpolate(("downloader", "http", "timeout",), None)
def __init__(self, output):
def __init__(self, session, output):
BasicDownloader.__init__(self)
self.session = requests.session()
self.session = session
self.out = output
def download_impl(self, url, pathfmt):
@@ -96,17 +95,3 @@ class Downloader(BasicDownloader):
# output for unrecoverable errors
self.out.error(pathfmt.path, msg, tries, 0)
def set_headers(self, headers):
"""Set headers for http requests"""
self.set_dict(self.session.headers, headers)
def set_cookies(self, cookies):
"""Set cookies for http requests"""
self.set_dict(self.session.cookies, cookies)
@staticmethod
def set_dict(dest, src):
"""Copy the contents of dictionary 'src' to 'dest'"""
dest.clear()
dest.update(src)