adjust value resolution for retries/timeout/verify options

This change introduces 'extractor.*.retries/timeout/verify' options
as a general way to set these values for all HTTP requests.

'downloader.http.retries/timeout/verify' is a way to override these
options for file downloads only and will fall back to 'extractor.*.…*
values if they haven't been explicitly set.

Also: downloader classes now take an extractor object as first argument
instead of a requests.session.
This commit is contained in:
Mike Fährmann
2018-10-06 19:59:19 +02:00
parent f647f5d9c3
commit 4a348990f4
7 changed files with 23 additions and 28 deletions

View File

@@ -15,8 +15,6 @@ import unittest
import threading
import http.server
import requests
import gallery_dl.downloader as downloader
import gallery_dl.extractor as extractor
import gallery_dl.config as config
@@ -29,6 +27,7 @@ class TestDownloaderBase(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.extractor = extractor.find("test:")
cls.dir = tempfile.TemporaryDirectory()
cls.fnum = 0
config.set(("base-directory",), cls.dir.name)
@@ -49,7 +48,7 @@ class TestDownloaderBase(unittest.TestCase):
"name": name,
"extension": extension,
}
pathfmt = PathFormat(extractor.find("test:"))
pathfmt = PathFormat(cls.extractor)
pathfmt.set_directory(kwdict)
pathfmt.set_keywords(kwdict)
@@ -90,8 +89,7 @@ class TestHTTPDownloader(TestDownloaderBase):
@classmethod
def setUpClass(cls):
TestDownloaderBase.setUpClass()
cls.downloader = downloader.find("http")(
requests.session(), NullOutput())
cls.downloader = downloader.find("http")(cls.extractor, NullOutput())
port = 8088
cls.address = "http://127.0.0.1:{}".format(port)
@@ -128,7 +126,7 @@ class TestTextDownloader(TestDownloaderBase):
@classmethod
def setUpClass(cls):
TestDownloaderBase.setUpClass()
cls.downloader = downloader.find("text")(None, NullOutput())
cls.downloader = downloader.find("text")(cls.extractor, NullOutput())
def test_text_download(self):
self._run_test("text:foobar", None, "foobar", "txt", "txt")
@@ -146,8 +144,8 @@ class TestTextDownloader(TestDownloaderBase):
class FakeDownloader(DownloaderBase):
scheme = "fake"
def __init__(self, session, output):
DownloaderBase.__init__(self, session, output)
def __init__(self, extractor, output):
DownloaderBase.__init__(self, extractor, output)
def connect(self, url, offset):
pass