implement 'downloader' options per extractor category
by setting options inside 'http' or 'ytdl' inside extractor options
or inside subcategory options
{
"extractor": {
"mastodon": {
"http": {
"rate": "10k"
}
},
"mastodon.social": {
"http": {
"rate": "100k"
}
}
},
"downloader": {
"rate": "100m"
}
}
Sets download speed to
- 10k for mastodon.social URLs
- 100k for mastodon sites in general
- 100m for all other sites
This commit is contained in:
@@ -20,7 +20,6 @@ import tempfile
|
||||
import threading
|
||||
import http.server
|
||||
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from gallery_dl import downloader, extractor, output, config, path # noqa E402
|
||||
from gallery_dl.downloader.http import MIME_TYPES, SIGNATURE_CHECKS # noqa E402
|
||||
@@ -55,6 +54,9 @@ class TestDownloaderModule(unittest.TestCase):
|
||||
else:
|
||||
del sys.modules["youtube_dl"]
|
||||
|
||||
def setUp(self):
|
||||
downloader._cache.clear()
|
||||
|
||||
def tearDown(self):
|
||||
downloader._cache.clear()
|
||||
|
||||
@@ -107,6 +109,63 @@ class TestDownloaderModule(unittest.TestCase):
|
||||
self.assertEqual(import_module.call_count, 1)
|
||||
|
||||
|
||||
class TestDownloaderConfig(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
config.clear()
|
||||
|
||||
def tearDown(self):
|
||||
config.clear()
|
||||
|
||||
def test_default_http(self):
|
||||
job = FakeJob()
|
||||
extr = job.extractor
|
||||
dl = downloader.find("http")(job)
|
||||
|
||||
self.assertEqual(dl.adjust_extension, True)
|
||||
self.assertEqual(dl.chunk_size, 32768)
|
||||
self.assertEqual(dl.metadata, None)
|
||||
self.assertEqual(dl.progress, 3.0)
|
||||
self.assertEqual(dl.validate, True)
|
||||
self.assertEqual(dl.headers, None)
|
||||
self.assertEqual(dl.minsize, None)
|
||||
self.assertEqual(dl.maxsize, None)
|
||||
self.assertEqual(dl.mtime, True)
|
||||
self.assertEqual(dl.rate, None)
|
||||
self.assertEqual(dl.part, True)
|
||||
self.assertEqual(dl.partdir, None)
|
||||
|
||||
self.assertIs(dl.interval_429, extr._interval_429)
|
||||
self.assertIs(dl.retry_codes, extr._retry_codes)
|
||||
self.assertIs(dl.retries, extr._retries)
|
||||
self.assertIs(dl.timeout, extr._timeout)
|
||||
self.assertIs(dl.proxies, extr._proxies)
|
||||
self.assertIs(dl.verify, extr._verify)
|
||||
|
||||
def test_config_http(self):
|
||||
config.set((), "rate", 42)
|
||||
config.set((), "mtime", False)
|
||||
config.set((), "headers", {"foo": "bar"})
|
||||
config.set(("downloader",), "retries", -1)
|
||||
config.set(("downloader", "http"), "filesize-min", "10k")
|
||||
config.set(("extractor", "generic"), "verify", False)
|
||||
config.set(("extractor", "generic", "example.org"), "timeout", 10)
|
||||
config.set(("extractor", "generic", "http"), "rate", "1k")
|
||||
config.set(
|
||||
("extractor", "generic", "example.org", "http"), "headers", {})
|
||||
|
||||
job = FakeJob()
|
||||
dl = downloader.find("http")(job)
|
||||
|
||||
self.assertEqual(dl.headers, {})
|
||||
self.assertEqual(dl.minsize, 10240)
|
||||
self.assertEqual(dl.retries, float("inf"))
|
||||
self.assertEqual(dl.timeout, 10)
|
||||
self.assertEqual(dl.verify, False)
|
||||
self.assertEqual(dl.mtime, False)
|
||||
self.assertEqual(dl.rate, 1024)
|
||||
|
||||
|
||||
class TestDownloaderBase(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user