diff --git a/docs/configuration.rst b/docs/configuration.rst index 546f45b0..d76fc856 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -3009,7 +3009,7 @@ Description Any file smaller/larger than this limit will not be downloaded. Possible values are valid integer or floating-point numbers - optionally followed by one of ``k``, ``m``. ``g``, ``t`` or ``p``. + optionally followed by one of ``k``, ``m``. ``g``, ``t``, or ``p``. These suffixes are case-insensitive. @@ -3079,7 +3079,7 @@ Description Maximum download rate in bytes per second. Possible values are valid integer or floating-point numbers - optionally followed by one of ``k``, ``m``. ``g``, ``t`` or ``p``. + optionally followed by one of ``k``, ``m``. ``g``, ``t``, or ``p``. These suffixes are case-insensitive. @@ -3140,6 +3140,22 @@ Description contains JPEG/JFIF data. +downloader.http.chunk-size +-------------------------- +Type + ``integer`` or ``string`` +Default + ``32768`` +Example + ``"50k"``, ``"0.8M"`` +Description + Number of bytes per downloaded chunk. + + Possible values are integer numbers + optionally followed by one of ``k``, ``m``. ``g``, ``t``, or ``p``. + These suffixes are case-insensitive. + + downloader.http.headers ----------------------- Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 1fcbb3b2..9349398e 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -381,6 +381,7 @@ "http": { "adjust-extensions": true, + "chunk-size": 32768, "headers": null }, diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 4e074665..9ed3f720 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -27,10 +27,10 @@ class HttpDownloader(DownloaderBase): def __init__(self, job): DownloaderBase.__init__(self, job) extractor = job.extractor - self.chunk_size = 16384 self.downloading = False self.adjust_extension = self.config("adjust-extensions", True) + self.chunk_size = self.config("chunk-size", 32768) self.progress = self.config("progress", 3.0) self.headers = self.config("headers") self.minsize = self.config("filesize-min") @@ -55,6 +55,13 @@ class HttpDownloader(DownloaderBase): self.log.warning( "Invalid maximum file size (%r)", self.maxsize) self.maxsize = maxsize + if isinstance(self.chunk_size, str): + chunk_size = text.parse_bytes(self.chunk_size) + if not chunk_size: + self.log.warning( + "Invalid chunk size (%r)", self.chunk_size) + chunk_size = 32768 + self.chunk_size = chunk_size if self.rate: rate = text.parse_bytes(self.rate) if rate: