add options to set maximum download rate

- -r/--limit-rate as cmdline option
- downloader.http.rate as config option

This implementation very roughly uses the idea of the token bucket
algorithm [1] and mostly uses Wget's approach [2] as inspiration.

[1] https://en.wikipedia.org/wiki/Token_bucket
[2] http://git.savannah.gnu.org/cgit/wget.git/tree/src/retr.c?h=v1.19.2&id=ba6b44f6745b14dce414761a8e4b35d31b176bba#n111

docs/configuration.rst

@@ -120,6 +120,20 @@ Description Alternate location for ``.part`` files.
 =========== =====
 
 
+downloader.http.rate
+--------------------
+=========== =====
+Type        ``string``
+Default     ``null``
+Examples    ``"32000"``, ``"500k"``, ``"2.5M"``
+Description Maximum download rate in bytes per second.
+
+            Possible values are valid integer or floating-point numbers
+            optionally followed by one of ``k``, ``m``. ``g``, ``t`` or ``p``.
+            These suffixes are case-insensitive.
+=========== =====
+
+
 downloader.http.retries
 -----------------------
 =========== =====

docs/gallery-dl.conf

@@ -7,6 +7,7 @@
         "part-directory": null,
         "http":
         {
+            "rate": null,
             "retries": 5,
             "timeout": 30,
             "verify": true

gallery_dl/downloader/http.py

@@ -8,6 +8,7 @@
 
 """Downloader module for http:// and https:// URLs"""
 
+import time
 import mimetypes
 from .common import DownloaderBase
 from .. import util, exception
@@ -22,6 +23,15 @@ class Downloader(DownloaderBase):
         self.retries = self.config("retries", 5)
         self.timeout = self.config("timeout", 30)
         self.verify = self.config("verify", True)
+        self.rate = self.config("rate")
+        self.chunk_size = 16384
+
+        if self.rate:
+            self.rate = util.parse_bytes(self.rate)
+            if not self.rate:
+                self.log.warning("Invalid rate limit specified")
+            elif self.rate < self.chunk_size:
+                self.chunk_size = self.rate
 
     def connect(self, url, offset):
         headers = {}
@@ -50,9 +60,21 @@ class Downloader(DownloaderBase):
         return offset, util.safe_int(size)
 
     def receive(self, file):
-        for data in self.response.iter_content(16384):
+        if self.rate:
+            total = 0            # total amount of bytes received
+            start = time.time()  # start time
+
+        for data in self.response.iter_content(self.chunk_size):
             file.write(data)
 
+            if self.rate:
+                total += len(data)
+                expected = total / self.rate  # expected elapsed time
+                delta = time.time() - start   # actual elapsed time since start
+                if delta < expected:
+                    # sleep if less time passed than expected
+                    time.sleep(expected - delta)
+
     def reset(self):
         if self.response:
             self.response.close()

gallery_dl/option.py

@@ -135,6 +135,11 @@ def build_parser():
     )
 
     downloader = parser.add_argument_group("Downloader Options")
+    downloader.add_argument(
+        "-r", "--limit-rate",
+        metavar="RATE", action=ConfigAction, dest="rate",
+        help="Maximum download rate (e.g. 500k or 2.5M)",
+    )
     downloader.add_argument(
         "-R", "--retries",
         metavar="RETRIES", action=ConfigAction, dest="retries", type=int,