[dl:http] implement dynamic download 'rate' limits (#7638)

docs/configuration.rst

@@ -5972,11 +5972,15 @@ Description
 downloader.*.rate
 -----------------
 Type
-    ``string``
+    * ``string``
+    * ``list`` with 2 ``strings``
 Default
     ``null``
 Example
-    ``"32000"``, ``"500k"``, ``"2.5M"``
+    * ``"32000"``
+    * ``"500k"``
+    * ``"1M - 2.5M"``
+    * ``["1M", "2.5M"]``
 Description
     Maximum download rate in bytes per second.
 
@@ -5984,6 +5988,10 @@ Description
     optionally followed by one of ``k``, ``m``. ``g``, ``t``, or ``p``.
     These suffixes are case-insensitive.
 
+    If given as a range, the maximum download rate
+    will be randomly chosen before each download.
+    (see `random.randint() <https://docs.python.org/3/library/random.html#random.randint>`_)
+
 
 downloader.*.retries
 --------------------

docs/options.md

@@ -77,7 +77,8 @@
     --no-check-certificate      Disable HTTPS certificate validation
 
 ## Downloader Options:
-    -r, --limit-rate RATE       Maximum download rate (e.g. 500k or 2.5M)
+    -r, --limit-rate RATE       Maximum download rate (e.g. 500k, 2.5M, or
+                                800k-2M)
     --chunk-size SIZE           Size of in-memory data chunks (default: 32k)
     --sleep SECONDS             Number of seconds to wait before each download.
                                 This can be either a constant value or a range

gallery_dl/downloader/http.py

@@ -68,14 +68,17 @@ class HttpDownloader(DownloaderBase):
                 chunk_size = 32768
             self.chunk_size = chunk_size
         if self.rate:
-            rate = text.parse_bytes(self.rate)
-            if rate:
-                if rate < self.chunk_size:
-                    self.chunk_size = rate
-                self.rate = rate
+            func = util.build_selection_func(self.rate, 0, text.parse_bytes)
+            value = func()
+            if value:
+                # wrong when func() returns from a range
+                if value < self.chunk_size:
+                    self.chunk_size = value
+                self.rate = func
                 self.receive = self._receive_rate
             else:
                 self.log.warning("Invalid rate limit (%r)", self.rate)
+                self.rate = False
         if self.progress is not None:
             self.receive = self._receive_rate
             if self.progress < 0.0:
@@ -361,7 +364,7 @@ class HttpDownloader(DownloaderBase):
             write(data)
 
     def _receive_rate(self, fp, content, bytes_total, bytes_start):
-        rate = self.rate
+        rate = self.rate() if self.rate else None
         write = fp.write
         progress = self.progress
 
@@ -382,7 +385,7 @@ class HttpDownloader(DownloaderBase):
                         int(bytes_downloaded / time_elapsed),
                     )
 
-            if rate:
+            if rate is not None:
                 time_expected = bytes_downloaded / rate
                 if time_expected > time_elapsed:
                     time.sleep(time_expected - time_elapsed)

gallery_dl/option.py

@@ -485,7 +485,7 @@ def build_parser():
     downloader.add_argument(
         "-r", "--limit-rate",
         dest="rate", metavar="RATE", action=ConfigAction,
-        help="Maximum download rate (e.g. 500k or 2.5M)",
+        help="Maximum download rate (e.g. 500k, 2.5M, or 800k-2M)",
     )
     downloader.add_argument(
         "--chunk-size",

gallery_dl/ytdl.py

@@ -51,7 +51,12 @@ def construct_YoutubeDL(module, obj, user_opts, system_opts=None):
     if opts.get("updatetime") is None:
         opts["updatetime"] = config("mtime", True)
     if opts.get("ratelimit") is None:
-        opts["ratelimit"] = text.parse_bytes(config("rate"), None)
+        rate = config("rate")
+        if rate:
+            func = util.build_selection_func(rate, 0, text.parse_bytes)
+            opts["ratelimit"] = func() or None
+        else:
+            opts["ratelimit"] = None
     if opts.get("min_filesize") is None:
         opts["min_filesize"] = text.parse_bytes(config("filesize-min"), None)
     if opts.get("max_filesize") is None:

test/test_downloader.py

@@ -163,7 +163,7 @@ class TestDownloaderConfig(unittest.TestCase):
         self.assertEqual(dl.timeout, 10)
         self.assertEqual(dl.verify, False)
         self.assertEqual(dl.mtime, False)
-        self.assertEqual(dl.rate, 42)
+        self.assertEqual(dl.rate(), 42)
         self.assertEqual(dl.part, False)