[downloader:http] add 'retry-codes' option (#3313)

docs/configuration.rst

@@ -3280,6 +3280,24 @@ Description
     Additional HTTP headers to send when downloading files,
 
 
+downloader.http.retry-codes
+---------------------------
+Type
+    ``list`` of ``integers``
+Default
+    ``[429]``
+Description
+    Additional `HTTP response status codes <https://developer.mozilla.org/en-US/docs/Web/HTTP/Status>`__
+    to retry a download on.
+
+    Codes ``200``, ``206``, and ``416`` (when resuming a `partial <downloader.*.part_>`__
+    download) will never be retried and always count as success,
+    regardless of this option.
+
+    Codes ``500`` - ``599`` (server error responses)  will always be retried,
+    regardless of this option.
+
+
 downloader.ytdl.format
 ----------------------
 Type

gallery_dl/downloader/http.py

@@ -37,6 +37,7 @@ class HttpDownloader(DownloaderBase):
         self.minsize = self.config("filesize-min")
         self.maxsize = self.config("filesize-max")
         self.retries = self.config("retries", extractor._retries)
+        self.retry_codes = self.config("retry-codes")
         self.timeout = self.config("timeout", extractor._timeout)
         self.verify = self.config("verify", extractor._verify)
         self.mtime = self.config("mtime", True)
@@ -44,6 +45,8 @@ class HttpDownloader(DownloaderBase):
 
         if self.retries < 0:
             self.retries = float("inf")
+        if self.retry_codes is None:
+            self.retry_codes = [429]
         if self.minsize:
             minsize = text.parse_bytes(self.minsize)
             if not minsize:
@@ -98,6 +101,13 @@ class HttpDownloader(DownloaderBase):
         adjust_extension = kwdict.get(
             "_http_adjust_extension", self.adjust_extension)
 
+        codes = kwdict.get("_http_retry_codes")
+        if codes:
+            retry_codes = self.retry_codes.copy()
+            retry_codes += codes
+        else:
+            retry_codes = self.retry_codes
+
         if self.part and not metadata:
             pathfmt.part_enable(self.partdir)
 
@@ -158,7 +168,7 @@ class HttpDownloader(DownloaderBase):
                 break
             else:
                 msg = "'{} {}' for '{}'".format(code, response.reason, url)
-                if code == 429 or 500 <= code < 600:  # Server Error
+                if code in retry_codes or 500 <= code < 600:
                     continue
                 self.log.warning(msg)
                 return False