use values of 'retries' options correctly

The RE-tries option now specifies exactly that: the maximum number a
failed HTTP request is re-tried. For example a value of 2 will now
correctly stop after 3 attempts: the initial one + 2 re-tries.

The maximum wait-time now also caps at 30min and increases exponentially
for both extractor.request() and downloader.http.download().

docs/configuration.rst

@@ -300,7 +300,7 @@ extractor.*.retries
 -------------------
 =========== =====
 Type        ``integer``
-Default     ``5``
+Default     ``4``
 Description Number of times a failed HTTP request is retried before giving up.
 =========== =====
 

docs/gallery-dl.conf

@@ -154,7 +154,7 @@
         {
             "mtime": true,
             "rate": null,
-            "retries": 5,
+            "retries": 4,
             "timeout": 30.0,
             "verify": true
         },
@@ -164,7 +164,7 @@
             "format": null,
             "mtime": true,
             "rate": null,
-            "retries": 5,
+            "retries": 4,
             "timeout": 30.0,
             "verify": true
         }

gallery_dl/downloader/http.py

@@ -64,10 +64,10 @@ class HttpDownloader(DownloaderBase):
             if tries:
                 if response:
                     response.close()
-                self.log.warning("%s (%d/%d)", msg, tries, self.retries)
-                if tries >= self.retries:
+                self.log.warning("%s (%s/%s)", msg, tries, self.retries+1)
+                if tries > self.retries:
                     return False
-                time.sleep(tries)
+                time.sleep(min(2 ** (tries-1), 1800))
             tries += 1
 
             # check for .part file
@@ -142,6 +142,7 @@ class HttpDownloader(DownloaderBase):
                 if size and file.tell() < size:
                     msg = "filesize mismatch ({} < {})".format(
                         file.tell(), size)
+                    print()
                     continue
 
                 # check filename extension

gallery_dl/extractor/common.py

@@ -39,7 +39,7 @@ class Extractor():
         self._init_headers()
         self._init_cookies()
         self._init_proxies()
-        self._retries = self.config("retries", 5)
+        self._retries = self.config("retries", 4)
         self._timeout = self.config("timeout", 30)
         self._verify = self.config("verify", True)
 
@@ -65,9 +65,9 @@ class Extractor():
 
     def request(self, url, method="GET", *, session=None,
                 encoding=None, expect=(), retries=None, **kwargs):
-        tries = 0
-        retries = retries or self._retries
-        session = session or self.session
+        tries = 1
+        retries = self._retries if retries is None else retries
+        session = self.session if session is None else session
         kwargs.setdefault("timeout", self._timeout)
         kwargs.setdefault("verify", self._verify)
 
@@ -98,11 +98,11 @@ class Extractor():
                 if code < 500 and code != 429:
                     break
 
-            tries += 1
-            self.log.debug("%s (%d/%d)", msg, tries, retries)
-            if tries >= retries:
+            self.log.debug("%s (%s/%s)", msg, tries, retries+1)
+            if tries > retries:
                 break
-            time.sleep(2 ** tries)
+            time.sleep(min(2 ** (tries-1), 1800))
+            tries += 1
 
         raise exception.HttpError(msg)
 

gallery_dl/option.py

@@ -174,7 +174,7 @@ def build_parser():
     downloader.add_argument(
         "-R", "--retries",
         dest="retries", metavar="RETRIES", type=int, action=ConfigAction,
-        help="Number of retries (default: 5)",
+        help="Number of retries (default: 4)",
     )
     downloader.add_argument(
         "-A", "--abort",