From f7b5c4c3e744c474ee37f04538fd395f3c282019 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sun, 30 Jun 2019 21:27:28 +0200
Subject: [PATCH] use values of 'retries' options correctly

The RE-tries option now specifies exactly that: the maximum number a
failed HTTP request is re-tried. For example a value of 2 will now
correctly stop after 3 attempts: the initial one + 2 re-tries.

The maximum wait-time now also caps at 30min and increases exponentially
for both extractor.request() and downloader.http.download().
---
 docs/configuration.rst         |  2 +-
 docs/gallery-dl.conf           |  4 ++--
 gallery_dl/downloader/http.py  |  7 ++++---
 gallery_dl/extractor/common.py | 16 ++++++++--------
 gallery_dl/option.py           |  2 +-
 5 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/docs/configuration.rst b/docs/configuration.rst
index db8fa0f7..861297fc 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -300,7 +300,7 @@ extractor.*.retries
 -------------------
 =========== =====
 Type        ``integer``
-Default     ``5``
+Default     ``4``
 Description Number of times a failed HTTP request is retried before giving up.
 =========== =====
 
diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf
index d956f961..d67b7fc7 100644
--- a/docs/gallery-dl.conf
+++ b/docs/gallery-dl.conf
@@ -154,7 +154,7 @@
         {
             "mtime": true,
             "rate": null,
-            "retries": 5,
+            "retries": 4,
             "timeout": 30.0,
             "verify": true
         },
@@ -164,7 +164,7 @@
             "format": null,
             "mtime": true,
             "rate": null,
-            "retries": 5,
+            "retries": 4,
             "timeout": 30.0,
             "verify": true
         }
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index e69e1c02..87f745b2 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -64,10 +64,10 @@ class HttpDownloader(DownloaderBase):
             if tries:
                 if response:
                     response.close()
-                self.log.warning("%s (%d/%d)", msg, tries, self.retries)
-                if tries >= self.retries:
+                self.log.warning("%s (%s/%s)", msg, tries, self.retries+1)
+                if tries > self.retries:
                     return False
-                time.sleep(tries)
+                time.sleep(min(2 ** (tries-1), 1800))
             tries += 1
 
             # check for .part file
@@ -142,6 +142,7 @@ class HttpDownloader(DownloaderBase):
                 if size and file.tell() < size:
                     msg = "filesize mismatch ({} < {})".format(
                         file.tell(), size)
+                    print()
                     continue
 
                 # check filename extension
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py
index 175af63d..70428650 100644
--- a/gallery_dl/extractor/common.py
+++ b/gallery_dl/extractor/common.py
@@ -39,7 +39,7 @@ class Extractor():
         self._init_headers()
         self._init_cookies()
         self._init_proxies()
-        self._retries = self.config("retries", 5)
+        self._retries = self.config("retries", 4)
         self._timeout = self.config("timeout", 30)
         self._verify = self.config("verify", True)
 
@@ -65,9 +65,9 @@ class Extractor():
 
     def request(self, url, method="GET", *, session=None,
                 encoding=None, expect=(), retries=None, **kwargs):
-        tries = 0
-        retries = retries or self._retries
-        session = session or self.session
+        tries = 1
+        retries = self._retries if retries is None else retries
+        session = self.session if session is None else session
         kwargs.setdefault("timeout", self._timeout)
         kwargs.setdefault("verify", self._verify)
 
@@ -98,11 +98,11 @@ class Extractor():
                 if code < 500 and code != 429:
                     break
 
-            tries += 1
-            self.log.debug("%s (%d/%d)", msg, tries, retries)
-            if tries >= retries:
+            self.log.debug("%s (%s/%s)", msg, tries, retries+1)
+            if tries > retries:
                 break
-            time.sleep(2 ** tries)
+            time.sleep(min(2 ** (tries-1), 1800))
+            tries += 1
 
         raise exception.HttpError(msg)
 
diff --git a/gallery_dl/option.py b/gallery_dl/option.py
index 8877e080..d6470ed7 100644
--- a/gallery_dl/option.py
+++ b/gallery_dl/option.py
@@ -174,7 +174,7 @@ def build_parser():
     downloader.add_argument(
         "-R", "--retries",
         dest="retries", metavar="RETRIES", type=int, action=ConfigAction,
-        help="Number of retries (default: 5)",
+        help="Number of retries (default: 4)",
     )
     downloader.add_argument(
         "-A", "--abort",