From ebe9b0a04c21e25f1b3aed539793604e17b175f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Wed, 6 Dec 2017 22:35:05 +0100
Subject: [PATCH] another attempt at downloader retry behavior

This commit changes the general behavior from
'Retry on every exception and abort on DownloadError' to
'Only retry on DownloadRetry exceptions and abort on every other one'

The previous version would have retried on several states which
would have no chance of ever succeeding (invalid URLs, etc.)
---
 gallery_dl/downloader/common.py | 10 +++++-----
 gallery_dl/downloader/http.py   | 16 ++++++++++------
 gallery_dl/exception.py         |  4 ++--
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/gallery_dl/downloader/common.py b/gallery_dl/downloader/common.py
index a4732dff..70813897 100644
--- a/gallery_dl/downloader/common.py
+++ b/gallery_dl/downloader/common.py
@@ -74,14 +74,14 @@ class DownloaderBase():
             # connect to (remote) source
             try:
                 offset, size = self.connect(url, filesize)
-            except exception.DownloadError as exc:
-                self.out.error(pathfmt.path, exc, 0, 0)
-                return False
+            except exception.DownloadRetry as exc:
+                msg = exc
+                continue
             except exception.DownloadComplete:
                 break
             except Exception as exc:
-                msg = exc
-                continue
+                self.out.error(pathfmt.path, exc, 0, 0)
+                return False
 
             # check response
             if not offset:
diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py
index 2477a3a6..bf461ae2 100644
--- a/gallery_dl/downloader/http.py
+++ b/gallery_dl/downloader/http.py
@@ -10,6 +10,7 @@
 
 import time
 import mimetypes
+from requests.exceptions import ConnectionError, Timeout
 from .common import DownloaderBase
 from .. import util, exception
 
@@ -38,9 +39,12 @@ class Downloader(DownloaderBase):
         if offset:
             headers["Range"] = "bytes={}-".format(offset)
 
-        self.response = self.session.request(
-            "GET", url, stream=True, headers=headers, allow_redirects=True,
-            timeout=self.timeout, verify=self.verify)
+        try:
+            self.response = self.session.request(
+                "GET", url, stream=True, headers=headers, allow_redirects=True,
+                timeout=self.timeout, verify=self.verify)
+        except (ConnectionError, Timeout) as exc:
+            raise exception.DownloadRetry(exc)
 
         code = self.response.status_code
         if code == 200:  # OK
@@ -50,9 +54,9 @@ class Downloader(DownloaderBase):
             size = self.response.headers["Content-Range"].rpartition("/")[2]
         elif code == 416:  # Requested Range Not Satisfiable
             raise exception.DownloadComplete()
-        elif 400 <= code < 500 and code != 429:  # Client Error
-            raise exception.DownloadError(
-                "{} Client Error: {} for url: {}".format(
+        elif code == 429 or 500 <= code < 600:  # Server Error
+            raise exception.DownloadRetry(
+                "{} Server Error: {} for url: {}".format(
                     code, self.response.reason, url))
         else:
             self.response.raise_for_status()
diff --git a/gallery_dl/exception.py b/gallery_dl/exception.py
index f35f5089..8feb8325 100644
--- a/gallery_dl/exception.py
+++ b/gallery_dl/exception.py
@@ -50,8 +50,8 @@ class HttpError(ExtractionError):
     """HTTP request during extraction failed"""
 
 
-class DownloadError(GalleryDLException):
-    """Error during file download"""
+class DownloadRetry(GalleryDLException):
+    """Download attempt failed and should be retried"""
 
 
 class DownloadComplete(GalleryDLException):