another attempt at downloader retry behavior

This commit changes the general behavior from
'Retry on every exception and abort on DownloadError' to
'Only retry on DownloadRetry exceptions and abort on every other one'

The previous version would have retried on several states which
would have no chance of ever succeeding (invalid URLs, etc.)

gallery_dl/downloader/common.py

@@ -74,14 +74,14 @@ class DownloaderBase():
             # connect to (remote) source
             try:
                 offset, size = self.connect(url, filesize)
-            except exception.DownloadError as exc:
-                self.out.error(pathfmt.path, exc, 0, 0)
-                return False
+            except exception.DownloadRetry as exc:
+                msg = exc
+                continue
             except exception.DownloadComplete:
                 break
             except Exception as exc:
-                msg = exc
-                continue
+                self.out.error(pathfmt.path, exc, 0, 0)
+                return False
 
             # check response
             if not offset:

gallery_dl/downloader/http.py

@@ -10,6 +10,7 @@
 
 import time
 import mimetypes
+from requests.exceptions import ConnectionError, Timeout
 from .common import DownloaderBase
 from .. import util, exception
 
@@ -38,9 +39,12 @@ class Downloader(DownloaderBase):
         if offset:
             headers["Range"] = "bytes={}-".format(offset)
 
-        self.response = self.session.request(
-            "GET", url, stream=True, headers=headers, allow_redirects=True,
-            timeout=self.timeout, verify=self.verify)
+        try:
+            self.response = self.session.request(
+                "GET", url, stream=True, headers=headers, allow_redirects=True,
+                timeout=self.timeout, verify=self.verify)
+        except (ConnectionError, Timeout) as exc:
+            raise exception.DownloadRetry(exc)
 
         code = self.response.status_code
         if code == 200:  # OK
@@ -50,9 +54,9 @@ class Downloader(DownloaderBase):
             size = self.response.headers["Content-Range"].rpartition("/")[2]
         elif code == 416:  # Requested Range Not Satisfiable
             raise exception.DownloadComplete()
-        elif 400 <= code < 500 and code != 429:  # Client Error
-            raise exception.DownloadError(
-                "{} Client Error: {} for url: {}".format(
+        elif code == 429 or 500 <= code < 600:  # Server Error
+            raise exception.DownloadRetry(
+                "{} Server Error: {} for url: {}".format(
                     code, self.response.reason, url))
         else:
             self.response.raise_for_status()