improve downloader retry behavior

- only retry download on 5xx and 429 status codes
- immediately fail on 4xx status codes

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## Unreleased
+- Improved retry behavior for failed HTTP downloads
+
 ## 1.0.1 - 2017-11-10
 - Added support for:
   - `xvideos` - https://www.xvideos.com/ ([#45](https://github.com/mikf/gallery-dl/issues/45))

gallery_dl/downloader/common.py

@@ -11,7 +11,7 @@
 import os
 import time
 import logging
-from .. import config, util
+from .. import config, util, exception
 from requests.exceptions import RequestException
 
 
@@ -65,7 +65,7 @@ class DownloaderBase():
                 self.out.error(pathfmt.path, msg, tries, self.retries)
                 if tries >= self.retries:
                     return False
-                time.sleep(1)
+                time.sleep(tries)
             tries += 1
 
             # check for .part file
@@ -74,6 +74,11 @@ class DownloaderBase():
             # connect to (remote) source
             try:
                 offset, size = self.connect(url, filesize)
+            except exception.DownloadError as exc:
+                self.out.error(pathfmt.path, exc, 0, 0)
+                return False
+            except exception.DownloadComplete:
+                break
             except Exception as exc:
                 msg = exc
                 continue
@@ -83,8 +88,6 @@ class DownloaderBase():
                 mode = "wb"
                 if filesize:
                     self.log.info("Unable to resume partial download")
-            elif offset == -1:
-                break  # early finish
             else:
                 mode = "ab"
                 self.log.info("Resuming download at byte %d", offset)
@@ -124,8 +127,7 @@ class DownloaderBase():
 
         Returns a 2-tuple containing the actual offset and expected filesize.
         If the returned offset-value is greater than zero, all received data
-        will be appended to the existing .part file. If it is '-1', the
+        will be appended to the existing .part file.
-        download will finish early and be considered successfull.
         Return '0' as second tuple-field to indicate an unknown filesize.
         """
 

gallery_dl/downloader/http.py

@@ -10,7 +10,7 @@
 
 import mimetypes
 from .common import DownloaderBase
-from .. import util
+from .. import util, exception
 
 
 class Downloader(DownloaderBase):
@@ -33,14 +33,17 @@ class Downloader(DownloaderBase):
             timeout=self.timeout, verify=self.verify)
 
         code = self.response.status_code
-        if code == 200:
+        if code == 200:  # OK
             offset = 0
             size = self.response.headers.get("Content-Length")
-        elif code == 206:
+        elif code == 206:  # Partial Content
             size = self.response.headers["Content-Range"].rpartition("/")[2]
-        elif code == 416:
+        elif code == 416:  # Requested Range Not Satisfiable
-            # file is already complete
+            raise exception.DownloadComplete()
-            return -1, 0
+        elif 400 <= code < 500 and code != 429:  # Client Error
+            raise exception.DownloadError(
+                "{} Client Error: {} for url: {}".format(
+                    code, self.response.reason, url))
         else:
             self.response.raise_for_status()
 

gallery_dl/exception.py

@@ -17,6 +17,8 @@ Exception
       |    +-- AuthorizationError
       |    +-- NotFoundError
       |    +-- HttpError
+      +-- DownloadError
+      +-- DownloadComplete
       +-- NoExtractorError
       +-- FormatError
       +-- FilterError
@@ -48,6 +50,14 @@ class HttpError(ExtractionError):
     """HTTP request during extraction failed"""
 
 
+class DownloadError(GalleryDLException):
+    """Error during file download"""
+
+
+class DownloadComplete(GalleryDLException):
+    """Output file of attempted download is already complete"""
+
+
 class NoExtractorError(GalleryDLException):
     """No extractor can handle the given URL"""
 

gallery_dl/output.py

@@ -89,9 +89,10 @@ class TerminalOutput(NullOutput):
         if tries <= 1 and path:
             print("\r", end="")
             safeprint(self.shorten(CHAR_ERROR + path))
+        if max_tries > 1:
+            error = "{} ({}/{})".format(error, tries, max_tries)
         print("\r[Error] ", end="")
-        safeprint(error, end="")
+        safeprint(error)
-        print(" (", tries, "/", max_tries, ")", sep="")
 
     def shorten(self, txt):
         """Reduce the length of 'txt' to the width of the terminal"""
@@ -119,8 +120,9 @@ class ColorOutput(TerminalOutput):
     def error(self, path, error, tries, max_tries):
         if tries <= 1 and path:
             print("\r\033[1;31m", self.shorten(path), sep="")
-        print("\r\033[0;31m[Error]\033[0m ", error,
+        if max_tries > 1:
-              " (", tries, "/", max_tries, ")", sep="")
+            error = "{} ({}/{})".format(error, tries, max_tries)
+        print("\r\033[0;31m[Error]\033[0m", error)
 
 
 if os.name == "nt":

gallery_dl/version.py

@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.0.1"
+__version__ = "1.0.2-dev"