add options to control usage of .part files (#29)

- '--no-part' command line option to disable them
- 'downloader.http.part' and 'downloader.text.part' config options

Disabling .part files restores the behaviour of the old downloader
implementation.

gallery_dl/downloader/common.py

@@ -9,31 +9,41 @@
 """Common classes and constants used by downloader modules."""
 
 import os
-import os.path
-import logging
 import time
+import logging
 
 
 class DownloaderBase():
     """Base class for downloaders"""
     retries = 1
     mode = "b"
+    part = True
 
     def __init__(self, session, output):
         self.session = session
         self.out = output
         self.log = logging.getLogger("download")
+        self.downloading = False
 
     def download(self, url, pathfmt):
         """Download the resource at 'url' and write it to a file-like object"""
+        try:
+            self.download_impl(url, pathfmt)
+        finally:
+            # remove file from incomplete downloads
+            if self.downloading and not self.part:
+                try:
+                    os.remove(pathfmt.realpath)
+                except (OSError, AttributeError):
+                    pass
+
+    def download_impl(self, url, pathfmt):
+        """Actual implementaion of the download process"""
         tries = 0
         msg = ""
 
-        if not pathfmt.has_extension:
-            pathfmt.set_extension("part", False)
-            partpath = pathfmt.realpath
-        else:
-            partpath = pathfmt.realpath + ".part"
+        if self.part:
+            pathfmt.part_enable()
 
         while True:
             if tries:
@@ -45,12 +55,7 @@ class DownloaderBase():
             self.reset()
 
             # check for .part file
-            filesize = 0
-            if os.path.isfile(partpath):
-                try:
-                    filesize = os.path.getsize(partpath)
-                except OSError:
-                    pass
+            filesize = pathfmt.part_size()
 
             # connect to (remote) source
             try:
@@ -78,7 +83,8 @@ class DownloaderBase():
                     return True
 
             self.out.start(pathfmt.path)
-            with open(partpath, mode) as file:
+            self.downloading = True
+            with pathfmt.open(mode) as file:
                 # download content
                 try:
                     self.receive(file)
@@ -95,7 +101,9 @@ class DownloaderBase():
                     continue
             break
 
-        os.rename(partpath, pathfmt.realpath)
+        self.downloading = False
+        if self.part:
+            pathfmt.part_move()
         self.out.success(pathfmt.path, tries)
         return True
 

gallery_dl/downloader/http.py

@@ -13,10 +13,15 @@ from .common import DownloaderBase
 from .. import config, util
 
 
+def _conf(key, default=None):
+    return config.interpolate(("downloader", "http", key), default)
+
+
 class Downloader(DownloaderBase):
-    retries = config.interpolate(("downloader", "http", "retries",), 5)
-    timeout = config.interpolate(("downloader", "http", "timeout",), 30)
-    verify = config.interpolate(("downloader", "http", "verify",), True)
+    retries = _conf("retries", 5)
+    timeout = _conf("timeout", 30)
+    verify = _conf("verify", True)
+    part = _conf("part", True)
 
     def __init__(self, session, output):
         DownloaderBase.__init__(self, session, output)

gallery_dl/downloader/text.py

@@ -9,9 +9,11 @@
 """Downloader module for text: URLs"""
 
 from .common import DownloaderBase
+from .. import config
 
 
 class Downloader(DownloaderBase):
+    part = config.interpolate(("downloader", "text", "part"), True)
     mode = "t"
 
     def __init__(self, session, output):