Merge branch 'archive'

gallery_dl/util.py

@@ -19,6 +19,7 @@ import shutil
 import string
 import _string
 import hashlib
+import sqlite3
 import datetime
 import itertools
 import urllib.parse
@@ -373,22 +374,31 @@ class PathFormat():
         if os.altsep:
             self.basedirectory = self.basedirectory.replace(os.altsep, os.sep)
 
-        skipmode = extractor.config("skip", True)
-        if skipmode == "abort":
-            self.exists = self._exists_abort
-        elif skipmode == "exit":
-            self.exists = self._exists_exit
-        elif not skipmode:
-            self.exists = lambda: False
+        skip = extractor.config("skip", True)
+        if skip:
+            if skip == "abort":
+                self._skipexc = exception.StopExtraction
+            elif skip == "exit":
+                self._skipexc = exit
+            else:
+                self._skipexc = None
+        else:
+            self.exists = lambda x=None: False
 
     def open(self, mode="wb"):
         """Open file and return a corresponding file object"""
         return open(self.partpath or self.realpath, mode)
 
-    def exists(self):
-        """Return True if 'path' is complete and refers to an existing path"""
-        if self.has_extension:
-            return os.path.exists(self.realpath)
+    def exists(self, archive=None):
+        if (self.has_extension and os.path.exists(self.realpath) or
+                archive and archive.check(self.keywords)):
+            if self._skipexc:
+                raise self._skipexc()
+            if not self.has_extension:
+                self.set_extension("")
+                if self.path[-1] == ".":
+                    self.path = self.path[:-1]
+            return True
         return False
 
     def set_directory(self, keywords):
@@ -473,16 +483,6 @@ class PathFormat():
         shutil.copyfile(self.partpath, self.realpath)
         os.unlink(self.partpath)
 
-    def _exists_abort(self):
-        if self.has_extension and os.path.exists(self.realpath):
-            raise exception.StopExtraction()
-        return False
-
-    def _exists_exit(self):
-        if self.has_extension and os.path.exists(self.realpath):
-            exit()
-        return False
-
     @staticmethod
     def adjust_path(path):
         """Enable longer-than-260-character paths on windows"""
@@ -535,3 +535,30 @@ class OAuthSession():
             OAuthSession.quote(str(key)) + "=" + OAuthSession.quote(str(value))
             for key, value in sorted(params.items()) if value
         )
+
+
+class DownloadArchive():
+
+    def __init__(self, extractor, path):
+        con = sqlite3.connect(path)
+        con.isolation_level = None
+        self.cursor = con.cursor()
+        self.cursor.execute("CREATE TABLE IF NOT EXISTS archive "
+                            "(entry PRIMARY KEY) WITHOUT ROWID")
+        self.keygen = (
+            extractor.category +
+            (extractor.archive_fmt or extractor.filename_fmt)
+        ).format_map
+        self._key = None
+
+    def check(self, kwdict):
+        """Return True if item described by 'kwdict' exists in archive"""
+        self._key = self.keygen(kwdict)
+        self.cursor.execute(
+            "SELECT 1 FROM archive WHERE entry=? LIMIT 1", (self._key,))
+        return self.cursor.fetchone()
+
+    def add(self):
+        """Add last item used in 'check()' to archive"""
+        self.cursor.execute(
+            "INSERT OR IGNORE INTO archive VALUES (?)", (self._key,))