implement 'archive-pragma' option

docs/configuration.rst

@@ -760,6 +760,19 @@ Description
     Prefix for archive IDs.
 
 
+extractor.*.archive-pragma
+--------------------------
+Type
+    ``list`` of ``strings``
+Example
+    ``["journal_mode=WAL", "synchronous=NORMAL"]``
+Description
+    A list of SQLite ``PRAGMA`` statements to run during archive initialization.
+
+    See `<https://www.sqlite.org/pragma.html>`__
+    for available ``PRAGMA`` statements and further details.
+
+
 extractor.*.postprocessors
 --------------------------
 Type
@@ -4027,9 +4040,11 @@ Description
     File to store IDs of executed commands in,
     similar to `extractor.*.archive`_.
 
-    ``archive-format`` and ``archive-prefix`` options,
+    ``archive-format``, ``archive-prefix``, and ``archive-pragma`` options,
-    akin to `extractor.*.archive-format`_ and `extractor.*.archive-prefix`_,
+    akin to
-    are supported as well.
+    `extractor.*.archive-format`_,
+    `extractor.*.archive-prefix`_, and
+    `extractor.*.archive-pragma`_, are supported as well.
 
 
 exec.async
@@ -4287,9 +4302,11 @@ Description
     File to store IDs of generated metadata files in,
     similar to `extractor.*.archive`_.
 
-    ``archive-format`` and ``archive-prefix`` options,
+    ``archive-format``, ``archive-prefix``, and ``archive-pragma`` options,
-    akin to `extractor.*.archive-format`_ and `extractor.*.archive-prefix`_,
+    akin to
-    are supported as well.
+    `extractor.*.archive-format`_,
+    `extractor.*.archive-prefix`_, and
+    `extractor.*.archive-pragma`_, are supported as well.
 
 
 metadata.mtime

docs/gallery-dl-example.conf

@@ -5,6 +5,7 @@
 
         "#": "set global archive file for all extractors",
         "archive": "~/gallery-dl/archive.sqlite3",
+        "archive-pragma": ["journal_mode=WAL", "synchronous=NORMAL"],
 
         "#": "add two custom keywords into the metadata dictionary",
         "#": "these can be used to further refine your output directories or filenames",
@@ -36,7 +37,7 @@
 
         "pixiv":
         {
-            "#": "override global archive setting for pixiv",
+            "#": "override global archive path for pixiv",
             "archive": "~/gallery-dl/archive-pixiv.sqlite3",
 
             "#": "set custom directory and filename format strings for all pixiv downloads",

gallery_dl/job.py

@@ -436,10 +436,12 @@ class DownloadJob(Job):
             archive = util.expand_path(archive)
             archive_format = (cfg("archive-prefix", extr.category) +
                               cfg("archive-format", extr.archive_fmt))
+            archive_pragma = (cfg("archive-pragma"))
             try:
                 if "{" in archive:
                     archive = formatter.parse(archive).format_map(kwdict)
-                self.archive = util.DownloadArchive(archive, archive_format)
+                self.archive = util.DownloadArchive(
+                    archive, archive_format, archive_pragma)
             except Exception as exc:
                 extr.log.warning(
                     "Failed to open download archive at '%s' ('%s: %s')",

gallery_dl/postprocessor/common.py

@@ -36,7 +36,9 @@ class PostProcessor():
                     archive = formatter.parse(archive).format_map(
                         job.pathfmt.kwdict)
                 self.archive = util.DownloadArchive(
-                    archive, archive_format, "_archive_" + self.name)
+                    archive, archive_format,
+                    options.get("archive-pragma"),
+                    "_archive_" + self.name)
             except Exception as exc:
                 self.log.warning(
                     "Failed to open %s archive at '%s' ('%s: %s')",

gallery_dl/util.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2017-2022 Mike Fährmann
+# Copyright 2017-2023 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -831,7 +831,8 @@ class ExtendedUrl():
 
 class DownloadArchive():
 
-    def __init__(self, path, format_string, cache_key="_archive_key"):
+    def __init__(self, path, format_string, pragma=None,
+                 cache_key="_archive_key"):
         try:
             con = sqlite3.connect(path, timeout=60, check_same_thread=False)
         except sqlite3.OperationalError:
@@ -839,20 +840,23 @@ class DownloadArchive():
             con = sqlite3.connect(path, timeout=60, check_same_thread=False)
         con.isolation_level = None
 
-        self.close = con.close
-        self.cursor = con.cursor()
-
         from . import formatter
         self.keygen = formatter.parse(format_string).format_map
+        self.close = con.close
+        self.cursor = cursor = con.cursor()
         self._cache_key = cache_key
 
+        if pragma:
+            for stmt in pragma:
+                cursor.execute("PRAGMA " + stmt)
+
         try:
-            self.cursor.execute("CREATE TABLE IF NOT EXISTS archive "
+            cursor.execute("CREATE TABLE IF NOT EXISTS archive "
-                                "(entry TEXT PRIMARY KEY) WITHOUT ROWID")
+                           "(entry TEXT PRIMARY KEY) WITHOUT ROWID")
         except sqlite3.OperationalError:
             # fallback for missing WITHOUT ROWID support (#553)
-            self.cursor.execute("CREATE TABLE IF NOT EXISTS archive "
+            cursor.execute("CREATE TABLE IF NOT EXISTS archive "
-                                "(entry TEXT PRIMARY KEY)")
+                           "(entry TEXT PRIMARY KEY)")
 
     def check(self, kwdict):
         """Return True if the item described by 'kwdict' exists in archive"""