From 762a68996bf093aef604f1744486a5ee6fd94858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 5 Feb 2023 16:05:13 +0100 Subject: [PATCH] implement 'archive-pragma' option --- docs/configuration.rst | 29 +++++++++++++++++++++++------ docs/gallery-dl-example.conf | 3 ++- gallery_dl/job.py | 4 +++- gallery_dl/postprocessor/common.py | 4 +++- gallery_dl/util.py | 22 +++++++++++++--------- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index d532de65..1c7611ba 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -760,6 +760,19 @@ Description Prefix for archive IDs. +extractor.*.archive-pragma +-------------------------- +Type + ``list`` of ``strings`` +Example + ``["journal_mode=WAL", "synchronous=NORMAL"]`` +Description + A list of SQLite ``PRAGMA`` statements to run during archive initialization. + + See ``__ + for available ``PRAGMA`` statements and further details. + + extractor.*.postprocessors -------------------------- Type @@ -4027,9 +4040,11 @@ Description File to store IDs of executed commands in, similar to `extractor.*.archive`_. - ``archive-format`` and ``archive-prefix`` options, - akin to `extractor.*.archive-format`_ and `extractor.*.archive-prefix`_, - are supported as well. + ``archive-format``, ``archive-prefix``, and ``archive-pragma`` options, + akin to + `extractor.*.archive-format`_, + `extractor.*.archive-prefix`_, and + `extractor.*.archive-pragma`_, are supported as well. exec.async @@ -4287,9 +4302,11 @@ Description File to store IDs of generated metadata files in, similar to `extractor.*.archive`_. - ``archive-format`` and ``archive-prefix`` options, - akin to `extractor.*.archive-format`_ and `extractor.*.archive-prefix`_, - are supported as well. + ``archive-format``, ``archive-prefix``, and ``archive-pragma`` options, + akin to + `extractor.*.archive-format`_, + `extractor.*.archive-prefix`_, and + `extractor.*.archive-pragma`_, are supported as well. metadata.mtime diff --git a/docs/gallery-dl-example.conf b/docs/gallery-dl-example.conf index 92509b54..ef7b3b50 100644 --- a/docs/gallery-dl-example.conf +++ b/docs/gallery-dl-example.conf @@ -5,6 +5,7 @@ "#": "set global archive file for all extractors", "archive": "~/gallery-dl/archive.sqlite3", + "archive-pragma": ["journal_mode=WAL", "synchronous=NORMAL"], "#": "add two custom keywords into the metadata dictionary", "#": "these can be used to further refine your output directories or filenames", @@ -36,7 +37,7 @@ "pixiv": { - "#": "override global archive setting for pixiv", + "#": "override global archive path for pixiv", "archive": "~/gallery-dl/archive-pixiv.sqlite3", "#": "set custom directory and filename format strings for all pixiv downloads", diff --git a/gallery_dl/job.py b/gallery_dl/job.py index f7d84f03..13ce3341 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -436,10 +436,12 @@ class DownloadJob(Job): archive = util.expand_path(archive) archive_format = (cfg("archive-prefix", extr.category) + cfg("archive-format", extr.archive_fmt)) + archive_pragma = (cfg("archive-pragma")) try: if "{" in archive: archive = formatter.parse(archive).format_map(kwdict) - self.archive = util.DownloadArchive(archive, archive_format) + self.archive = util.DownloadArchive( + archive, archive_format, archive_pragma) except Exception as exc: extr.log.warning( "Failed to open download archive at '%s' ('%s: %s')", diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py index ee183988..c28d060d 100644 --- a/gallery_dl/postprocessor/common.py +++ b/gallery_dl/postprocessor/common.py @@ -36,7 +36,9 @@ class PostProcessor(): archive = formatter.parse(archive).format_map( job.pathfmt.kwdict) self.archive = util.DownloadArchive( - archive, archive_format, "_archive_" + self.name) + archive, archive_format, + options.get("archive-pragma"), + "_archive_" + self.name) except Exception as exc: self.log.warning( "Failed to open %s archive at '%s' ('%s: %s')", diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 6b9c457e..30f99a18 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2017-2022 Mike Fährmann +# Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -831,7 +831,8 @@ class ExtendedUrl(): class DownloadArchive(): - def __init__(self, path, format_string, cache_key="_archive_key"): + def __init__(self, path, format_string, pragma=None, + cache_key="_archive_key"): try: con = sqlite3.connect(path, timeout=60, check_same_thread=False) except sqlite3.OperationalError: @@ -839,20 +840,23 @@ class DownloadArchive(): con = sqlite3.connect(path, timeout=60, check_same_thread=False) con.isolation_level = None - self.close = con.close - self.cursor = con.cursor() - from . import formatter self.keygen = formatter.parse(format_string).format_map + self.close = con.close + self.cursor = cursor = con.cursor() self._cache_key = cache_key + if pragma: + for stmt in pragma: + cursor.execute("PRAGMA " + stmt) + try: - self.cursor.execute("CREATE TABLE IF NOT EXISTS archive " - "(entry TEXT PRIMARY KEY) WITHOUT ROWID") + cursor.execute("CREATE TABLE IF NOT EXISTS archive " + "(entry TEXT PRIMARY KEY) WITHOUT ROWID") except sqlite3.OperationalError: # fallback for missing WITHOUT ROWID support (#553) - self.cursor.execute("CREATE TABLE IF NOT EXISTS archive " - "(entry TEXT PRIMARY KEY)") + cursor.execute("CREATE TABLE IF NOT EXISTS archive " + "(entry TEXT PRIMARY KEY)") def check(self, kwdict): """Return True if the item described by 'kwdict' exists in archive"""