github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implement 'archive-pragma' option

Browse Source
This commit is contained in:
Mike Fährmann
2023-02-05 16:05:13 +01:00
parent bbf0911a46
commit 762a68996b
5 changed files with 44 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gallery_dl/job.py
View File

@@ -436,10 +436,12 @@ class DownloadJob(Job):
archive = util.expand_path(archive)
archive_format = (cfg("archive-prefix", extr.category) +
cfg("archive-format", extr.archive_fmt))
archive_pragma = (cfg("archive-pragma"))
try:
if "{" in archive:
archive = formatter.parse(archive).format_map(kwdict)
self.archive = util.DownloadArchive(archive, archive_format)
self.archive = util.DownloadArchive(
archive, archive_format, archive_pragma)
except Exception as exc:
extr.log.warning(
"Failed to open download archive at '%s' ('%s: %s')",

4
gallery_dl/postprocessor/common.py
View File

@@ -36,7 +36,9 @@ class PostProcessor():
archive = formatter.parse(archive).format_map(
job.pathfmt.kwdict)
self.archive = util.DownloadArchive(
archive, archive_format, "_archive_" + self.name)
archive, archive_format,
options.get("archive-pragma"),
"_archive_" + self.name)
except Exception as exc:
self.log.warning(
"Failed to open %s archive at '%s' ('%s: %s')",

22
gallery_dl/util.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2017-2022 Mike Fährmann
# Copyright 2017-2023 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -831,7 +831,8 @@ class ExtendedUrl():
class DownloadArchive():
def __init__(self, path, format_string, cache_key="_archive_key"):
def __init__(self, path, format_string, pragma=None,
cache_key="_archive_key"):
try:
con = sqlite3.connect(path, timeout=60, check_same_thread=False)
except sqlite3.OperationalError:
@@ -839,20 +840,23 @@ class DownloadArchive():
con = sqlite3.connect(path, timeout=60, check_same_thread=False)
con.isolation_level = None
self.close = con.close
self.cursor = con.cursor()
from . import formatter
self.keygen = formatter.parse(format_string).format_map
self.close = con.close
self.cursor = cursor = con.cursor()
self._cache_key = cache_key
if pragma:
for stmt in pragma:
cursor.execute("PRAGMA " + stmt)
try:
self.cursor.execute("CREATE TABLE IF NOT EXISTS archive "
"(entry TEXT PRIMARY KEY) WITHOUT ROWID")
cursor.execute("CREATE TABLE IF NOT EXISTS archive "
"(entry TEXT PRIMARY KEY) WITHOUT ROWID")
except sqlite3.OperationalError:
# fallback for missing WITHOUT ROWID support (#553)
self.cursor.execute("CREATE TABLE IF NOT EXISTS archive "
"(entry TEXT PRIMARY KEY)")
cursor.execute("CREATE TABLE IF NOT EXISTS archive "
"(entry TEXT PRIMARY KEY)")
def check(self, kwdict):
"""Return True if the item described by 'kwdict' exists in archive"""