[archive] support PostgreSQL archives for post processors (#6152)

docs/configuration.rst

@@ -6114,16 +6114,18 @@ Description
 exec.archive
 ------------
 Type
-    |Path|_
+    * ``string``
+    * |Path|_
 Description
-    File to store IDs of executed commands in,
+    Database to store IDs of executed commands in,
     similar to `extractor.*.archive`_.
 
-    ``archive-format``, ``archive-prefix``, and ``archive-pragma`` options,
-    akin to
-    `extractor.*.archive-format`_,
-    `extractor.*.archive-prefix`_, and
-    `extractor.*.archive-pragma`_, are supported as well.
+    The following archive options are also supported:
+
+    * `archive-format <extractor.*.archive-format_>`__
+    * `archive-prefix <extractor.*.archive-prefix_>`__
+    * `archive-pragma <extractor.*.archive-pragma_>`__
+    * `archive-table  <extractor.*.archive-table_>`__
 
 
 exec.async
@@ -6563,16 +6565,18 @@ Description
 metadata.archive
 ----------------
 Type
-    |Path|_
+    * ``string``
+    * |Path|_
 Description
-    File to store IDs of generated metadata files in,
+    Database to store IDs of generated metadata files in,
     similar to `extractor.*.archive`_.
 
-    ``archive-format``, ``archive-prefix``, and ``archive-pragma`` options,
-    akin to
-    `extractor.*.archive-format`_,
-    `extractor.*.archive-prefix`_, and
-    `extractor.*.archive-pragma`_, are supported as well.
+    The following archive options are also supported:
+
+    * `archive-format <extractor.*.archive-format_>`__
+    * `archive-prefix <extractor.*.archive-prefix_>`__
+    * `archive-pragma <extractor.*.archive-pragma_>`__
+    * `archive-table  <extractor.*.archive-table_>`__
 
 
 metadata.mtime
@@ -6643,16 +6647,18 @@ Description
 python.archive
 --------------
 Type
-    |Path|_
+    * ``string``
+    * |Path|_
 Description
-    File to store IDs of called Python functions in,
+    Database to store IDs of called Python functions in,
     similar to `extractor.*.archive`_.
 
-    ``archive-format``, ``archive-prefix``, and ``archive-pragma`` options,
-    akin to
-    `extractor.*.archive-format`_,
-    `extractor.*.archive-prefix`_, and
-    `extractor.*.archive-pragma`_, are supported as well.
+    The following archive options are also supported:
+
+    * `archive-format <extractor.*.archive-format_>`__
+    * `archive-prefix <extractor.*.archive-prefix_>`__
+    * `archive-pragma <extractor.*.archive-pragma_>`__
+    * `archive-table  <extractor.*.archive-table_>`__
 
 
 python.event

gallery_dl/archive.py

@@ -16,7 +16,7 @@ log = logging.getLogger("archive")
 
 
 def connect(path, prefix, format,
-            table=None, mode=None, pragma=None, kwdict=None):
+            table=None, mode=None, pragma=None, kwdict=None, cache_key=None):
     keygen = formatter.parse(prefix + format).format_map
 
     if path.startswith(("postgres://", "postgresql://")):
@@ -36,7 +36,7 @@ def connect(path, prefix, format,
     if kwdict is not None and table:
         table = formatter.parse(table).format_map(kwdict)
 
-    return cls(path, keygen, table, pragma)
+    return cls(path, keygen, table, pragma, cache_key)
 
 
 def sanitize(name):

gallery_dl/postprocessor/common.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2018-2023 Mike Fährmann
+# Copyright 2018-2025 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -8,7 +8,7 @@
 
 """Common classes and constants used by postprocessor modules."""
 
-from .. import util, formatter, archive
+from .. import archive
 
 
 class PostProcessor():
@@ -25,11 +25,11 @@ class PostProcessor():
         archive_path = options.get("archive")
         if archive_path:
             extr = job.extractor
-            archive_path = util.expand_path(archive_path)
 
+            archive_table = options.get("archive-table")
             archive_prefix = options.get("archive-prefix")
             if archive_prefix is None:
-                archive_prefix = extr.category
+                archive_prefix = extr.category if archive_table is None else ""
 
             archive_format = options.get("archive-format")
             if archive_format is None:
@@ -38,13 +38,14 @@ class PostProcessor():
                 archive_format = prefix + extr.archive_fmt
 
             try:
-                if "{" in archive_path:
-                    archive_path = formatter.parse(archive_path).format_map(
-                        job.pathfmt.kwdict)
-                self.archive = archive.DownloadArchive(
+                self.archive = archive.connect(
                     archive_path,
-                    archive_prefix + archive_format,
+                    archive_prefix,
+                    archive_format,
+                    archive_table,
+                    "file",
                     options.get("archive-pragma"),
+                    job.pathfmt.kwdict,
                     "_archive_" + self.name,
                 )
             except Exception as exc: