[postprocessor:metadata] implement archive options (#2421)

'archive', 'archive-format', and 'archive-prefix'

gallery_dl/postprocessor/metadata.py

@@ -59,9 +59,35 @@ class MetadataPP(PostProcessor):
             events = events.split(",")
         job.register_hooks({event: self.run for event in events}, options)
 
+        archive = options.get("archive")
+        if archive:
+            extr = job.extractor
+            archive = util.expand_path(archive)
+            archive_format = (
+                options.get("archive-prefix", extr.category) +
+                options.get("archive-format", "_MD_" + extr.archive_fmt))
+            try:
+                if "{" in archive:
+                    archive = formatter.parse(archive).format_map(
+                        job.pathfmt.kwdict)
+                self.archive = util.DownloadArchive(
+                    archive, archive_format, "_archive_metadata")
+            except Exception as exc:
+                self.log.warning(
+                    "Failed to open download archive at '%s' ('%s: %s')",
+                    archive, exc.__class__.__name__, exc)
+            else:
+                self.log.debug("Using download archive '%s'", archive)
+        else:
+            self.archive = None
+
         self.mtime = options.get("mtime")
 
     def run(self, pathfmt):
+        archive = self.archive
+        if archive and archive.check(pathfmt.kwdict):
+            return
+
         directory = self._directory(pathfmt)
         path = directory + self._filename(pathfmt)
 
@@ -73,6 +99,9 @@ class MetadataPP(PostProcessor):
             with open(path, "w", encoding="utf-8") as fp:
                 self.write(fp, pathfmt.kwdict)
 
+        if archive:
+            archive.add(pathfmt.kwdict)
+
         if self.mtime:
             mtime = pathfmt.kwdict.get("_mtime")
             if mtime: