From 17f5ba43a86c6b21554405fc2e3e7087fc2ed1e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 30 Aug 2024 17:36:55 +0200 Subject: [PATCH] [pp:rename] add 'rename' post processor (#5846, #6044) renames previously downloaded files to a different filename format --- docs/configuration.rst | 41 ++++++++++++++++- gallery_dl/postprocessor/__init__.py | 1 + gallery_dl/postprocessor/rename.py | 66 ++++++++++++++++++++++++++++ test/test_postprocessor.py | 55 +++++++++++++++++++++++ 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 gallery_dl/postprocessor/rename.py diff --git a/docs/configuration.rst b/docs/configuration.rst index 5795fc03..049626d9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -6016,6 +6016,42 @@ Description or the |Path|_ to a `.py` file, +rename.from +----------- +Type + ``string`` +Description + The `format string`_ for filenames to rename. + + When no value is given, `extractor.*.filename`_ is used. + + +rename.to +--------- +Type + ``string`` +Description + The `format string`_ for target filenames. + + When no value is given, `extractor.*.filename`_ is used. + + Note: + With default settings, the potential download to `extractor.*.filename`_ + still happens, even when using this post processor. + Disabling `file downloads `__ + when using this option is recommended. + + +rename.skip +----------- +Type + ``bool`` +Default + ``true`` +Description + Do not rename a file when another file with the target name already exists. + + ugoira.extension ---------------- Type @@ -6340,7 +6376,7 @@ Type Example ``["~/urls.txt", "$HOME/input"]`` Description - Additional# input files. + Additional input files. signals-ignore @@ -6663,11 +6699,12 @@ Description Set file modification time according to its metadata ``python`` Call Python functions + ``rename`` + Rename previously downloaded files ``ugoira`` Convert Pixiv Ugoira to WebM using |ffmpeg| ``zip`` Store files in a ZIP archive - |ytdl| diff --git a/gallery_dl/postprocessor/__init__.py b/gallery_dl/postprocessor/__init__.py index 46905547..c17eacc6 100644 --- a/gallery_dl/postprocessor/__init__.py +++ b/gallery_dl/postprocessor/__init__.py @@ -15,6 +15,7 @@ modules = [ "metadata", "mtime", "python", + "rename", "ugoira", "zip", ] diff --git a/gallery_dl/postprocessor/rename.py b/gallery_dl/postprocessor/rename.py new file mode 100644 index 00000000..f42d63ab --- /dev/null +++ b/gallery_dl/postprocessor/rename.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Rename files""" + +from .common import PostProcessor +from .. import formatter +import os + + +class RenamePP(PostProcessor): + + def __init__(self, job, options): + PostProcessor.__init__(self, job) + + self.skip = options.get("skip", True) + old = options.get("from") + new = options.get("to") + + if old: + self._old = self._apply_format(old) + self._new = (self._apply_format(new) if new else + self._apply_pathfmt) + elif new: + self._old = self._apply_pathfmt + self._new = self._apply_format(new) + else: + raise ValueError("Option 'from' or 'to' is required") + + job.register_hooks({"prepare": self.run}, options) + + def run(self, pathfmt): + old = self._old(pathfmt) + path_old = pathfmt.realdirectory + old + + if os.path.exists(path_old): + new = self._new(pathfmt) + path_new = pathfmt.realdirectory + new + + if self.skip and os.path.exists(path_new): + return self.log.warning( + "Not renaming '%s' to '%s' since another file with the " + "same name exists", old, new) + + self.log.info("'%s' -> '%s'", old, new) + os.replace(path_old, path_new) + + def _apply_pathfmt(self, pathfmt): + return pathfmt.build_filename(pathfmt.kwdict) + + def _apply_format(self, format_string): + fmt = formatter.parse(format_string).format_map + + def apply(pathfmt): + return pathfmt.clean_path(pathfmt.clean_segment(fmt( + pathfmt.kwdict))) + + return apply + + +__postprocessor__ = RenamePP diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 841ebf0a..3089b82a 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -12,6 +12,7 @@ import sys import unittest from unittest.mock import Mock, mock_open, patch +import shutil import logging import zipfile import tempfile @@ -691,6 +692,60 @@ def calc(kwdict): """) +class RenameTest(BasePostprocessorTest): + + def _prepare(self, filename): + path = self.pathfmt.realdirectory + shutil.rmtree(path, ignore_errors=True) + os.makedirs(path, exist_ok=True) + + with open(path + filename, "w"): + pass + + return path + + def test_rename_from(self): + self._create({"from": "{id}.{extension}"}, {"id": 12345}) + path = self._prepare("12345.ext") + + self._trigger() + + self.assertEqual(os.listdir(path), ["file.ext"]) + + def test_rename_to(self): + self._create({"to": "{id}.{extension}"}, {"id": 12345}) + path = self._prepare("file.ext") + + self._trigger() + + self.assertEqual(os.listdir(path), ["12345.ext"]) + + def test_rename_from_to(self): + self._create({"from": "name", "to": "{id}"}, {"id": 12345}) + path = self._prepare("name") + + self._trigger() + + self.assertEqual(os.listdir(path), ["12345"]) + + def test_rename_noopt(self): + with self.assertRaises(ValueError): + self._create({}) + + def test_rename_skip(self): + self._create({"from": "{id}.{extension}"}, {"id": 12345}) + path = self._prepare("12345.ext") + with open(path + "file.ext", "w"): + pass + + with self.assertLogs("postprocessor.rename", level="WARNING") as cm: + self._trigger() + self.assertTrue(cm.output[0].startswith( + "WARNING:postprocessor.rename:Not renaming " + "'12345.ext' to 'file.ext'")) + self.assertEqual(sorted(os.listdir(path)), ["12345.ext", "file.ext"]) + + class ZipTest(BasePostprocessorTest): def test_zip_default(self):