github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[pp:ugoira] implement storing "original" frames in archives (#6147)

Browse Source 
… by using '"mode": "archive"'

- rename 'ffmpeg-demuxer' option to 'mode'
- add 'metadata' option
- add 'zip' as a possible `--ugoira` format

TODO: adjust file mtimes inside archives when 'mtime' is enabled
This commit is contained in:
Mike Fährmann
2024-09-09 21:10:51 +02:00
parent 4601aa9794
commit ff07aef776
3 changed files with 113 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
docs/configuration.rst
View File

@@ -3258,17 +3258,21 @@ Description
extractor.pixiv.ugoira
----------------------
Type
``bool``
* ``bool``
* ``string``
Default
``true``
Description
Download Pixiv's Ugoira animations or ignore them.
Download Pixiv's Ugoira animations.
These animations come as a ``.zip`` file containing all
animation frames in JPEG format.
These animations come as a ``.zip`` archive containing all
animation frames in JPEG format by default.
Set this option to ``"original"``
to download them as individual, higher-quality frames.
Use an `ugoira` post processor to convert them
to watchable videos. (Example__)
to watchable animations. (Example__)
.. __: https://github.com/mikf/gallery-dl/blob/v1.12.3/docs/gallery-dl-example.conf#L9-L14
@@ -6149,6 +6153,8 @@ Description
Additional |ffmpeg| command-line arguments.
ugoira.mode
-----------
ugoira.ffmpeg-demuxer
---------------------
Type
@@ -6163,6 +6169,7 @@ Description
* "`concat <https://ffmpeg.org/ffmpeg-formats.html#concat-1>`_" (inaccurate frame timecodes for non-uniform frame delays)
* "`image2 <https://ffmpeg.org/ffmpeg-formats.html#image2-1>`_" (accurate timecodes, requires nanosecond file timestamps, i.e. no Windows or macOS)
* "mkvmerge" (accurate timecodes, only WebM or MKV, requires `mkvmerge <ugoira.mkvmerge-location_>`__)
* "archive" (store "original" frames in a ``.zip`` archive)
`"auto"` will select `mkvmerge` if available and fall back to `concat` otherwise.
@@ -6260,6 +6267,21 @@ Description
to reduce an odd width/height by 1 pixel and make them even.
ugoira.metadata
---------------
Type
* ``bool``
* ``string``
Default
``true``
Description
When using ``"mode": "archive"``, save Ugoira frame delay data as
``animation.json`` within the archive file.
If this is a ``string``,
use it as alternate filename for frame delay files.
ugoira.mtime
------------
Type

9
gallery_dl/option.py
View File

@@ -131,12 +131,17 @@ class UgoiraAction(argparse.Action):
"[a] palettegen [p];[b][p] paletteuse"),
"repeat-last-frame": False,
}
elif value in ("mkv", "copy"):
elif value == "mkv" or value == "copy":
pp = {
"extension" : "mkv",
"ffmpeg-args" : ("-c:v", "copy"),
"repeat-last-frame": False,
}
elif value == "zip" or value == "archive":
pp = {
"mode" : "archive",
}
namespace.options.append(((), "ugoira", "original"))
else:
parser.error("Unsupported Ugoira format '{}'".format(value))
@@ -693,7 +698,7 @@ def build_parser():
dest="postprocessors", metavar="FMT", action=UgoiraAction,
help=("Convert Pixiv Ugoira to FMT using FFmpeg. "
"Supported formats are 'webm', 'mp4', 'gif', "
"'vp8', 'vp9', 'vp9-lossless', 'copy'."),
"'vp8', 'vp9', 'vp9-lossless', 'copy', 'zip'."),
)
postprocessor.add_argument(
"--ugoira-conv",

108
gallery_dl/postprocessor/ugoira.py
View File

@@ -29,12 +29,12 @@ class UgoiraPP(PostProcessor):
def __init__(self, job, options):
PostProcessor.__init__(self, job)
self.extension = options.get("extension") or "webm"
self.args = options.get("ffmpeg-args") or ()
self.twopass = options.get("ffmpeg-twopass", False)
self.output = options.get("ffmpeg-output", "error")
self.delete = not options.get("keep-files", False)
self.repeat = options.get("repeat-last-frame", True)
self.metadata = options.get("metadata", True)
self.mtime = options.get("mtime", True)
self.skip = options.get("skip", True)
self.uniform = self._convert_zip = self._convert_files = False
@@ -45,24 +45,31 @@ class UgoiraPP(PostProcessor):
mkvmerge = options.get("mkvmerge-location")
self.mkvmerge = util.expand_path(mkvmerge) if mkvmerge else "mkvmerge"
demuxer = options.get("ffmpeg-demuxer")
if demuxer is None or demuxer == "auto":
if self.extension in ("webm", "mkv") and (
ext = options.get("extension")
mode = options.get("mode") or options.get("ffmpeg-demuxer")
if mode is None or mode == "auto":
if ext in (None, "webm", "mkv") and (
mkvmerge or shutil.which("mkvmerge")):
demuxer = "mkvmerge"
mode = "mkvmerge"
else:
demuxer = "concat"
mode = "concat"
if demuxer == "mkvmerge":
if mode == "mkvmerge":
self._process = self._process_mkvmerge
self._finalize = self._finalize_mkvmerge
elif demuxer == "image2":
elif mode == "image2":
self._process = self._process_image2
self._finalize = None
elif mode == "archive":
if ext is None:
ext = "zip"
self._convert_impl = self.convert_to_archive
self._tempdir = util.NullContext
else:
self._process = self._process_concat
self._finalize = None
self.log.debug("using %s demuxer", demuxer)
self.extension = "webm" if ext is None else ext
self.log.debug("using %s demuxer", mode)
rate = options.get("framerate", "auto")
if rate == "uniform":
@@ -93,8 +100,8 @@ class UgoiraPP(PostProcessor):
job.register_hooks({
"prepare": self.prepare,
"file" : self.convert_zip,
"after" : self.convert_files,
"file" : self.convert_from_zip,
"after" : self.convert_from_files,
}, options)
def prepare(self, pathfmt):
@@ -117,7 +124,7 @@ class UgoiraPP(PostProcessor):
frame = self._frames[index].copy()
frame["index"] = index
frame["path"] = pathfmt.realpath
frame["ext"] = pathfmt.kwdict["extension"]
frame["ext"] = pathfmt.extension
if not index:
self._files = [frame]
@@ -126,31 +133,34 @@ class UgoiraPP(PostProcessor):
if len(self._files) >= len(self._frames):
self._convert_files = True
def convert_zip(self, pathfmt):
def convert_from_zip(self, pathfmt):
if not self._convert_zip:
return
self._convert_zip = False
self._zip_source = True
with tempfile.TemporaryDirectory() as tempdir:
try:
with zipfile.ZipFile(pathfmt.temppath) as zfile:
zfile.extractall(tempdir)
except FileNotFoundError:
pathfmt.realpath = pathfmt.temppath
return
with self._tempdir() as tempdir:
if tempdir:
try:
with zipfile.ZipFile(pathfmt.temppath) as zfile:
zfile.extractall(tempdir)
except FileNotFoundError:
pathfmt.realpath = pathfmt.temppath
return
if self.convert(pathfmt, tempdir):
if self.delete:
pathfmt.delete = True
else:
elif pathfmt.extension != "zip":
self.log.info(pathfmt.filename)
pathfmt.set_extension("zip")
pathfmt.build_path()
def convert_files(self, pathfmt):
def convert_from_files(self, pathfmt):
if not self._convert_files:
return
self._convert_files = False
self._zip_source = False
with tempfile.TemporaryDirectory() as tempdir:
for frame in self._files:
@@ -159,13 +169,14 @@ class UgoiraPP(PostProcessor):
frame["file"] = name = "{}.{}".format(
frame["file"].partition(".")[0], frame["ext"])
# move frame into tempdir
try:
self._copy_file(frame["path"], tempdir + "/" + name)
except OSError as exc:
self.log.debug("Unable to copy frame %s (%s: %s)",
name, exc.__class__.__name__, exc)
return
if tempdir:
# move frame into tempdir
try:
self._copy_file(frame["path"], tempdir + "/" + name)
except OSError as exc:
self.log.debug("Unable to copy frame %s (%s: %s)",
name, exc.__class__.__name__, exc)
return
pathfmt.kwdict["num"] = 0
self._frames = self._files
@@ -182,6 +193,9 @@ class UgoiraPP(PostProcessor):
if self.skip and pathfmt.exists():
return True
return self._convert_impl(pathfmt, tempdir)
def convert_to_animation(self, pathfmt, tempdir):
# process frames and collect command-line arguments
args = self._process(pathfmt, tempdir)
if self.args_pp:
@@ -222,6 +236,42 @@ class UgoiraPP(PostProcessor):
util.set_mtime(pathfmt.realpath, mtime)
return True
def convert_to_archive(self, pathfmt, tempdir):
frames = self._frames
if self.metadata:
if isinstance(self.metadata, str):
metaname = self.metadata
else:
metaname = "animation.json"
framedata = util.json_dumps([
{"file": frame["file"], "delay": frame["delay"]}
for frame in frames
]).encode()
if self._zip_source:
self.delete = False
if self.metadata:
with zipfile.ZipFile(pathfmt.temppath, "a") as zfile:
with zfile.open(metaname, "w") as fp:
fp.write(framedata)
else:
with zipfile.ZipFile(pathfmt.realpath, "w") as zfile:
for frame in frames:
zinfo = zipfile.ZipInfo.from_file(
frame["path"], frame["file"])
with open(frame["path"], "rb") as src, \
zfile.open(zinfo, "w") as dst:
shutil.copyfileobj(src, dst, 1024*8)
if self.metadata:
with zfile.open(metaname, "w") as fp:
fp.write(framedata)
return True
_convert_impl = convert_to_animation
_tempdir = tempfile.TemporaryDirectory
def _exec(self, args):
self.log.debug(args)
out = None if self.output else subprocess.DEVNULL