[ytdl] implement parsing ytdl command-line options (#1680)

- adds 'config-file' and 'cmdline-args' options
  for both ytdl downloader and extractor
- create 'ytdl' helper module, which combines YoutubeDL creation
  and option parsing.
- most likely a buggy mess due to incompatibilities between the
  original youtube-dl and yt-dlp.

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Changelog
 
+## Unreleased
+
 ## 1.19.2 - 2021-11-05
 ### Additions
 - [kemonoparty] add `comments` option ([#1980](https://github.com/mikf/gallery-dl/issues/1980))

gallery_dl/downloader/ytdl.py

@@ -9,7 +9,7 @@
 """Downloader module for URLs requiring youtube-dl support"""
 
 from .common import DownloaderBase
-from .. import text
+from .. import ytdl, text
 import os
 
 
@@ -17,70 +17,53 @@ class YoutubeDLDownloader(DownloaderBase):
     scheme = "ytdl"
 
     def __init__(self, job):
-        module = __import__(self.config("module") or "youtube_dl")
-
         DownloaderBase.__init__(self, job)
-        extractor = job.extractor
 
+        extractor = job.extractor
         retries = self.config("retries", extractor._retries)
-        options = {
+        self.ytdl_opts = {
-            "format": self.config("format") or None,
-            "ratelimit": text.parse_bytes(self.config("rate"), None),
             "retries": retries+1 if retries >= 0 else float("inf"),
             "socket_timeout": self.config("timeout", extractor._timeout),
             "nocheckcertificate": not self.config("verify", extractor._verify),
-            "nopart": not self.part,
-            "updatetime": self.config("mtime", True),
-            "proxy": extractor.session.proxies.get("http"),
-            "min_filesize": text.parse_bytes(
-                self.config("filesize-min"), None),
-            "max_filesize": text.parse_bytes(
-                self.config("filesize-max"), None),
         }
 
-        raw_options = self.config("raw-options")
+        self.ytdl_instance = None
-        if raw_options:
-            options.update(raw_options)
-
-        self.progress = self.config("progress", 3.0)
-        if self.progress is not None:
-            options["progress_hooks"] = (self._progress_hook,)
-
-        if self.config("logging", True):
-            options["logger"] = self.log
         self.forward_cookies = self.config("forward-cookies", False)
-
+        self.progress = self.config("progress", 3.0)
         self.outtmpl = self.config("outtmpl")
-        if self.outtmpl == "default":
-            self.outtmpl = module.DEFAULT_OUTTMPL
-
-        self.ytdl = module.YoutubeDL(options)
 
     def download(self, url, pathfmt):
         kwdict = pathfmt.kwdict
 
-        ytdl = kwdict.pop("_ytdl_instance", None)
+        ytdl_instance = kwdict.pop("_ytdl_instance", None)
-        if ytdl:
+        if not ytdl_instance:
-            if self.progress is not None and not ytdl._progress_hooks:
+            ytdl_instance = self.ytdl_instance
-                ytdl.add_progress_hook(self._progress_hook)
+            if not ytdl_instance:
-        else:
+                module = __import__(self.config("module") or "youtube_dl")
-            ytdl = self.ytdl
+                self.ytdl_instance = ytdl_instance = ytdl.construct_YoutubeDL(
+                    module, self, self.ytdl_opts)
+                if self.outtmpl == "default":
+                    self.outtmpl = module.DEFAULT_OUTTMPL
             if self.forward_cookies:
-                set_cookie = ytdl.cookiejar.set_cookie
+                set_cookie = ytdl_instance.cookiejar.set_cookie
                 for cookie in self.session.cookies:
                     set_cookie(cookie)
 
+        if self.progress is not None and not ytdl_instance._progress_hooks:
+            ytdl_instance.add_progress_hook(self._progress_hook)
+
         info_dict = kwdict.pop("_ytdl_info_dict", None)
         if not info_dict:
             try:
-                info_dict = ytdl.extract_info(url[5:], download=False)
+                info_dict = ytdl_instance.extract_info(url[5:], download=False)
             except Exception:
                 return False
 
         if "entries" in info_dict:
             index = kwdict.get("_ytdl_index")
             if index is None:
-                return self._download_playlist(ytdl, pathfmt, info_dict)
+                return self._download_playlist(
+                    ytdl_instance, pathfmt, info_dict)
             else:
                 info_dict = info_dict["entries"][index]
 
@@ -88,9 +71,9 @@ class YoutubeDLDownloader(DownloaderBase):
         if extra:
             info_dict.update(extra)
 
-        return self._download_video(ytdl, pathfmt, info_dict)
+        return self._download_video(ytdl_instance, pathfmt, info_dict)
 
-    def _download_video(self, ytdl, pathfmt, info_dict):
+    def _download_video(self, ytdl_instance, pathfmt, info_dict):
         if "url" in info_dict:
             text.nameext_from_url(info_dict["url"], pathfmt.kwdict)
 
@@ -99,8 +82,9 @@ class YoutubeDLDownloader(DownloaderBase):
             info_dict["ext"] = "mkv"
 
         if self.outtmpl:
-            self._set_outtmpl(ytdl, self.outtmpl)
+            self._set_outtmpl(ytdl_instance, self.outtmpl)
-            pathfmt.filename = filename = ytdl.prepare_filename(info_dict)
+            pathfmt.filename = filename = \
+                ytdl_instance.prepare_filename(info_dict)
             pathfmt.extension = info_dict["ext"]
             pathfmt.path = pathfmt.directory + filename
             pathfmt.realpath = pathfmt.temppath = (
@@ -115,22 +99,22 @@ class YoutubeDLDownloader(DownloaderBase):
             pathfmt.temppath = os.path.join(
                 self.partdir, pathfmt.filename)
 
-        self._set_outtmpl(ytdl, pathfmt.temppath.replace("%", "%%"))
+        self._set_outtmpl(ytdl_instance, pathfmt.temppath.replace("%", "%%"))
 
         self.out.start(pathfmt.path)
         try:
-            ytdl.process_info(info_dict)
+            ytdl_instance.process_info(info_dict)
         except Exception:
             self.log.debug("Traceback", exc_info=True)
             return False
         return True
 
-    def _download_playlist(self, ytdl, pathfmt, info_dict):
+    def _download_playlist(self, ytdl_instance, pathfmt, info_dict):
         pathfmt.set_extension("%(playlist_index)s.%(ext)s")
-        self._set_outtmpl(ytdl, pathfmt.realpath)
+        self._set_outtmpl(ytdl_instance, pathfmt.realpath)
 
         for entry in info_dict["entries"]:
-            ytdl.process_info(entry)
+            ytdl_instance.process_info(entry)
         return True
 
     def _progress_hook(self, info):
@@ -144,11 +128,11 @@ class YoutubeDLDownloader(DownloaderBase):
             )
 
     @staticmethod
-    def _set_outtmpl(ytdl, outtmpl):
+    def _set_outtmpl(ytdl_instance, outtmpl):
         try:
-            ytdl.outtmpl_dict["default"] = outtmpl
+            ytdl_instance.outtmpl_dict["default"] = outtmpl
         except AttributeError:
-            ytdl.params["outtmpl"] = outtmpl
+            ytdl_instance.params["outtmpl"] = outtmpl
 
 
 def compatible_formats(formats):

gallery_dl/extractor/ytdl.py

@@ -9,7 +9,7 @@
 """Extractors for sites supported by youtube-dl"""
 
 from .common import Extractor, Message
-from .. import text, config, exception
+from .. import ytdl, config, exception
 
 
 class YoutubeDLExtractor(Extractor):
@@ -54,52 +54,40 @@ class YoutubeDLExtractor(Extractor):
         self.log.debug("Using %s", ytdl_module)
 
         # construct YoutubeDL object
-        options = {
+        extr_opts = {
-            "format"                 : self.config("format"),
+            "extract_flat"           : "in_playlist",
+            "force_generic_extractor": self.force_generic_extractor,
+        }
+        user_opts = {
             "retries"                : self._retries,
             "socket_timeout"         : self._timeout,
             "nocheckcertificate"     : not self._verify,
-            "proxy"                  : self.session.proxies.get("http"),
-            "force_generic_extractor": self.force_generic_extractor,
-            "nopart"                 : not self.config("part", True),
-            "updatetime"             : self.config("mtime", True),
-            "ratelimit"              : text.parse_bytes(
-                self.config("rate"), None),
-            "min_filesize"           : text.parse_bytes(
-                self.config("filesize-min"), None),
-            "max_filesize"           : text.parse_bytes(
-                self.config("filesize-max"), None),
         }
 
-        raw_options = self.config("raw-options")
-        if raw_options:
-            options.update(raw_options)
-        if self.config("logging", True):
-            options["logger"] = self.log
-        options["extract_flat"] = "in_playlist"
-
         username, password = self._get_auth_info()
         if username:
-            options["username"], options["password"] = username, password
+            user_opts["username"], user_opts["password"] = username, password
         del username, password
 
-        ytdl = ytdl_module.YoutubeDL(options)
+        ytdl_instance = ytdl.construct_YoutubeDL(
+            ytdl_module, self, user_opts, extr_opts)
 
         # transfer cookies to ytdl
         cookies = self.session.cookies
         if cookies:
-            set_cookie = self.ytdl.cookiejar.set_cookie
+            set_cookie = ytdl_instance.cookiejar.set_cookie
-            for cookie in self.session.cookies:
+            for cookie in cookies:
                 set_cookie(cookie)
 
         # extract youtube_dl info_dict
-        info_dict = ytdl._YoutubeDL__extract_info(
+        info_dict = ytdl_instance._YoutubeDL__extract_info(
             self.ytdl_url,
-            ytdl.get_info_extractor(self.ytdl_ie_key),
+            ytdl_instance.get_info_extractor(self.ytdl_ie_key),
             False, {}, True)
 
         if "entries" in info_dict:
-            results = self._process_entries(ytdl, info_dict["entries"])
+            results = self._process_entries(
+                ytdl_instance, info_dict["entries"])
         else:
             results = (info_dict,)
 
@@ -107,7 +95,7 @@ class YoutubeDLExtractor(Extractor):
         for info_dict in results:
             info_dict["extension"] = None
             info_dict["_ytdl_info_dict"] = info_dict
-            info_dict["_ytdl_instance"] = ytdl
+            info_dict["_ytdl_instance"] = ytdl_instance
 
             url = "ytdl:" + (info_dict.get("url") or
                              info_dict.get("webpage_url") or
@@ -116,15 +104,15 @@ class YoutubeDLExtractor(Extractor):
             yield Message.Directory, info_dict
             yield Message.Url, url, info_dict
 
-    def _process_entries(self, ytdl, entries):
+    def _process_entries(self, ytdl_instance, entries):
         for entry in entries:
             if entry.get("_type") in ("url", "url_transparent"):
-                info_dict = ytdl.extract_info(
+                info_dict = ytdl_instance.extract_info(
                     entry["url"], False,
                     ie_key=entry.get("ie_key"))
                 if "entries" in info_dict:
                     yield from self._process_entries(
-                        ytdl, info_dict["entries"])
+                        ytdl_instance, info_dict["entries"])
                 else:
                     yield info_dict
             else:

gallery_dl/version.py

@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.19.2"
+__version__ = "1.19.3-dev"

gallery_dl/ytdl.py

@@ -0,0 +1,501 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2021 Mike Fährmann
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+
+"""Helpers for interacting with youtube-dl"""
+
+import re
+import shlex
+import itertools
+from . import text, util, exception
+
+
+def construct_YoutubeDL(module, obj, user_opts, system_opts=None):
+    opts = argv = None
+    config = obj.config
+
+    cfg = config("config-file")
+    if cfg:
+        with open(util.expand_path(cfg)) as fp:
+            contents = fp.read()
+        argv = shlex.split(contents, comments=True)
+
+    cmd = config("cmdline-args")
+    if cmd:
+        if isinstance(cmd, str):
+            cmd = shlex.split(cmd)
+        argv = (argv + cmd) if argv else cmd
+
+    try:
+        opts = parse_command_line(module, argv) if argv else user_opts
+    except SystemExit:
+        raise exception.StopExtraction("Invalid command-line option")
+
+    if opts.get("format") is None:
+        opts["format"] = config("format")
+    if opts.get("proxy") is None:
+        opts["proxy"] = obj.session.proxies.get("http")
+    if opts.get("nopart") is None:
+        opts["nopart"] = not config("part", True)
+    if opts.get("updatetime") is None:
+        opts["updatetime"] = config("mtime", True)
+    if opts.get("ratelimit") is None:
+        opts["ratelimit"] = text.parse_bytes(config("rate"), None)
+    if opts.get("min_filesize") is None:
+        opts["min_filesize"] = text.parse_bytes(config("filesize-min"), None)
+    if opts.get("max_filesize") is None:
+        opts["max_filesize"] = text.parse_bytes(config("filesize-max"), None)
+
+    raw_opts = config("raw-options")
+    if raw_opts:
+        opts.update(raw_opts)
+    if config("logging", True):
+        opts["logger"] = obj.log
+    if system_opts:
+        opts.update(system_opts)
+
+    return module.YoutubeDL(opts)
+
+
+def parse_command_line(module, argv):
+    parser, opts, args = module.parseOpts(argv)
+
+    ytdlp = (module.__name__ == "yt_dlp")
+    std_headers = module.std_headers
+    parse_bytes = module.FileDownloader.parse_bytes
+
+    # HTTP headers
+    if opts.user_agent is not None:
+        std_headers["User-Agent"] = opts.user_agent
+    if opts.referer is not None:
+        std_headers["Referer"] = opts.referer
+    if opts.headers:
+        if isinstance(opts.headers, dict):
+            std_headers.update(opts.headers)
+        else:
+            for h in opts.headers:
+                key, _, value = h.partition(":")
+                std_headers[key] = value
+
+    if opts.ratelimit is not None:
+        opts.ratelimit = parse_bytes(opts.ratelimit)
+    if getattr(opts, "throttledratelimit", None) is not None:
+        opts.throttledratelimit = parse_bytes(opts.throttledratelimit)
+    if opts.min_filesize is not None:
+        opts.min_filesize = parse_bytes(opts.min_filesize)
+    if opts.max_filesize is not None:
+        opts.max_filesize = parse_bytes(opts.max_filesize)
+    if opts.max_sleep_interval is None:
+        opts.max_sleep_interval = opts.sleep_interval
+    if getattr(opts, "overwrites", None):
+        opts.continue_dl = False
+    if opts.retries is not None:
+        opts.retries = parse_retries(opts.retries)
+    if opts.fragment_retries is not None:
+        opts.fragment_retries = parse_retries(opts.fragment_retries)
+    if getattr(opts, "extractor_retries", None) is not None:
+        opts.extractor_retries = parse_retries(opts.extractor_retries)
+    if opts.buffersize is not None:
+        opts.buffersize = parse_bytes(opts.buffersize)
+    if opts.http_chunk_size is not None:
+        opts.http_chunk_size = parse_bytes(opts.http_chunk_size)
+    if opts.extractaudio:
+        opts.audioformat = opts.audioformat.lower()
+    if opts.audioquality:
+        opts.audioquality = opts.audioquality.strip("kK")
+    if opts.recodevideo is not None:
+        opts.recodevideo = opts.recodevideo.replace(" ", "")
+    if getattr(opts, "remuxvideo", None) is not None:
+        opts.remuxvideo = opts.remuxvideo.replace(" ", "")
+
+    if opts.date is not None:
+        date = module.DateRange.day(opts.date)
+    else:
+        date = module.DateRange(opts.dateafter, opts.datebefore)
+
+    compat_opts = getattr(opts, "compat_opts", ())
+
+    def _unused_compat_opt(name):
+        if name not in compat_opts:
+            return False
+        compat_opts.discard(name)
+        compat_opts.update(["*%s" % name])
+        return True
+
+    def set_default_compat(
+            compat_name, opt_name, default=True, remove_compat=True):
+        attr = getattr(opts, opt_name, None)
+        if compat_name in compat_opts:
+            if attr is None:
+                setattr(opts, opt_name, not default)
+                return True
+            else:
+                if remove_compat:
+                    _unused_compat_opt(compat_name)
+                return False
+        elif attr is None:
+            setattr(opts, opt_name, default)
+        return None
+
+    set_default_compat("abort-on-error", "ignoreerrors", "only_download")
+    set_default_compat("no-playlist-metafiles", "allow_playlist_files")
+    set_default_compat("no-clean-infojson", "clean_infojson")
+    if "format-sort" in compat_opts:
+        opts.format_sort.extend(module.InfoExtractor.FormatSort.ytdl_default)
+    _video_multistreams_set = set_default_compat(
+        "multistreams", "allow_multiple_video_streams",
+        False, remove_compat=False)
+    _audio_multistreams_set = set_default_compat(
+        "multistreams", "allow_multiple_audio_streams",
+        False, remove_compat=False)
+    if _video_multistreams_set is False and _audio_multistreams_set is False:
+        _unused_compat_opt("multistreams")
+
+    outtmpl = opts.outtmpl
+    outtmpl_default = \
+        outtmpl.get("default") if isinstance(outtmpl, dict) else outtmpl
+
+    if "filename" in compat_opts:
+        if outtmpl_default is None:
+            outtmpl_default = outtmpl["default"] = "%(title)s-%(id)s.%(ext)s"
+        else:
+            _unused_compat_opt("filename")
+
+    if opts.extractaudio and not opts.keepvideo and opts.format is None:
+        opts.format = "bestaudio/best"
+
+    def metadataparser_actions(f):
+        if isinstance(f, str):
+            return (module.MetadataFromFieldPP.to_action(f),)
+        return ((module.MetadataParserPP.Actions.REPLACE, x, *f[1:])
+                for x in f[0].split(","))
+
+    if getattr(opts, "parse_metadata", None) is None:
+        opts.parse_metadata = []
+    if opts.metafromtitle is not None:
+        opts.parse_metadata.append("title:%s" % opts.metafromtitle)
+    opts.parse_metadata = list(itertools.chain(*map(
+        metadataparser_actions, opts.parse_metadata)))
+
+    download_archive_fn = module.expand_path(opts.download_archive) \
+        if opts.download_archive is not None else opts.download_archive
+
+    printing_json = opts.dumpjson or opts.print_json or opts.dump_single_json
+    if getattr(opts, "getcomments", None) and not printing_json:
+        opts.writeinfojson = True
+
+    if getattr(opts, "no_sponsorblock", None):
+        opts.sponsorblock_mark = set()
+        opts.sponsorblock_remove = set()
+    else:
+        opts.sponsorblock_mark = \
+            getattr(opts, "sponsorblock_mark", None) or set()
+        opts.sponsorblock_remove = \
+            getattr(opts, "sponsorblock_remove", None) or set()
+    sponsorblock_query = opts.sponsorblock_mark | opts.sponsorblock_remove
+
+    addchapters = getattr(opts, "addchapters", None)
+    if (opts.addmetadata or opts.sponsorblock_mark) and addchapters is None:
+        addchapters = True
+    opts.remove_chapters = getattr(opts, "remove_chapters", None) or ()
+
+    # PostProcessors
+    postprocessors = []
+    if getattr(opts, "add_postprocessors", None):
+        postprocessors += list(opts.add_postprocessors)
+    if sponsorblock_query:
+        postprocessors.append({
+            "key": "SponsorBlock",
+            "categories": sponsorblock_query,
+            "api": opts.sponsorblock_api,
+            "when": "pre_process",
+        })
+    if opts.parse_metadata:
+        postprocessors.append({
+            "key": "MetadataParser",
+            "actions": opts.parse_metadata,
+            "when": "pre_process",
+        })
+    if opts.convertsubtitles:
+        postprocessors.append({
+            "key": "FFmpegSubtitlesConvertor",
+            "format": opts.convertsubtitles,
+            "when": "before_dl",
+        })
+    if getattr(opts, "convertthumbnails", None):
+        postprocessors.append({
+            "key": "FFmpegThumbnailsConvertor",
+            "format": opts.convertthumbnails,
+            "when": "before_dl",
+        })
+    if getattr(opts, "exec_before_dl_cmd", None):
+        postprocessors.append({
+            "key": "Exec",
+            "exec_cmd": opts.exec_before_dl_cmd,
+            "when": "before_dl",
+        })
+    if opts.extractaudio:
+        postprocessors.append({
+            "key": "FFmpegExtractAudio",
+            "preferredcodec": opts.audioformat,
+            "preferredquality": opts.audioquality,
+            "nopostoverwrites": opts.nopostoverwrites,
+        })
+    if getattr(opts, "remuxvideo", None):
+        postprocessors.append({
+            "key": "FFmpegVideoRemuxer",
+            "preferedformat": opts.remuxvideo,
+        })
+    if opts.recodevideo:
+        postprocessors.append({
+            "key": "FFmpegVideoConvertor",
+            "preferedformat": opts.recodevideo,
+        })
+    if opts.embedsubtitles:
+        pp = {"key": "FFmpegEmbedSubtitle"}
+        if ytdlp:
+            pp["already_have_subtitle"] = (
+                opts.writesubtitles and "no-keep-subs" not in compat_opts)
+        postprocessors.append(pp)
+        if not opts.writeautomaticsub and "no-keep-subs" not in compat_opts:
+            opts.writesubtitles = True
+    if opts.allsubtitles and not opts.writeautomaticsub:
+        opts.writesubtitles = True
+    remove_chapters_patterns, remove_ranges = [], []
+    for regex in opts.remove_chapters:
+        if regex.startswith("*"):
+            dur = list(map(module.parse_duration, regex[1:].split("-")))
+            if len(dur) == 2 and all(t is not None for t in dur):
+                remove_ranges.append(tuple(dur))
+                continue
+        remove_chapters_patterns.append(re.compile(regex))
+    if opts.remove_chapters or sponsorblock_query:
+        postprocessors.append({
+            "key": "ModifyChapters",
+            "remove_chapters_patterns": remove_chapters_patterns,
+            "remove_sponsor_segments": opts.sponsorblock_remove,
+            "remove_ranges": remove_ranges,
+            "sponsorblock_chapter_title": opts.sponsorblock_chapter_title,
+            "force_keyframes": opts.force_keyframes_at_cuts,
+        })
+    if opts.addmetadata or addchapters:
+        pp = {"key": "FFmpegMetadata"}
+        if ytdlp:
+            pp["add_chapters"] = addchapters
+            pp["add_metadata"] = opts.addmetadata
+        postprocessors.append(pp)
+    if getattr(opts, "sponskrub", False) is not False:
+        postprocessors.append({
+            "key": "SponSkrub",
+            "path": opts.sponskrub_path,
+            "args": opts.sponskrub_args,
+            "cut": opts.sponskrub_cut,
+            "force": opts.sponskrub_force,
+            "ignoreerror": opts.sponskrub is None,
+        })
+    if opts.embedthumbnail:
+        already_have_thumbnail = (opts.writethumbnail or
+                                  opts.write_all_thumbnails)
+        postprocessors.append({
+            "key": "EmbedThumbnail",
+            "already_have_thumbnail": already_have_thumbnail,
+        })
+        if not already_have_thumbnail:
+            opts.writethumbnail = True
+            opts.outtmpl["pl_thumbnail"] = ""
+    if getattr(opts, "split_chapters", None):
+        postprocessors.append({
+            "key": "FFmpegSplitChapters",
+            "force_keyframes": opts.force_keyframes_at_cuts,
+        })
+    if opts.xattrs:
+        postprocessors.append({"key": "XAttrMetadata"})
+    if opts.exec_cmd:
+        postprocessors.append({
+            "key": "Exec",
+            "exec_cmd": opts.exec_cmd,
+            "when": "after_move",
+        })
+
+    match_filter = (
+        None if opts.match_filter is None
+        else module.match_filter_func(opts.match_filter))
+
+    return {
+        "usenetrc": opts.usenetrc,
+        "netrc_location": getattr(opts, "netrc_location", None),
+        "username": opts.username,
+        "password": opts.password,
+        "twofactor": opts.twofactor,
+        "videopassword": opts.videopassword,
+        "ap_mso": opts.ap_mso,
+        "ap_username": opts.ap_username,
+        "ap_password": opts.ap_password,
+        "quiet": opts.quiet,
+        "no_warnings": opts.no_warnings,
+        "forceurl": opts.geturl,
+        "forcetitle": opts.gettitle,
+        "forceid": opts.getid,
+        "forcethumbnail": opts.getthumbnail,
+        "forcedescription": opts.getdescription,
+        "forceduration": opts.getduration,
+        "forcefilename": opts.getfilename,
+        "forceformat": opts.getformat,
+        "forceprint": getattr(opts, "forceprint", None) or (),
+        "forcejson": opts.dumpjson or opts.print_json,
+        "dump_single_json": opts.dump_single_json,
+        "force_write_download_archive": getattr(
+            opts, "force_write_download_archive", None),
+        "simulate": opts.simulate,
+        "skip_download": opts.skip_download,
+        "format": opts.format,
+        "allow_unplayable_formats": getattr(
+            opts, "allow_unplayable_formats", None),
+        "ignore_no_formats_error": getattr(
+            opts, "ignore_no_formats_error", None),
+        "format_sort": getattr(
+            opts, "format_sort", None),
+        "format_sort_force": getattr(
+            opts, "format_sort_force", None),
+        "allow_multiple_video_streams": opts.allow_multiple_video_streams,
+        "allow_multiple_audio_streams": opts.allow_multiple_audio_streams,
+        "check_formats": getattr(
+            opts, "check_formats", None),
+        "listformats": opts.listformats,
+        "listformats_table": getattr(
+            opts, "listformats_table", None),
+        "outtmpl": opts.outtmpl,
+        "outtmpl_na_placeholder": opts.outtmpl_na_placeholder,
+        "paths": getattr(opts, "paths", None),
+        "autonumber_size": opts.autonumber_size,
+        "autonumber_start": opts.autonumber_start,
+        "restrictfilenames": opts.restrictfilenames,
+        "windowsfilenames": getattr(opts, "windowsfilenames", None),
+        "ignoreerrors": opts.ignoreerrors,
+        "force_generic_extractor": opts.force_generic_extractor,
+        "ratelimit": opts.ratelimit,
+        "throttledratelimit": getattr(opts, "throttledratelimit", None),
+        "overwrites": getattr(opts, "overwrites", None),
+        "retries": opts.retries,
+        "fragment_retries": opts.fragment_retries,
+        "extractor_retries": getattr(opts, "extractor_retries", None),
+        "skip_unavailable_fragments": opts.skip_unavailable_fragments,
+        "keep_fragments": opts.keep_fragments,
+        "concurrent_fragment_downloads": getattr(
+            opts, "concurrent_fragment_downloads", None),
+        "buffersize": opts.buffersize,
+        "noresizebuffer": opts.noresizebuffer,
+        "http_chunk_size": opts.http_chunk_size,
+        "continuedl": opts.continue_dl,
+        "noprogress": True if opts.noprogress is None else opts.noprogress,
+        "playliststart": opts.playliststart,
+        "playlistend": opts.playlistend,
+        "playlistreverse": opts.playlist_reverse,
+        "playlistrandom": opts.playlist_random,
+        "noplaylist": opts.noplaylist,
+        "logtostderr": outtmpl_default == "-",
+        "consoletitle": opts.consoletitle,
+        "nopart": opts.nopart,
+        "updatetime": opts.updatetime,
+        "writedescription": opts.writedescription,
+        "writeannotations": opts.writeannotations,
+        "writeinfojson": opts.writeinfojson,
+        "allow_playlist_files": opts.allow_playlist_files,
+        "clean_infojson": opts.clean_infojson,
+        "getcomments": getattr(opts, "getcomments", None),
+        "writethumbnail": opts.writethumbnail,
+        "write_all_thumbnails": opts.write_all_thumbnails,
+        "writelink": getattr(opts, "writelink", None),
+        "writeurllink": getattr(opts, "writeurllink", None),
+        "writewebloclink": getattr(opts, "writewebloclink", None),
+        "writedesktoplink": getattr(opts, "writedesktoplink", None),
+        "writesubtitles": opts.writesubtitles,
+        "writeautomaticsub": opts.writeautomaticsub,
+        "allsubtitles": opts.allsubtitles,
+        "listsubtitles": opts.listsubtitles,
+        "subtitlesformat": opts.subtitlesformat,
+        "subtitleslangs": opts.subtitleslangs,
+        "matchtitle": module.decodeOption(opts.matchtitle),
+        "rejecttitle": module.decodeOption(opts.rejecttitle),
+        "max_downloads": opts.max_downloads,
+        "prefer_free_formats": opts.prefer_free_formats,
+        "trim_file_name": getattr(opts, "trim_file_name", None),
+        "verbose": opts.verbose,
+        "dump_intermediate_pages": opts.dump_intermediate_pages,
+        "write_pages": opts.write_pages,
+        "test": opts.test,
+        "keepvideo": opts.keepvideo,
+        "min_filesize": opts.min_filesize,
+        "max_filesize": opts.max_filesize,
+        "min_views": opts.min_views,
+        "max_views": opts.max_views,
+        "daterange": date,
+        "cachedir": opts.cachedir,
+        "youtube_print_sig_code": opts.youtube_print_sig_code,
+        "age_limit": opts.age_limit,
+        "download_archive": download_archive_fn,
+        "break_on_existing": getattr(opts, "break_on_existing", None),
+        "break_on_reject": getattr(opts, "break_on_reject", None),
+        "skip_playlist_after_errors": getattr(
+            opts, "skip_playlist_after_errors", None),
+        "cookiefile": opts.cookiefile,
+        "cookiesfrombrowser": getattr(opts, "cookiesfrombrowser", None),
+        "nocheckcertificate": opts.no_check_certificate,
+        "prefer_insecure": opts.prefer_insecure,
+        "proxy": opts.proxy,
+        "socket_timeout": opts.socket_timeout,
+        "bidi_workaround": opts.bidi_workaround,
+        "debug_printtraffic": opts.debug_printtraffic,
+        "prefer_ffmpeg": opts.prefer_ffmpeg,
+        "include_ads": opts.include_ads,
+        "default_search": opts.default_search,
+        "dynamic_mpd": getattr(opts, "dynamic_mpd", None),
+        "extractor_args": getattr(opts, "extractor_args", None),
+        "youtube_include_dash_manifest": getattr(
+            opts, "youtube_include_dash_manifest", None),
+        "youtube_include_hls_manifest": getattr(
+            opts, "youtube_include_hls_manifest", None),
+        "encoding": opts.encoding,
+        "extract_flat": opts.extract_flat,
+        "mark_watched": opts.mark_watched,
+        "merge_output_format": opts.merge_output_format,
+        "postprocessors": postprocessors,
+        "fixup": opts.fixup,
+        "source_address": opts.source_address,
+        "call_home": opts.call_home,
+        "sleep_interval_requests": getattr(
+            opts, "sleep_interval_requests", None),
+        "sleep_interval": opts.sleep_interval,
+        "max_sleep_interval": opts.max_sleep_interval,
+        "sleep_interval_subtitles": getattr(
+            opts, "sleep_interval_subtitles", None),
+        "external_downloader": opts.external_downloader,
+        "list_thumbnails": opts.list_thumbnails,
+        "playlist_items": opts.playlist_items,
+        "xattr_set_filesize": opts.xattr_set_filesize,
+        "match_filter": match_filter,
+        "no_color": opts.no_color,
+        "ffmpeg_location": opts.ffmpeg_location,
+        "hls_prefer_native": opts.hls_prefer_native,
+        "hls_use_mpegts": opts.hls_use_mpegts,
+        "hls_split_discontinuity": getattr(
+            opts, "hls_split_discontinuity", None),
+        "external_downloader_args": opts.external_downloader_args,
+        "postprocessor_args": opts.postprocessor_args,
+        "cn_verification_proxy": opts.cn_verification_proxy,
+        "geo_verification_proxy": opts.geo_verification_proxy,
+        "geo_bypass": opts.geo_bypass,
+        "geo_bypass_country": opts.geo_bypass_country,
+        "geo_bypass_ip_block": opts.geo_bypass_ip_block,
+        "compat_opts": compat_opts,
+    }
+
+
+def parse_retries(retries, name=""):
+    if retries in ("inf", "infinite"):
+        return float("inf")
+    return int(retries)