From efa178cc91407673f427f33811035a4ef8395c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 7 Nov 2021 02:44:11 +0100 Subject: [PATCH] [ytdl] implement parsing ytdl command-line options (#1680) - adds 'config-file' and 'cmdline-args' options for both ytdl downloader and extractor - create 'ytdl' helper module, which combines YoutubeDL creation and option parsing. - most likely a buggy mess due to incompatibilities between the original youtube-dl and yt-dlp. --- CHANGELOG.md | 2 + gallery_dl/downloader/ytdl.py | 84 +++--- gallery_dl/extractor/ytdl.py | 50 ++-- gallery_dl/version.py | 2 +- gallery_dl/ytdl.py | 501 ++++++++++++++++++++++++++++++++++ 5 files changed, 557 insertions(+), 82 deletions(-) create mode 100644 gallery_dl/ytdl.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fa15402..28a01eea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## Unreleased + ## 1.19.2 - 2021-11-05 ### Additions - [kemonoparty] add `comments` option ([#1980](https://github.com/mikf/gallery-dl/issues/1980)) diff --git a/gallery_dl/downloader/ytdl.py b/gallery_dl/downloader/ytdl.py index f4d3e05f..b1239bbb 100644 --- a/gallery_dl/downloader/ytdl.py +++ b/gallery_dl/downloader/ytdl.py @@ -9,7 +9,7 @@ """Downloader module for URLs requiring youtube-dl support""" from .common import DownloaderBase -from .. import text +from .. import ytdl, text import os @@ -17,70 +17,53 @@ class YoutubeDLDownloader(DownloaderBase): scheme = "ytdl" def __init__(self, job): - module = __import__(self.config("module") or "youtube_dl") - DownloaderBase.__init__(self, job) - extractor = job.extractor + extractor = job.extractor retries = self.config("retries", extractor._retries) - options = { - "format": self.config("format") or None, - "ratelimit": text.parse_bytes(self.config("rate"), None), + self.ytdl_opts = { "retries": retries+1 if retries >= 0 else float("inf"), "socket_timeout": self.config("timeout", extractor._timeout), "nocheckcertificate": not self.config("verify", extractor._verify), - "nopart": not self.part, - "updatetime": self.config("mtime", True), - "proxy": extractor.session.proxies.get("http"), - "min_filesize": text.parse_bytes( - self.config("filesize-min"), None), - "max_filesize": text.parse_bytes( - self.config("filesize-max"), None), } - raw_options = self.config("raw-options") - if raw_options: - options.update(raw_options) - - self.progress = self.config("progress", 3.0) - if self.progress is not None: - options["progress_hooks"] = (self._progress_hook,) - - if self.config("logging", True): - options["logger"] = self.log + self.ytdl_instance = None self.forward_cookies = self.config("forward-cookies", False) - + self.progress = self.config("progress", 3.0) self.outtmpl = self.config("outtmpl") - if self.outtmpl == "default": - self.outtmpl = module.DEFAULT_OUTTMPL - - self.ytdl = module.YoutubeDL(options) def download(self, url, pathfmt): kwdict = pathfmt.kwdict - ytdl = kwdict.pop("_ytdl_instance", None) - if ytdl: - if self.progress is not None and not ytdl._progress_hooks: - ytdl.add_progress_hook(self._progress_hook) - else: - ytdl = self.ytdl + ytdl_instance = kwdict.pop("_ytdl_instance", None) + if not ytdl_instance: + ytdl_instance = self.ytdl_instance + if not ytdl_instance: + module = __import__(self.config("module") or "youtube_dl") + self.ytdl_instance = ytdl_instance = ytdl.construct_YoutubeDL( + module, self, self.ytdl_opts) + if self.outtmpl == "default": + self.outtmpl = module.DEFAULT_OUTTMPL if self.forward_cookies: - set_cookie = ytdl.cookiejar.set_cookie + set_cookie = ytdl_instance.cookiejar.set_cookie for cookie in self.session.cookies: set_cookie(cookie) + if self.progress is not None and not ytdl_instance._progress_hooks: + ytdl_instance.add_progress_hook(self._progress_hook) + info_dict = kwdict.pop("_ytdl_info_dict", None) if not info_dict: try: - info_dict = ytdl.extract_info(url[5:], download=False) + info_dict = ytdl_instance.extract_info(url[5:], download=False) except Exception: return False if "entries" in info_dict: index = kwdict.get("_ytdl_index") if index is None: - return self._download_playlist(ytdl, pathfmt, info_dict) + return self._download_playlist( + ytdl_instance, pathfmt, info_dict) else: info_dict = info_dict["entries"][index] @@ -88,9 +71,9 @@ class YoutubeDLDownloader(DownloaderBase): if extra: info_dict.update(extra) - return self._download_video(ytdl, pathfmt, info_dict) + return self._download_video(ytdl_instance, pathfmt, info_dict) - def _download_video(self, ytdl, pathfmt, info_dict): + def _download_video(self, ytdl_instance, pathfmt, info_dict): if "url" in info_dict: text.nameext_from_url(info_dict["url"], pathfmt.kwdict) @@ -99,8 +82,9 @@ class YoutubeDLDownloader(DownloaderBase): info_dict["ext"] = "mkv" if self.outtmpl: - self._set_outtmpl(ytdl, self.outtmpl) - pathfmt.filename = filename = ytdl.prepare_filename(info_dict) + self._set_outtmpl(ytdl_instance, self.outtmpl) + pathfmt.filename = filename = \ + ytdl_instance.prepare_filename(info_dict) pathfmt.extension = info_dict["ext"] pathfmt.path = pathfmt.directory + filename pathfmt.realpath = pathfmt.temppath = ( @@ -115,22 +99,22 @@ class YoutubeDLDownloader(DownloaderBase): pathfmt.temppath = os.path.join( self.partdir, pathfmt.filename) - self._set_outtmpl(ytdl, pathfmt.temppath.replace("%", "%%")) + self._set_outtmpl(ytdl_instance, pathfmt.temppath.replace("%", "%%")) self.out.start(pathfmt.path) try: - ytdl.process_info(info_dict) + ytdl_instance.process_info(info_dict) except Exception: self.log.debug("Traceback", exc_info=True) return False return True - def _download_playlist(self, ytdl, pathfmt, info_dict): + def _download_playlist(self, ytdl_instance, pathfmt, info_dict): pathfmt.set_extension("%(playlist_index)s.%(ext)s") - self._set_outtmpl(ytdl, pathfmt.realpath) + self._set_outtmpl(ytdl_instance, pathfmt.realpath) for entry in info_dict["entries"]: - ytdl.process_info(entry) + ytdl_instance.process_info(entry) return True def _progress_hook(self, info): @@ -144,11 +128,11 @@ class YoutubeDLDownloader(DownloaderBase): ) @staticmethod - def _set_outtmpl(ytdl, outtmpl): + def _set_outtmpl(ytdl_instance, outtmpl): try: - ytdl.outtmpl_dict["default"] = outtmpl + ytdl_instance.outtmpl_dict["default"] = outtmpl except AttributeError: - ytdl.params["outtmpl"] = outtmpl + ytdl_instance.params["outtmpl"] = outtmpl def compatible_formats(formats): diff --git a/gallery_dl/extractor/ytdl.py b/gallery_dl/extractor/ytdl.py index d380daba..e3049e30 100644 --- a/gallery_dl/extractor/ytdl.py +++ b/gallery_dl/extractor/ytdl.py @@ -9,7 +9,7 @@ """Extractors for sites supported by youtube-dl""" from .common import Extractor, Message -from .. import text, config, exception +from .. import ytdl, config, exception class YoutubeDLExtractor(Extractor): @@ -54,52 +54,40 @@ class YoutubeDLExtractor(Extractor): self.log.debug("Using %s", ytdl_module) # construct YoutubeDL object - options = { - "format" : self.config("format"), + extr_opts = { + "extract_flat" : "in_playlist", + "force_generic_extractor": self.force_generic_extractor, + } + user_opts = { "retries" : self._retries, "socket_timeout" : self._timeout, "nocheckcertificate" : not self._verify, - "proxy" : self.session.proxies.get("http"), - "force_generic_extractor": self.force_generic_extractor, - "nopart" : not self.config("part", True), - "updatetime" : self.config("mtime", True), - "ratelimit" : text.parse_bytes( - self.config("rate"), None), - "min_filesize" : text.parse_bytes( - self.config("filesize-min"), None), - "max_filesize" : text.parse_bytes( - self.config("filesize-max"), None), } - raw_options = self.config("raw-options") - if raw_options: - options.update(raw_options) - if self.config("logging", True): - options["logger"] = self.log - options["extract_flat"] = "in_playlist" - username, password = self._get_auth_info() if username: - options["username"], options["password"] = username, password + user_opts["username"], user_opts["password"] = username, password del username, password - ytdl = ytdl_module.YoutubeDL(options) + ytdl_instance = ytdl.construct_YoutubeDL( + ytdl_module, self, user_opts, extr_opts) # transfer cookies to ytdl cookies = self.session.cookies if cookies: - set_cookie = self.ytdl.cookiejar.set_cookie - for cookie in self.session.cookies: + set_cookie = ytdl_instance.cookiejar.set_cookie + for cookie in cookies: set_cookie(cookie) # extract youtube_dl info_dict - info_dict = ytdl._YoutubeDL__extract_info( + info_dict = ytdl_instance._YoutubeDL__extract_info( self.ytdl_url, - ytdl.get_info_extractor(self.ytdl_ie_key), + ytdl_instance.get_info_extractor(self.ytdl_ie_key), False, {}, True) if "entries" in info_dict: - results = self._process_entries(ytdl, info_dict["entries"]) + results = self._process_entries( + ytdl_instance, info_dict["entries"]) else: results = (info_dict,) @@ -107,7 +95,7 @@ class YoutubeDLExtractor(Extractor): for info_dict in results: info_dict["extension"] = None info_dict["_ytdl_info_dict"] = info_dict - info_dict["_ytdl_instance"] = ytdl + info_dict["_ytdl_instance"] = ytdl_instance url = "ytdl:" + (info_dict.get("url") or info_dict.get("webpage_url") or @@ -116,15 +104,15 @@ class YoutubeDLExtractor(Extractor): yield Message.Directory, info_dict yield Message.Url, url, info_dict - def _process_entries(self, ytdl, entries): + def _process_entries(self, ytdl_instance, entries): for entry in entries: if entry.get("_type") in ("url", "url_transparent"): - info_dict = ytdl.extract_info( + info_dict = ytdl_instance.extract_info( entry["url"], False, ie_key=entry.get("ie_key")) if "entries" in info_dict: yield from self._process_entries( - ytdl, info_dict["entries"]) + ytdl_instance, info_dict["entries"]) else: yield info_dict else: diff --git a/gallery_dl/version.py b/gallery_dl/version.py index 48817be0..376f59c4 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,4 +6,4 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.19.2" +__version__ = "1.19.3-dev" diff --git a/gallery_dl/ytdl.py b/gallery_dl/ytdl.py new file mode 100644 index 00000000..02f9d77e --- /dev/null +++ b/gallery_dl/ytdl.py @@ -0,0 +1,501 @@ +# -*- coding: utf-8 -*- + +# Copyright 2021 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +"""Helpers for interacting with youtube-dl""" + +import re +import shlex +import itertools +from . import text, util, exception + + +def construct_YoutubeDL(module, obj, user_opts, system_opts=None): + opts = argv = None + config = obj.config + + cfg = config("config-file") + if cfg: + with open(util.expand_path(cfg)) as fp: + contents = fp.read() + argv = shlex.split(contents, comments=True) + + cmd = config("cmdline-args") + if cmd: + if isinstance(cmd, str): + cmd = shlex.split(cmd) + argv = (argv + cmd) if argv else cmd + + try: + opts = parse_command_line(module, argv) if argv else user_opts + except SystemExit: + raise exception.StopExtraction("Invalid command-line option") + + if opts.get("format") is None: + opts["format"] = config("format") + if opts.get("proxy") is None: + opts["proxy"] = obj.session.proxies.get("http") + if opts.get("nopart") is None: + opts["nopart"] = not config("part", True) + if opts.get("updatetime") is None: + opts["updatetime"] = config("mtime", True) + if opts.get("ratelimit") is None: + opts["ratelimit"] = text.parse_bytes(config("rate"), None) + if opts.get("min_filesize") is None: + opts["min_filesize"] = text.parse_bytes(config("filesize-min"), None) + if opts.get("max_filesize") is None: + opts["max_filesize"] = text.parse_bytes(config("filesize-max"), None) + + raw_opts = config("raw-options") + if raw_opts: + opts.update(raw_opts) + if config("logging", True): + opts["logger"] = obj.log + if system_opts: + opts.update(system_opts) + + return module.YoutubeDL(opts) + + +def parse_command_line(module, argv): + parser, opts, args = module.parseOpts(argv) + + ytdlp = (module.__name__ == "yt_dlp") + std_headers = module.std_headers + parse_bytes = module.FileDownloader.parse_bytes + + # HTTP headers + if opts.user_agent is not None: + std_headers["User-Agent"] = opts.user_agent + if opts.referer is not None: + std_headers["Referer"] = opts.referer + if opts.headers: + if isinstance(opts.headers, dict): + std_headers.update(opts.headers) + else: + for h in opts.headers: + key, _, value = h.partition(":") + std_headers[key] = value + + if opts.ratelimit is not None: + opts.ratelimit = parse_bytes(opts.ratelimit) + if getattr(opts, "throttledratelimit", None) is not None: + opts.throttledratelimit = parse_bytes(opts.throttledratelimit) + if opts.min_filesize is not None: + opts.min_filesize = parse_bytes(opts.min_filesize) + if opts.max_filesize is not None: + opts.max_filesize = parse_bytes(opts.max_filesize) + if opts.max_sleep_interval is None: + opts.max_sleep_interval = opts.sleep_interval + if getattr(opts, "overwrites", None): + opts.continue_dl = False + if opts.retries is not None: + opts.retries = parse_retries(opts.retries) + if opts.fragment_retries is not None: + opts.fragment_retries = parse_retries(opts.fragment_retries) + if getattr(opts, "extractor_retries", None) is not None: + opts.extractor_retries = parse_retries(opts.extractor_retries) + if opts.buffersize is not None: + opts.buffersize = parse_bytes(opts.buffersize) + if opts.http_chunk_size is not None: + opts.http_chunk_size = parse_bytes(opts.http_chunk_size) + if opts.extractaudio: + opts.audioformat = opts.audioformat.lower() + if opts.audioquality: + opts.audioquality = opts.audioquality.strip("kK") + if opts.recodevideo is not None: + opts.recodevideo = opts.recodevideo.replace(" ", "") + if getattr(opts, "remuxvideo", None) is not None: + opts.remuxvideo = opts.remuxvideo.replace(" ", "") + + if opts.date is not None: + date = module.DateRange.day(opts.date) + else: + date = module.DateRange(opts.dateafter, opts.datebefore) + + compat_opts = getattr(opts, "compat_opts", ()) + + def _unused_compat_opt(name): + if name not in compat_opts: + return False + compat_opts.discard(name) + compat_opts.update(["*%s" % name]) + return True + + def set_default_compat( + compat_name, opt_name, default=True, remove_compat=True): + attr = getattr(opts, opt_name, None) + if compat_name in compat_opts: + if attr is None: + setattr(opts, opt_name, not default) + return True + else: + if remove_compat: + _unused_compat_opt(compat_name) + return False + elif attr is None: + setattr(opts, opt_name, default) + return None + + set_default_compat("abort-on-error", "ignoreerrors", "only_download") + set_default_compat("no-playlist-metafiles", "allow_playlist_files") + set_default_compat("no-clean-infojson", "clean_infojson") + if "format-sort" in compat_opts: + opts.format_sort.extend(module.InfoExtractor.FormatSort.ytdl_default) + _video_multistreams_set = set_default_compat( + "multistreams", "allow_multiple_video_streams", + False, remove_compat=False) + _audio_multistreams_set = set_default_compat( + "multistreams", "allow_multiple_audio_streams", + False, remove_compat=False) + if _video_multistreams_set is False and _audio_multistreams_set is False: + _unused_compat_opt("multistreams") + + outtmpl = opts.outtmpl + outtmpl_default = \ + outtmpl.get("default") if isinstance(outtmpl, dict) else outtmpl + + if "filename" in compat_opts: + if outtmpl_default is None: + outtmpl_default = outtmpl["default"] = "%(title)s-%(id)s.%(ext)s" + else: + _unused_compat_opt("filename") + + if opts.extractaudio and not opts.keepvideo and opts.format is None: + opts.format = "bestaudio/best" + + def metadataparser_actions(f): + if isinstance(f, str): + return (module.MetadataFromFieldPP.to_action(f),) + return ((module.MetadataParserPP.Actions.REPLACE, x, *f[1:]) + for x in f[0].split(",")) + + if getattr(opts, "parse_metadata", None) is None: + opts.parse_metadata = [] + if opts.metafromtitle is not None: + opts.parse_metadata.append("title:%s" % opts.metafromtitle) + opts.parse_metadata = list(itertools.chain(*map( + metadataparser_actions, opts.parse_metadata))) + + download_archive_fn = module.expand_path(opts.download_archive) \ + if opts.download_archive is not None else opts.download_archive + + printing_json = opts.dumpjson or opts.print_json or opts.dump_single_json + if getattr(opts, "getcomments", None) and not printing_json: + opts.writeinfojson = True + + if getattr(opts, "no_sponsorblock", None): + opts.sponsorblock_mark = set() + opts.sponsorblock_remove = set() + else: + opts.sponsorblock_mark = \ + getattr(opts, "sponsorblock_mark", None) or set() + opts.sponsorblock_remove = \ + getattr(opts, "sponsorblock_remove", None) or set() + sponsorblock_query = opts.sponsorblock_mark | opts.sponsorblock_remove + + addchapters = getattr(opts, "addchapters", None) + if (opts.addmetadata or opts.sponsorblock_mark) and addchapters is None: + addchapters = True + opts.remove_chapters = getattr(opts, "remove_chapters", None) or () + + # PostProcessors + postprocessors = [] + if getattr(opts, "add_postprocessors", None): + postprocessors += list(opts.add_postprocessors) + if sponsorblock_query: + postprocessors.append({ + "key": "SponsorBlock", + "categories": sponsorblock_query, + "api": opts.sponsorblock_api, + "when": "pre_process", + }) + if opts.parse_metadata: + postprocessors.append({ + "key": "MetadataParser", + "actions": opts.parse_metadata, + "when": "pre_process", + }) + if opts.convertsubtitles: + postprocessors.append({ + "key": "FFmpegSubtitlesConvertor", + "format": opts.convertsubtitles, + "when": "before_dl", + }) + if getattr(opts, "convertthumbnails", None): + postprocessors.append({ + "key": "FFmpegThumbnailsConvertor", + "format": opts.convertthumbnails, + "when": "before_dl", + }) + if getattr(opts, "exec_before_dl_cmd", None): + postprocessors.append({ + "key": "Exec", + "exec_cmd": opts.exec_before_dl_cmd, + "when": "before_dl", + }) + if opts.extractaudio: + postprocessors.append({ + "key": "FFmpegExtractAudio", + "preferredcodec": opts.audioformat, + "preferredquality": opts.audioquality, + "nopostoverwrites": opts.nopostoverwrites, + }) + if getattr(opts, "remuxvideo", None): + postprocessors.append({ + "key": "FFmpegVideoRemuxer", + "preferedformat": opts.remuxvideo, + }) + if opts.recodevideo: + postprocessors.append({ + "key": "FFmpegVideoConvertor", + "preferedformat": opts.recodevideo, + }) + if opts.embedsubtitles: + pp = {"key": "FFmpegEmbedSubtitle"} + if ytdlp: + pp["already_have_subtitle"] = ( + opts.writesubtitles and "no-keep-subs" not in compat_opts) + postprocessors.append(pp) + if not opts.writeautomaticsub and "no-keep-subs" not in compat_opts: + opts.writesubtitles = True + if opts.allsubtitles and not opts.writeautomaticsub: + opts.writesubtitles = True + remove_chapters_patterns, remove_ranges = [], [] + for regex in opts.remove_chapters: + if regex.startswith("*"): + dur = list(map(module.parse_duration, regex[1:].split("-"))) + if len(dur) == 2 and all(t is not None for t in dur): + remove_ranges.append(tuple(dur)) + continue + remove_chapters_patterns.append(re.compile(regex)) + if opts.remove_chapters or sponsorblock_query: + postprocessors.append({ + "key": "ModifyChapters", + "remove_chapters_patterns": remove_chapters_patterns, + "remove_sponsor_segments": opts.sponsorblock_remove, + "remove_ranges": remove_ranges, + "sponsorblock_chapter_title": opts.sponsorblock_chapter_title, + "force_keyframes": opts.force_keyframes_at_cuts, + }) + if opts.addmetadata or addchapters: + pp = {"key": "FFmpegMetadata"} + if ytdlp: + pp["add_chapters"] = addchapters + pp["add_metadata"] = opts.addmetadata + postprocessors.append(pp) + if getattr(opts, "sponskrub", False) is not False: + postprocessors.append({ + "key": "SponSkrub", + "path": opts.sponskrub_path, + "args": opts.sponskrub_args, + "cut": opts.sponskrub_cut, + "force": opts.sponskrub_force, + "ignoreerror": opts.sponskrub is None, + }) + if opts.embedthumbnail: + already_have_thumbnail = (opts.writethumbnail or + opts.write_all_thumbnails) + postprocessors.append({ + "key": "EmbedThumbnail", + "already_have_thumbnail": already_have_thumbnail, + }) + if not already_have_thumbnail: + opts.writethumbnail = True + opts.outtmpl["pl_thumbnail"] = "" + if getattr(opts, "split_chapters", None): + postprocessors.append({ + "key": "FFmpegSplitChapters", + "force_keyframes": opts.force_keyframes_at_cuts, + }) + if opts.xattrs: + postprocessors.append({"key": "XAttrMetadata"}) + if opts.exec_cmd: + postprocessors.append({ + "key": "Exec", + "exec_cmd": opts.exec_cmd, + "when": "after_move", + }) + + match_filter = ( + None if opts.match_filter is None + else module.match_filter_func(opts.match_filter)) + + return { + "usenetrc": opts.usenetrc, + "netrc_location": getattr(opts, "netrc_location", None), + "username": opts.username, + "password": opts.password, + "twofactor": opts.twofactor, + "videopassword": opts.videopassword, + "ap_mso": opts.ap_mso, + "ap_username": opts.ap_username, + "ap_password": opts.ap_password, + "quiet": opts.quiet, + "no_warnings": opts.no_warnings, + "forceurl": opts.geturl, + "forcetitle": opts.gettitle, + "forceid": opts.getid, + "forcethumbnail": opts.getthumbnail, + "forcedescription": opts.getdescription, + "forceduration": opts.getduration, + "forcefilename": opts.getfilename, + "forceformat": opts.getformat, + "forceprint": getattr(opts, "forceprint", None) or (), + "forcejson": opts.dumpjson or opts.print_json, + "dump_single_json": opts.dump_single_json, + "force_write_download_archive": getattr( + opts, "force_write_download_archive", None), + "simulate": opts.simulate, + "skip_download": opts.skip_download, + "format": opts.format, + "allow_unplayable_formats": getattr( + opts, "allow_unplayable_formats", None), + "ignore_no_formats_error": getattr( + opts, "ignore_no_formats_error", None), + "format_sort": getattr( + opts, "format_sort", None), + "format_sort_force": getattr( + opts, "format_sort_force", None), + "allow_multiple_video_streams": opts.allow_multiple_video_streams, + "allow_multiple_audio_streams": opts.allow_multiple_audio_streams, + "check_formats": getattr( + opts, "check_formats", None), + "listformats": opts.listformats, + "listformats_table": getattr( + opts, "listformats_table", None), + "outtmpl": opts.outtmpl, + "outtmpl_na_placeholder": opts.outtmpl_na_placeholder, + "paths": getattr(opts, "paths", None), + "autonumber_size": opts.autonumber_size, + "autonumber_start": opts.autonumber_start, + "restrictfilenames": opts.restrictfilenames, + "windowsfilenames": getattr(opts, "windowsfilenames", None), + "ignoreerrors": opts.ignoreerrors, + "force_generic_extractor": opts.force_generic_extractor, + "ratelimit": opts.ratelimit, + "throttledratelimit": getattr(opts, "throttledratelimit", None), + "overwrites": getattr(opts, "overwrites", None), + "retries": opts.retries, + "fragment_retries": opts.fragment_retries, + "extractor_retries": getattr(opts, "extractor_retries", None), + "skip_unavailable_fragments": opts.skip_unavailable_fragments, + "keep_fragments": opts.keep_fragments, + "concurrent_fragment_downloads": getattr( + opts, "concurrent_fragment_downloads", None), + "buffersize": opts.buffersize, + "noresizebuffer": opts.noresizebuffer, + "http_chunk_size": opts.http_chunk_size, + "continuedl": opts.continue_dl, + "noprogress": True if opts.noprogress is None else opts.noprogress, + "playliststart": opts.playliststart, + "playlistend": opts.playlistend, + "playlistreverse": opts.playlist_reverse, + "playlistrandom": opts.playlist_random, + "noplaylist": opts.noplaylist, + "logtostderr": outtmpl_default == "-", + "consoletitle": opts.consoletitle, + "nopart": opts.nopart, + "updatetime": opts.updatetime, + "writedescription": opts.writedescription, + "writeannotations": opts.writeannotations, + "writeinfojson": opts.writeinfojson, + "allow_playlist_files": opts.allow_playlist_files, + "clean_infojson": opts.clean_infojson, + "getcomments": getattr(opts, "getcomments", None), + "writethumbnail": opts.writethumbnail, + "write_all_thumbnails": opts.write_all_thumbnails, + "writelink": getattr(opts, "writelink", None), + "writeurllink": getattr(opts, "writeurllink", None), + "writewebloclink": getattr(opts, "writewebloclink", None), + "writedesktoplink": getattr(opts, "writedesktoplink", None), + "writesubtitles": opts.writesubtitles, + "writeautomaticsub": opts.writeautomaticsub, + "allsubtitles": opts.allsubtitles, + "listsubtitles": opts.listsubtitles, + "subtitlesformat": opts.subtitlesformat, + "subtitleslangs": opts.subtitleslangs, + "matchtitle": module.decodeOption(opts.matchtitle), + "rejecttitle": module.decodeOption(opts.rejecttitle), + "max_downloads": opts.max_downloads, + "prefer_free_formats": opts.prefer_free_formats, + "trim_file_name": getattr(opts, "trim_file_name", None), + "verbose": opts.verbose, + "dump_intermediate_pages": opts.dump_intermediate_pages, + "write_pages": opts.write_pages, + "test": opts.test, + "keepvideo": opts.keepvideo, + "min_filesize": opts.min_filesize, + "max_filesize": opts.max_filesize, + "min_views": opts.min_views, + "max_views": opts.max_views, + "daterange": date, + "cachedir": opts.cachedir, + "youtube_print_sig_code": opts.youtube_print_sig_code, + "age_limit": opts.age_limit, + "download_archive": download_archive_fn, + "break_on_existing": getattr(opts, "break_on_existing", None), + "break_on_reject": getattr(opts, "break_on_reject", None), + "skip_playlist_after_errors": getattr( + opts, "skip_playlist_after_errors", None), + "cookiefile": opts.cookiefile, + "cookiesfrombrowser": getattr(opts, "cookiesfrombrowser", None), + "nocheckcertificate": opts.no_check_certificate, + "prefer_insecure": opts.prefer_insecure, + "proxy": opts.proxy, + "socket_timeout": opts.socket_timeout, + "bidi_workaround": opts.bidi_workaround, + "debug_printtraffic": opts.debug_printtraffic, + "prefer_ffmpeg": opts.prefer_ffmpeg, + "include_ads": opts.include_ads, + "default_search": opts.default_search, + "dynamic_mpd": getattr(opts, "dynamic_mpd", None), + "extractor_args": getattr(opts, "extractor_args", None), + "youtube_include_dash_manifest": getattr( + opts, "youtube_include_dash_manifest", None), + "youtube_include_hls_manifest": getattr( + opts, "youtube_include_hls_manifest", None), + "encoding": opts.encoding, + "extract_flat": opts.extract_flat, + "mark_watched": opts.mark_watched, + "merge_output_format": opts.merge_output_format, + "postprocessors": postprocessors, + "fixup": opts.fixup, + "source_address": opts.source_address, + "call_home": opts.call_home, + "sleep_interval_requests": getattr( + opts, "sleep_interval_requests", None), + "sleep_interval": opts.sleep_interval, + "max_sleep_interval": opts.max_sleep_interval, + "sleep_interval_subtitles": getattr( + opts, "sleep_interval_subtitles", None), + "external_downloader": opts.external_downloader, + "list_thumbnails": opts.list_thumbnails, + "playlist_items": opts.playlist_items, + "xattr_set_filesize": opts.xattr_set_filesize, + "match_filter": match_filter, + "no_color": opts.no_color, + "ffmpeg_location": opts.ffmpeg_location, + "hls_prefer_native": opts.hls_prefer_native, + "hls_use_mpegts": opts.hls_use_mpegts, + "hls_split_discontinuity": getattr( + opts, "hls_split_discontinuity", None), + "external_downloader_args": opts.external_downloader_args, + "postprocessor_args": opts.postprocessor_args, + "cn_verification_proxy": opts.cn_verification_proxy, + "geo_verification_proxy": opts.geo_verification_proxy, + "geo_bypass": opts.geo_bypass, + "geo_bypass_country": opts.geo_bypass_country, + "geo_bypass_ip_block": opts.geo_bypass_ip_block, + "compat_opts": compat_opts, + } + + +def parse_retries(retries, name=""): + if retries in ("inf", "infinite"): + return float("inf") + return int(retries)