implement a 'fallback' option (closes #1770)

This commit is contained in:
Mike Fährmann
2021-08-16 01:47:59 +02:00
parent c866fcba48
commit d320ee6251
3 changed files with 26 additions and 12 deletions

View File

@@ -194,16 +194,6 @@ Description
Share number of skipped downloads between parent and child extractors.
extractor.*.url-metadata
------------------------
Type
``string``
Default
``null``
Description
Insert a file's download URL into its metadata dictionary as the given name.
extractor.*.path-restrict
-------------------------
Type
@@ -507,6 +497,16 @@ Description
`format strings`_.
extractor.*.url-metadata
------------------------
Type
``string``
Default
``null``
Description
Insert a file's download URL into its metadata dictionary as the given name.
extractor.*.category-transfer
-----------------------------
Type
@@ -671,6 +671,16 @@ Description
will be executed as normal.
extractor.*.fallback
--------------------
Type
``bool``
Default
``true``
Description
Use fallback download URLs when a download fails.
extractor.*.image-range
-----------------------
Type

View File

@@ -14,6 +14,7 @@
"retries": 4,
"timeout": 30.0,
"verify": true,
"fallback": true,
"sleep": 0,
"sleep-request": 0,

View File

@@ -199,6 +199,7 @@ class DownloadJob(Job):
Job.__init__(self, url, parent)
self.log = self.get_logger("download")
self.blacklist = None
self.fallback = None
self.archive = None
self.sleep = None
self.hooks = ()
@@ -237,8 +238,9 @@ class DownloadJob(Job):
# download from URL
if not self.download(url):
# use fallback URLs if available
for num, url in enumerate(kwdict.get("_fallback", ()), 1):
# use fallback URLs if available/enabled
fallback = kwdict.get("_fallback", ()) if self.fallback else ()
for num, url in enumerate(fallback, 1):
util.remove_file(pathfmt.temppath)
self.log.info("Trying fallback URL #%d", num)
if self.download(url):
@@ -394,6 +396,7 @@ class DownloadJob(Job):
pathfmt.set_directory(kwdict)
self.sleep = cfg("sleep")
self.fallback = cfg("fallback", True)
if not cfg("download", True):
# monkey-patch method to do nothing and always return True
self.download = pathfmt.fix_extension