add '-s/--simulate' to run data extraction without download
Useful for quick testing (even though -g and -j kind of do the same) and to fill a download archive without actually downloading the files. -s does the same as the default behaviour, except downloading stuff. Maybe it should get a more fitting name, as it does actually write to disk (cache, archive)?
This commit is contained in:
@@ -275,6 +275,8 @@ def main():
|
|||||||
jobtype = job.KeywordJob
|
jobtype = job.KeywordJob
|
||||||
elif args.list_data:
|
elif args.list_data:
|
||||||
jobtype = job.DataJob
|
jobtype = job.DataJob
|
||||||
|
elif args.simulate:
|
||||||
|
jobtype = job.SimulationJob
|
||||||
else:
|
else:
|
||||||
jobtype = job.DownloadJob
|
jobtype = job.DownloadJob
|
||||||
|
|
||||||
|
|||||||
@@ -209,7 +209,7 @@ class DownloadJob(Job):
|
|||||||
|
|
||||||
def handle_queue(self, url, keywords):
|
def handle_queue(self, url, keywords):
|
||||||
try:
|
try:
|
||||||
DownloadJob(url, self).run()
|
self.__class__(url, self).run()
|
||||||
except exception.NoExtractorError:
|
except exception.NoExtractorError:
|
||||||
self._write_unsupported(url)
|
self._write_unsupported(url)
|
||||||
|
|
||||||
@@ -227,6 +227,18 @@ class DownloadJob(Job):
|
|||||||
return instance
|
return instance
|
||||||
|
|
||||||
|
|
||||||
|
class SimulationJob(DownloadJob):
|
||||||
|
"""Simulate the extraction process without downloading anything"""
|
||||||
|
|
||||||
|
def handle_url(self, url, keywords, fallback=None):
|
||||||
|
self.pathfmt.set_keywords(keywords)
|
||||||
|
self.out.skip(self.pathfmt.path)
|
||||||
|
if self.sleep:
|
||||||
|
time.sleep(self.sleep)
|
||||||
|
if self.archive:
|
||||||
|
self.archive.add(keywords)
|
||||||
|
|
||||||
|
|
||||||
class KeywordJob(Job):
|
class KeywordJob(Job):
|
||||||
"""Print available keywords"""
|
"""Print available keywords"""
|
||||||
|
|
||||||
|
|||||||
@@ -113,6 +113,10 @@ def build_parser():
|
|||||||
"-j", "--dump-json", dest="list_data", action="store_true",
|
"-j", "--dump-json", dest="list_data", action="store_true",
|
||||||
help="Print JSON information",
|
help="Print JSON information",
|
||||||
)
|
)
|
||||||
|
output.add_argument(
|
||||||
|
"-s", "--simulate", dest="simulate", action="store_true",
|
||||||
|
help="Simulate data extraction; do not download anything",
|
||||||
|
)
|
||||||
output.add_argument(
|
output.add_argument(
|
||||||
"-K", "--list-keywords", dest="list_keywords", action="store_true",
|
"-K", "--list-keywords", dest="list_keywords", action="store_true",
|
||||||
help=("Print a list of available keywords and example values "
|
help=("Print a list of available keywords and example values "
|
||||||
|
|||||||
Reference in New Issue
Block a user