github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[job] use identity checks

Browse Source 
micro optimizations...
This commit is contained in:
Mike Fährmann
2025-10-14 12:04:24 +02:00
parent 899dcc62cf
commit cda63b94bb
1 changed files with 28 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
gallery_dl/job.py
View File

@@ -145,7 +145,7 @@ class Job():
# sleep before extractor start # sleep before extractor start
sleep = util.build_duration_func( sleep = util.build_duration_func(
extractor.config("sleep-extractor")) extractor.config("sleep-extractor"))
if sleep: if sleep is not None:
extractor.sleep(sleep(), "extractor") extractor.sleep(sleep(), "extractor")
try: try:
@@ -167,8 +167,7 @@ class Job():
self.status |= exc.code self.status |= exc.code
except OSError as exc: except OSError as exc:
log.debug("", exc_info=exc) log.debug("", exc_info=exc)
name = exc.__class__.__name__ if (name := exc.__class__.__name__) == "JSONDecodeError":
if name == "JSONDecodeError":
log.error("Failed to parse JSON data: %s: %s", name, exc) log.error("Failed to parse JSON data: %s: %s", name, exc)
self.status |= 1 self.status |= 1
else: # regular OSError else: # regular OSError
@@ -257,10 +256,10 @@ class Job():
def _prepare_predicates(self, target, skip=True): def _prepare_predicates(self, target, skip=True):
predicates = [] predicates = []
if self.extractor.config(target + "-unique"): if self.extractor.config(f"{target}-unique"):
predicates.append(util.UniquePredicate()) predicates.append(util.UniquePredicate())
if pfilter := self.extractor.config(target + "-filter"): if pfilter := self.extractor.config(f"{target}-filter"):
try: try:
pred = util.FilterPredicate(pfilter, target) pred = util.FilterPredicate(pfilter, target)
except (SyntaxError, ValueError, TypeError) as exc: except (SyntaxError, ValueError, TypeError) as exc:
@@ -268,7 +267,7 @@ class Job():
else: else:
predicates.append(pred) predicates.append(pred)
if prange := self.extractor.config(target + "-range"): if prange := self.extractor.config(f"{target}-range"):
try: try:
pred = util.RangePredicate(prange) pred = util.RangePredicate(prange)
except ValueError as exc: except ValueError as exc:
@@ -288,7 +287,7 @@ class Job():
return self._logger_adapter(logger, self) return self._logger_adapter(logger, self)
def _write_unsupported(self, url): def _write_unsupported(self, url):
if self.ulog: if self.ulog is not None:
self.ulog.info(url) self.ulog.info(url)
@@ -321,7 +320,7 @@ class DownloadJob(Job):
for callback in hooks["prepare"]: for callback in hooks["prepare"]:
callback(pathfmt) callback(pathfmt)
if archive and archive.check(kwdict): if archive is not None and archive.check(kwdict):
pathfmt.fix_extension() pathfmt.fix_extension()
self.handle_skip() self.handle_skip()
return return
@@ -330,7 +329,7 @@ class DownloadJob(Job):
pathfmt.build_path() pathfmt.build_path()
if pathfmt.exists(): if pathfmt.exists():
if archive and self._archive_write_skip: if archive is not None and self._archive_write_skip:
archive.add(kwdict) archive.add(kwdict)
self.handle_skip() self.handle_skip()
return return
@@ -340,12 +339,12 @@ class DownloadJob(Job):
callback(pathfmt) callback(pathfmt)
if kwdict.pop("_file_recheck", False) and pathfmt.exists(): if kwdict.pop("_file_recheck", False) and pathfmt.exists():
if archive and self._archive_write_skip: if archive is not None and self._archive_write_skip:
archive.add(kwdict) archive.add(kwdict)
self.handle_skip() self.handle_skip()
return return
if self.sleep: if self.sleep is not None:
self.extractor.sleep(self.sleep(), "download") self.extractor.sleep(self.sleep(), "download")
# download from URL # download from URL
@@ -369,7 +368,7 @@ class DownloadJob(Job):
return return
if not pathfmt.temppath: if not pathfmt.temppath:
if archive and self._archive_write_skip: if archive is not None and self._archive_write_skip:
archive.add(kwdict) archive.add(kwdict)
self.handle_skip() self.handle_skip()
return return
@@ -383,17 +382,17 @@ class DownloadJob(Job):
pathfmt.finalize() pathfmt.finalize()
self.out.success(pathfmt.path) self.out.success(pathfmt.path)
self._skipcnt = 0 self._skipcnt = 0
if archive and self._archive_write_file: if archive is not None and self._archive_write_file:
archive.add(kwdict) archive.add(kwdict)
if "after" in hooks: if "after" in hooks:
for callback in hooks["after"]: for callback in hooks["after"]:
callback(pathfmt) callback(pathfmt)
if archive and self._archive_write_after: if archive is not None and self._archive_write_after:
archive.add(kwdict) archive.add(kwdict)
def handle_directory(self, kwdict): def handle_directory(self, kwdict):
"""Set and create the target directory for downloads""" """Set and create the target directory for downloads"""
if not self.pathfmt: if self.pathfmt is None:
self.initialize(kwdict) self.initialize(kwdict)
else: else:
if "post-after" in self.hooks: if "post-after" in self.hooks:
@@ -511,7 +510,7 @@ class DownloadJob(Job):
self.out.skip(pathfmt.path) self.out.skip(pathfmt.path)
if self._skipexc: if self._skipexc:
if not self._skipftr or self._skipftr(pathfmt.kwdict): if self._skipftr is None or self._skipftr(pathfmt.kwdict):
self._skipcnt += 1 self._skipcnt += 1
if self._skipcnt >= self._skipmax: if self._skipcnt >= self._skipmax:
raise self._skipexc raise self._skipexc
@@ -555,7 +554,7 @@ class DownloadJob(Job):
cfg = extr.config cfg = extr.config
pathfmt = self.pathfmt = path.PathFormat(extr) pathfmt = self.pathfmt = path.PathFormat(extr)
if kwdict: if kwdict is not None:
pathfmt.set_directory(kwdict) pathfmt.set_directory(kwdict)
self.sleep = util.build_duration_func(cfg("sleep")) self.sleep = util.build_duration_func(cfg("sleep"))
@@ -625,7 +624,7 @@ class DownloadJob(Job):
else: else:
# monkey-patch methods to always return False # monkey-patch methods to always return False
pathfmt.exists = lambda x=None: False pathfmt.exists = lambda x=None: False
if self.archive: if self.archive is not None:
self.archive.check = pathfmt.exists self.archive.check = pathfmt.exists
if not cfg("postprocess", True): if not cfg("postprocess", True):
@@ -685,7 +684,7 @@ class DownloadJob(Job):
pp_dict["__init__"] = None pp_dict["__init__"] = None
pp_cls = postprocessor.find(name) pp_cls = postprocessor.find(name)
if not pp_cls: if pp_cls is None:
pp_log.warning("module '%s' not found", name) pp_log.warning("module '%s' not found", name)
continue continue
try: try:
@@ -710,15 +709,11 @@ class DownloadJob(Job):
condition = util.compile_filter(expr) condition = util.compile_filter(expr)
for hook, callback in hooks.items(): for hook, callback in hooks.items():
self.hooks[hook].append(functools.partial( self.hooks[hook].append(functools.partial(
self._call_hook, callback, condition)) _call_hook_condition, callback, condition))
else: else:
for hook, callback in hooks.items(): for hook, callback in hooks.items():
self.hooks[hook].append(callback) self.hooks[hook].append(callback)
def _call_hook(self, callback, condition, pathfmt):
if condition(pathfmt.kwdict):
callback(pathfmt)
def _build_extractor_filter(self): def _build_extractor_filter(self):
clist = self.extractor.config("whitelist") clist = self.extractor.config("whitelist")
if clist is not None: if clist is not None:
@@ -734,20 +729,25 @@ class DownloadJob(Job):
return util.build_extractor_filter(clist, negate, special) return util.build_extractor_filter(clist, negate, special)
def _call_hook_condition(callback, condition, pathfmt):
if condition(pathfmt.kwdict):
callback(pathfmt)
class SimulationJob(DownloadJob): class SimulationJob(DownloadJob):
"""Simulate the extraction process without downloading anything""" """Simulate the extraction process without downloading anything"""
def handle_url(self, url, kwdict): def handle_url(self, url, kwdict):
ext = kwdict["extension"] or "jpg" ext = kwdict["extension"] or "jpg"
kwdict["extension"] = self.pathfmt.extension_map(ext, ext) kwdict["extension"] = self.pathfmt.extension_map(ext, ext)
if self.sleep: if self.sleep is not None:
self.extractor.sleep(self.sleep(), "download") self.extractor.sleep(self.sleep(), "download")
if self.archive and self._archive_write_skip: if self.archive is not None and self._archive_write_skip:
self.archive.add(kwdict) self.archive.add(kwdict)
self.out.skip(self.pathfmt.build_filename(kwdict)) self.out.skip(self.pathfmt.build_filename(kwdict))
def handle_directory(self, kwdict): def handle_directory(self, kwdict):
if not self.pathfmt: if self.pathfmt is None:
self.initialize() self.initialize()
@@ -935,7 +935,7 @@ class DataJob(Job):
extractor = self.extractor extractor = self.extractor
sleep = util.build_duration_func( sleep = util.build_duration_func(
extractor.config("sleep-extractor")) extractor.config("sleep-extractor"))
if sleep: if sleep is not None:
extractor.sleep(sleep(), "extractor") extractor.sleep(sleep(), "extractor")
# collect data # collect data