1087 lines
37 KiB
Python
1087 lines
37 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# Copyright 2015-2026 Mike Fährmann
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
|
|
import sys
|
|
import errno
|
|
import logging
|
|
import functools
|
|
import collections
|
|
|
|
from . import (
|
|
extractor,
|
|
downloader,
|
|
postprocessor,
|
|
archive,
|
|
config,
|
|
exception,
|
|
formatter,
|
|
output,
|
|
path,
|
|
text,
|
|
util,
|
|
version,
|
|
)
|
|
from .extractor.message import Message
|
|
stdout_write = output.stdout_write
|
|
FLAGS = util.FLAGS
|
|
|
|
|
|
class Job():
|
|
"""Base class for Job types"""
|
|
ulog = None
|
|
_logger_adapter = output.LoggerAdapter
|
|
|
|
def __init__(self, extr, parent=None):
|
|
if isinstance(extr, str):
|
|
extr = extractor.find(extr)
|
|
if not extr:
|
|
raise exception.NoExtractorError()
|
|
|
|
self.extractor = extr
|
|
self.pathfmt = None
|
|
self.status = 0
|
|
self.kwdict = {}
|
|
self.kwdict_eval = False
|
|
|
|
if cfgpath := self._build_config_path(parent):
|
|
if isinstance(cfgpath, list):
|
|
extr.config = extr._config_shared
|
|
extr.config_accumulate = extr._config_shared_accumulate
|
|
extr._cfgpath = cfgpath
|
|
|
|
if actions := extr.config("actions"):
|
|
from .actions import LoggerAdapter, parse_logging
|
|
self._logger_adapter = LoggerAdapter
|
|
self._logger_actions = parse_logging(actions)
|
|
|
|
path_proxy = output.PathfmtProxy(self)
|
|
self._logger_extra = {
|
|
"job" : self,
|
|
"extractor": extr,
|
|
"path" : path_proxy,
|
|
"keywords" : output.KwdictProxy(self),
|
|
}
|
|
extr.log = self._wrap_logger(extr.log)
|
|
extr.log.debug("Using %s for '%s'", extr.__class__.__name__, extr.url)
|
|
|
|
self.metadata_url = extr.config2(
|
|
"metadata-url", "url-metadata", "_url")
|
|
self.metadata_http = extr.config2(
|
|
"metadata-http", "http-metadata")
|
|
metadata_path = extr.config2(
|
|
"metadata-path", "path-metadata", "_path")
|
|
metadata_version = extr.config2(
|
|
"metadata-version", "version-metadata")
|
|
metadata_extractor = extr.config2(
|
|
"metadata-extractor", "extractor-metadata", "_extr")
|
|
|
|
if metadata_path:
|
|
self.kwdict[metadata_path] = path_proxy
|
|
if metadata_extractor:
|
|
self.kwdict[metadata_extractor] = extr
|
|
if metadata_version:
|
|
self.kwdict[metadata_version] = {
|
|
"version" : version.__version__,
|
|
"is_executable" : util.EXECUTABLE,
|
|
"current_git_head": util.git_head()
|
|
}
|
|
# user-supplied metadata
|
|
kwdict = extr.config("keywords")
|
|
if kwdict_global := extr.config("keywords-global"):
|
|
kwdict = {**kwdict_global, **kwdict} if kwdict else kwdict_global
|
|
elif not kwdict:
|
|
return
|
|
|
|
if extr.config("keywords-eval"):
|
|
self.kwdict_eval = []
|
|
for key, value in kwdict.items():
|
|
if isinstance(value, str):
|
|
fmt = formatter.parse(value, None, util.identity)
|
|
self.kwdict_eval.append((key, fmt.format_map))
|
|
else:
|
|
self.kwdict[key] = value
|
|
else:
|
|
self.kwdict.update(kwdict)
|
|
|
|
def _build_config_path(self, parent):
|
|
extr = self.extractor
|
|
cfgpath = []
|
|
|
|
if parent is None:
|
|
self.parents = ()
|
|
else:
|
|
pextr = parent.extractor
|
|
if extr.category == pextr.category or \
|
|
extr.category in parent.parents:
|
|
parents = parent.parents
|
|
else:
|
|
parents = parent.parents + (pextr.category,)
|
|
self.parents = parents
|
|
|
|
if pextr.config("category-transfer", pextr.categorytransfer):
|
|
extr.category = pextr.category
|
|
extr.subcategory = pextr.subcategory
|
|
return pextr._cfgpath
|
|
|
|
if parents:
|
|
sub = extr.subcategory
|
|
for category in parents:
|
|
cat = f"{category}>{extr.category}"
|
|
cfgpath.append((cat, sub))
|
|
cfgpath.append((category + ">*", sub))
|
|
cfgpath.append((extr.category, sub))
|
|
|
|
if extr.basecategory:
|
|
if not cfgpath:
|
|
cfgpath.append((extr.category, extr.subcategory))
|
|
if extr.basesubcategory:
|
|
cfgpath.append((extr.basesubcategory, extr.subcategory))
|
|
cfgpath.append((extr.basecategory, extr.subcategory))
|
|
|
|
return cfgpath
|
|
|
|
def run(self):
|
|
"""Execute or run the job"""
|
|
extractor = self.extractor
|
|
log = extractor.log
|
|
|
|
self._init()
|
|
|
|
# sleep before extractor start
|
|
sleep = util.build_duration_func(
|
|
extractor.config("sleep-extractor"))
|
|
if sleep is not None:
|
|
extractor.sleep(sleep(), "extractor")
|
|
|
|
try:
|
|
msg = self.dispatch(extractor)
|
|
except exception.StopExtraction as exc:
|
|
if exc.depth > 1 and exc.target != extractor.__class__.subcategory:
|
|
exc.depth -= 1
|
|
raise
|
|
pass
|
|
except exception.AbortExtraction as exc:
|
|
log.traceback(exc)
|
|
log.error(exc.message)
|
|
self.status |= exc.code
|
|
except (exception.TerminateExtraction, exception.RestartExtraction):
|
|
raise
|
|
except exception.GalleryDLException as exc:
|
|
log.error("%s: %s", exc.__class__.__name__, exc)
|
|
log.traceback(exc)
|
|
self.status |= exc.code
|
|
except OSError as exc:
|
|
log.traceback(exc)
|
|
if (name := exc.__class__.__name__) == "JSONDecodeError":
|
|
log.error("Failed to parse JSON data: %s: %s", name, exc)
|
|
self.status |= 1
|
|
else: # regular OSError
|
|
log.error("Unable to download data: %s: %s", name, exc)
|
|
self.status |= 128
|
|
except Exception as exc:
|
|
log.error(("An unexpected error occurred: %s - %s. "
|
|
"Please run gallery-dl again with the --verbose flag, "
|
|
"copy its output and report this issue on "
|
|
"https://github.com/mikf/gallery-dl/issues ."),
|
|
exc.__class__.__name__, exc)
|
|
log.traceback(exc)
|
|
self.status |= 1
|
|
except BaseException:
|
|
self.status |= 1
|
|
raise
|
|
else:
|
|
if msg is None:
|
|
log.info("No results for %s", extractor.url)
|
|
finally:
|
|
if extractor.status:
|
|
self.status |= extractor.status
|
|
self.handle_finalize()
|
|
if extractor.finalize is not None:
|
|
extractor.finalize(self.status)
|
|
|
|
return self.status
|
|
|
|
def dispatch(self, messages):
|
|
"""Call the appropriate message handler"""
|
|
msg = None
|
|
process = True
|
|
|
|
for msg, url, kwdict in messages:
|
|
|
|
if msg == Message.Directory:
|
|
self.update_kwdict(kwdict)
|
|
if self.pred_post(url, kwdict):
|
|
process = True
|
|
self.handle_directory(kwdict)
|
|
else:
|
|
process = None
|
|
if FLAGS.POST is not None:
|
|
FLAGS.process("POST")
|
|
|
|
elif process is None:
|
|
continue
|
|
elif FLAGS.POST is False:
|
|
FLAGS.POST = process = None
|
|
continue
|
|
|
|
elif msg == Message.Url:
|
|
if self.metadata_url:
|
|
kwdict[self.metadata_url] = url
|
|
self.update_kwdict(kwdict)
|
|
if self.pred_url(url, kwdict):
|
|
if FLAGS.FILE is False:
|
|
FLAGS.FILE = None
|
|
continue
|
|
self.handle_url(url, kwdict)
|
|
if FLAGS.FILE is not None:
|
|
FLAGS.process("FILE")
|
|
|
|
elif msg == Message.Queue:
|
|
self.update_kwdict(kwdict)
|
|
if self.metadata_url:
|
|
kwdict[self.metadata_url] = url
|
|
if self.pred_queue(url, kwdict):
|
|
if FLAGS.CHILD is False:
|
|
FLAGS.CHILD = None
|
|
continue
|
|
self.handle_queue(url, kwdict)
|
|
if FLAGS.CHILD is not None:
|
|
FLAGS.process("CHILD")
|
|
|
|
return msg
|
|
|
|
def handle_url(self, url, kwdict):
|
|
"""Handle Message.Url"""
|
|
|
|
def handle_directory(self, kwdict):
|
|
"""Handle Message.Directory"""
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
"""Handle Message.Queue"""
|
|
|
|
def handle_finalize(self):
|
|
"""Handle job finalization"""
|
|
|
|
def update_kwdict(self, kwdict):
|
|
"""Update 'kwdict' with additional metadata"""
|
|
extr = self.extractor
|
|
kwdict["category"] = extr.category
|
|
kwdict["subcategory"] = extr.subcategory
|
|
if self.metadata_http:
|
|
kwdict.pop(self.metadata_http, None)
|
|
if extr.kwdict:
|
|
kwdict.update(extr.kwdict)
|
|
if self.kwdict:
|
|
kwdict.update(self.kwdict)
|
|
if self.kwdict_eval:
|
|
for key, valuegen in self.kwdict_eval:
|
|
kwdict[key] = valuegen(kwdict)
|
|
|
|
def initialize(self):
|
|
pass
|
|
|
|
def _init(self):
|
|
extr = self.extractor
|
|
|
|
extr.initialize()
|
|
self.pred_url = self._prepare_predicates(
|
|
"file", "image", extr.skip)
|
|
self.pred_post = self._prepare_predicates(
|
|
"post", None, None)
|
|
self.pred_queue = self._prepare_predicates(
|
|
"child", "chapter", None)
|
|
|
|
init = extr.config("init", False)
|
|
if init and init != "lazy":
|
|
self.initialize()
|
|
|
|
def _prepare_predicates(self, target, alt=None, skip=None):
|
|
predicates = []
|
|
extr = self.extractor
|
|
|
|
if extr.config(target + "-unique") or \
|
|
alt is not None and extr.config(alt + "-unique"):
|
|
predicates.append(util.predicate_unique())
|
|
|
|
if (pfilter := extr.config(target + "-filter")) or \
|
|
alt is not None and (pfilter := extr.config(alt + "-filter")):
|
|
try:
|
|
predicates.append(util.predicate_filter(pfilter, target))
|
|
except (SyntaxError, ValueError, TypeError) as exc:
|
|
extr.log.warning(exc)
|
|
|
|
if (prange := extr.config(target + "-range")) or \
|
|
alt is not None and (prange := extr.config(alt + "-range")):
|
|
try:
|
|
if pfilter:
|
|
skip = None
|
|
flag = target if alt is not None else None
|
|
predicates.append(util.predicate_range(prange, skip, flag))
|
|
except ValueError as exc:
|
|
extr.log.warning("invalid %s range: %s", target, exc)
|
|
|
|
return util.predicate_build(predicates)
|
|
|
|
def get_logger(self, name):
|
|
return self._wrap_logger(logging.getLogger(name))
|
|
|
|
def _wrap_logger(self, logger):
|
|
return self._logger_adapter(logger, self)
|
|
|
|
def _write_unsupported(self, url):
|
|
if self.ulog is not None:
|
|
self.ulog.info(url)
|
|
|
|
|
|
class DownloadJob(Job):
|
|
"""Download images into appropriate directory/filename locations"""
|
|
|
|
def __init__(self, url, parent=None):
|
|
Job.__init__(self, url, parent)
|
|
self.log = self.get_logger("download")
|
|
self.fallback = None
|
|
self.archive = None
|
|
self.sleep = None
|
|
self.hooks = ()
|
|
self.downloaders = {}
|
|
self.out = output.select()
|
|
self.visited = set() if parent is None else parent.visited
|
|
self._extractor_filter = None
|
|
self._skipcnt = 0
|
|
|
|
def handle_url(self, url, kwdict):
|
|
"""Download the resource specified in 'url'"""
|
|
hooks = self.hooks
|
|
pathfmt = self.pathfmt
|
|
archive = self.archive
|
|
|
|
# prepare download
|
|
pathfmt.set_filename(kwdict)
|
|
|
|
if "prepare" in hooks:
|
|
for callback in hooks["prepare"]:
|
|
callback(pathfmt)
|
|
|
|
if archive is not None and archive.check(kwdict):
|
|
pathfmt.fix_extension()
|
|
self.handle_skip()
|
|
return
|
|
|
|
if pathfmt.extension and not self.metadata_http:
|
|
pathfmt.build_path()
|
|
|
|
if pathfmt.exists():
|
|
if archive is not None and self._archive_write_skip:
|
|
archive.add(kwdict)
|
|
self.handle_skip()
|
|
return
|
|
|
|
if "prepare-after" in hooks:
|
|
for callback in hooks["prepare-after"]:
|
|
callback(pathfmt)
|
|
|
|
if kwdict.pop("_file_recheck", False) and pathfmt.exists():
|
|
if archive is not None and self._archive_write_skip:
|
|
archive.add(kwdict)
|
|
self.handle_skip()
|
|
return
|
|
|
|
if self.sleep is not None:
|
|
self.extractor.sleep(self.sleep(), "download")
|
|
|
|
# download from URL
|
|
if not self.download(url):
|
|
|
|
# use fallback URLs if available/enabled
|
|
fallback = kwdict.get("_fallback", ()) if self.fallback else ()
|
|
for num, url in enumerate(fallback, 1):
|
|
util.remove_file(pathfmt.temppath)
|
|
self.log.info("Trying fallback URL #%d", num)
|
|
if self.download(url):
|
|
break
|
|
else:
|
|
# download failed
|
|
self.status |= 4
|
|
self.log.error("Failed to download %s",
|
|
pathfmt.filename or url)
|
|
if "error" in hooks:
|
|
for callback in hooks["error"]:
|
|
callback(pathfmt)
|
|
return
|
|
|
|
if not pathfmt.temppath:
|
|
if archive is not None and self._archive_write_skip:
|
|
archive.add(kwdict)
|
|
self.handle_skip()
|
|
return
|
|
|
|
# run post processors
|
|
if "file" in hooks:
|
|
for callback in hooks["file"]:
|
|
callback(pathfmt)
|
|
|
|
# download succeeded
|
|
pathfmt.finalize()
|
|
self.out.success(pathfmt.path)
|
|
self._skipcnt = 0
|
|
if archive is not None and self._archive_write_file:
|
|
archive.add(kwdict)
|
|
if "after" in hooks:
|
|
for callback in hooks["after"]:
|
|
callback(pathfmt)
|
|
if archive is not None and self._archive_write_after:
|
|
archive.add(kwdict)
|
|
|
|
def handle_directory(self, kwdict):
|
|
"""Set and create the target directory for downloads"""
|
|
if self.pathfmt is None:
|
|
self.initialize(kwdict)
|
|
else:
|
|
if "post-after" in self.hooks:
|
|
for callback in self.hooks["post-after"]:
|
|
callback(self.pathfmt)
|
|
if FLAGS.POST is not None:
|
|
FLAGS.process("POST")
|
|
self.pathfmt.set_directory(kwdict)
|
|
if "post" in self.hooks:
|
|
for callback in self.hooks["post"]:
|
|
callback(self.pathfmt)
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
if url in self.visited:
|
|
return
|
|
self.visited.add(url)
|
|
|
|
if "child" in self.hooks:
|
|
pathfmt = self.pathfmt
|
|
pathfmt.kwdict = kwdict
|
|
for callback in self.hooks["child"]:
|
|
callback(pathfmt)
|
|
|
|
if cls := kwdict.get("_extractor"):
|
|
extr = cls.from_url(url)
|
|
else:
|
|
if extr := extractor.find(url):
|
|
if self._extractor_filter is None:
|
|
self._extractor_filter = self._build_extractor_filter()
|
|
if not self._extractor_filter(extr):
|
|
extr = None
|
|
|
|
if extr:
|
|
job = self.__class__(extr, self)
|
|
pfmt = self.pathfmt
|
|
pextr = self.extractor
|
|
parent = pextr.config("parent", pextr.parent)
|
|
|
|
if pfmt and pextr.config("parent-directory", parent):
|
|
extr._parentdir = pfmt.directory
|
|
else:
|
|
extr._parentdir = pextr._parentdir
|
|
|
|
if pmeta := pextr.config2(
|
|
"parent-metadata", "metadata-parent", parent or "_parent"):
|
|
if isinstance(pmeta, str):
|
|
data = self.kwdict.copy()
|
|
if kwdict:
|
|
data.update(kwdict)
|
|
job.kwdict[pmeta] = data
|
|
else:
|
|
if self.kwdict:
|
|
job.kwdict.update(self.kwdict)
|
|
if kwdict:
|
|
job.kwdict.update(kwdict)
|
|
if "_extractor" in kwdict:
|
|
del job.kwdict["_extractor"]
|
|
|
|
if pextr.config("parent-session", parent):
|
|
extr.session = pextr.session
|
|
|
|
while True:
|
|
try:
|
|
if pextr.config("parent-skip", parent):
|
|
job._skipcnt = self._skipcnt
|
|
status = job.run()
|
|
self._skipcnt = job._skipcnt
|
|
else:
|
|
status = job.run()
|
|
|
|
if status:
|
|
self.status |= status
|
|
if (status & 95 and # not FormatError or OSError
|
|
"_fallback" in kwdict and self.fallback):
|
|
fallback = kwdict["_fallback"] = \
|
|
iter(kwdict["_fallback"])
|
|
try:
|
|
url = next(fallback)
|
|
except StopIteration:
|
|
pass
|
|
else:
|
|
pextr.log.info("Downloading fallback URL")
|
|
text.nameext_from_url(url, kwdict)
|
|
if kwdict["filename"].startswith((
|
|
"HLS", "DASH")):
|
|
kwdict["filename"] = url.rsplit("/", 2)[-2]
|
|
if url.startswith("ytdl:"):
|
|
kwdict["extension"] = "mp4"
|
|
self.handle_url(url, kwdict)
|
|
break
|
|
except exception.RestartExtraction:
|
|
pass
|
|
|
|
else:
|
|
self._write_unsupported(url)
|
|
|
|
if "child-after" in self.hooks:
|
|
pathfmt = self.pathfmt
|
|
pathfmt.kwdict = kwdict
|
|
for callback in self.hooks["child-after"]:
|
|
callback(pathfmt)
|
|
|
|
def handle_finalize(self):
|
|
if self.archive:
|
|
if not self.status:
|
|
self.archive.finalize()
|
|
self.archive.close()
|
|
|
|
if pathfmt := self.pathfmt:
|
|
hooks = self.hooks
|
|
if "post-after" in hooks:
|
|
for callback in hooks["post-after"]:
|
|
callback(pathfmt)
|
|
|
|
self.extractor.cookies_store()
|
|
|
|
if self.status:
|
|
if "finalize-error" in hooks:
|
|
for callback in hooks["finalize-error"]:
|
|
callback(pathfmt)
|
|
else:
|
|
if "finalize-success" in hooks:
|
|
for callback in hooks["finalize-success"]:
|
|
callback(pathfmt)
|
|
if "finalize" in hooks:
|
|
for callback in hooks["finalize"]:
|
|
callback(pathfmt)
|
|
|
|
def handle_skip(self):
|
|
pathfmt = self.pathfmt
|
|
if "skip" in self.hooks:
|
|
for callback in self.hooks["skip"]:
|
|
callback(pathfmt)
|
|
self.out.skip(pathfmt.path)
|
|
|
|
if self._skipexc is not None:
|
|
if self._skipftr is None or self._skipftr(pathfmt.kwdict):
|
|
self._skipcnt += 1
|
|
if self._skipcnt >= self._skipmax:
|
|
raise self._skipexc
|
|
|
|
if self.sleep_skip is not None:
|
|
self.extractor.sleep(self.sleep_skip(), "skip")
|
|
|
|
def download(self, url):
|
|
"""Download 'url'"""
|
|
if downloader := self.get_downloader(url[:url.find(":")]):
|
|
try:
|
|
return downloader.download(url, self.pathfmt)
|
|
except OSError as exc:
|
|
if exc.errno == errno.ENOSPC:
|
|
raise
|
|
self.log.warning("%s: %s", exc.__class__.__name__, exc)
|
|
return False
|
|
self._write_unsupported(url)
|
|
return False
|
|
|
|
def get_downloader(self, scheme):
|
|
"""Return a downloader suitable for 'scheme'"""
|
|
try:
|
|
return self.downloaders[scheme]
|
|
except KeyError:
|
|
pass
|
|
|
|
cls = downloader.find(scheme)
|
|
if cls and config.get(("downloader", cls.scheme), "enabled", True):
|
|
instance = cls(self)
|
|
else:
|
|
instance = None
|
|
self.log.error("'%s:' URLs are not supported/enabled", scheme)
|
|
|
|
if cls and cls.scheme == "http":
|
|
self.downloaders["http"] = self.downloaders["https"] = instance
|
|
else:
|
|
self.downloaders[scheme] = instance
|
|
return instance
|
|
|
|
def initialize(self, kwdict=None):
|
|
"""initialize PathFormat, postprocessors, archive, options, etc"""
|
|
extr = self.extractor
|
|
cfg = extr.config
|
|
|
|
pathfmt = self.pathfmt = path.PathFormat(extr)
|
|
if kwdict is not None:
|
|
pathfmt.set_directory(kwdict)
|
|
|
|
self.sleep = util.build_duration_func(cfg("sleep"))
|
|
self.sleep_skip = util.build_duration_func(cfg("sleep-skip"))
|
|
self.fallback = cfg("fallback", True)
|
|
if not cfg("download", True):
|
|
# monkey-patch method to do nothing and always return True
|
|
self.download = pathfmt.fix_extension
|
|
|
|
if archive_path := cfg("archive"):
|
|
archive_table = cfg("archive-table")
|
|
archive_prefix = cfg("archive-prefix")
|
|
if archive_prefix is None:
|
|
archive_prefix = extr.category if archive_table is None else ""
|
|
|
|
archive_format = cfg("archive-format")
|
|
if archive_format is None:
|
|
archive_format = extr.archive_fmt
|
|
|
|
try:
|
|
self.archive = archive.connect(
|
|
archive_path,
|
|
archive_prefix,
|
|
archive_format,
|
|
archive_table,
|
|
cfg("archive-mode"),
|
|
cfg("archive-pragma"),
|
|
kwdict,
|
|
)
|
|
except Exception as exc:
|
|
extr.log.warning(
|
|
"Failed to open download archive at '%s' (%s: %s)",
|
|
archive_path, exc.__class__.__name__, exc)
|
|
else:
|
|
extr.log.debug("Using download archive '%s'", archive_path)
|
|
|
|
events = cfg("archive-event")
|
|
if events is None:
|
|
self._archive_write_file = True
|
|
self._archive_write_skip = False
|
|
self._archive_write_after = False
|
|
else:
|
|
if isinstance(events, str):
|
|
events = events.split(",")
|
|
self._archive_write_file = ("file" in events)
|
|
self._archive_write_skip = ("skip" in events)
|
|
self._archive_write_after = ("after" in events)
|
|
|
|
if skip := cfg("skip", True):
|
|
self._skipexc = None
|
|
if skip == "enumerate":
|
|
pathfmt.check_file = pathfmt._enum_file
|
|
elif isinstance(skip, str):
|
|
skip, _, smax = skip.partition(":")
|
|
if skip == "abort":
|
|
smax, _, sarg = smax.partition(":")
|
|
self._skipexc = exception.StopExtraction(sarg or None)
|
|
elif skip == "terminate":
|
|
self._skipexc = exception.TerminateExtraction
|
|
elif skip == "exit":
|
|
self._skipexc = SystemExit
|
|
self._skipmax = text.parse_int(smax)
|
|
|
|
if skip_filter := cfg("skip-filter"):
|
|
self._skipftr = util.compile_filter(skip_filter)
|
|
else:
|
|
self._skipftr = None
|
|
else:
|
|
# monkey-patch methods to always return False
|
|
pathfmt.exists = lambda x=None: False
|
|
if self.archive is not None:
|
|
self.archive.check = pathfmt.exists
|
|
|
|
if not cfg("postprocess", True):
|
|
return
|
|
|
|
if postprocessors := extr.config_accumulate("postprocessors"):
|
|
self.hooks = collections.defaultdict(list)
|
|
|
|
pp_log = self.get_logger("postprocessor")
|
|
pp_conf = config.get((), "postprocessor") or {}
|
|
pp_opts = cfg("postprocessor-options")
|
|
pp_list = []
|
|
|
|
for pp_dict in postprocessors:
|
|
if isinstance(pp_dict, str):
|
|
pp_dict = pp_conf.get(pp_dict) or {"name": pp_dict}
|
|
elif "type" in pp_dict:
|
|
pp_type = pp_dict["type"]
|
|
if pp_type in pp_conf:
|
|
pp = pp_conf[pp_type].copy()
|
|
pp.update(pp_dict)
|
|
pp_dict = pp
|
|
if "name" not in pp_dict:
|
|
pp_dict["name"] = pp_type
|
|
if pp_opts:
|
|
pp_dict = pp_dict.copy()
|
|
pp_dict.update(pp_opts)
|
|
|
|
clist = pp_dict.get("whitelist")
|
|
if clist is not None:
|
|
negate = False
|
|
else:
|
|
clist = pp_dict.get("blacklist")
|
|
negate = True
|
|
if clist and not util.build_extractor_filter(
|
|
clist, negate)(extr):
|
|
continue
|
|
|
|
name = pp_dict.get("name", "")
|
|
if "__init__" not in pp_dict:
|
|
name, sep, event = name.rpartition("@")
|
|
if sep:
|
|
pp_dict["name"] = name
|
|
if "event" not in pp_dict:
|
|
pp_dict["event"] = event
|
|
else:
|
|
name = event
|
|
|
|
name, sep, mode = name.rpartition("/")
|
|
if sep:
|
|
pp_dict["name"] = name
|
|
if "mode" not in pp_dict:
|
|
pp_dict["mode"] = mode
|
|
else:
|
|
name = mode
|
|
|
|
pp_dict["__init__"] = None
|
|
|
|
pp_cls = postprocessor.find(name)
|
|
if pp_cls is None:
|
|
pp_log.warning("module '%s' not found", name)
|
|
continue
|
|
try:
|
|
pp_obj = pp_cls(self, pp_dict)
|
|
except Exception as exc:
|
|
pp_log.traceback(exc)
|
|
pp_log.error("'%s' initialization failed: %s: %s",
|
|
name, exc.__class__.__name__, exc)
|
|
else:
|
|
pp_list.append(pp_obj)
|
|
|
|
if pp_list:
|
|
extr.log.debug("Active postprocessor modules: %s", pp_list)
|
|
if "init" in self.hooks:
|
|
for callback in self.hooks["init"]:
|
|
callback(pathfmt)
|
|
|
|
def register_hooks(self, hooks, options=None):
|
|
expr = options.get("filter") if options else None
|
|
|
|
if expr:
|
|
condition = util.compile_filter(expr)
|
|
for hook, callback in hooks.items():
|
|
self.hooks[hook].append(functools.partial(
|
|
_call_hook_condition, callback, condition))
|
|
else:
|
|
for hook, callback in hooks.items():
|
|
self.hooks[hook].append(callback)
|
|
|
|
def _build_extractor_filter(self):
|
|
clist = self.extractor.config("whitelist")
|
|
if clist is not None:
|
|
negate = False
|
|
special = None
|
|
else:
|
|
clist = self.extractor.config("blacklist")
|
|
negate = True
|
|
special = util.SPECIAL_EXTRACTORS
|
|
if clist is None:
|
|
clist = (self.extractor.category,)
|
|
|
|
return util.build_extractor_filter(clist, negate, special)
|
|
|
|
|
|
def _call_hook_condition(callback, condition, pathfmt):
|
|
if condition(pathfmt.kwdict):
|
|
callback(pathfmt)
|
|
|
|
|
|
class SimulationJob(DownloadJob):
|
|
"""Simulate the extraction process without downloading anything"""
|
|
|
|
def handle_url(self, url, kwdict):
|
|
ext = kwdict["extension"] or "jpg"
|
|
kwdict["extension"] = self.pathfmt.extension_map(ext, ext)
|
|
if self.sleep is not None:
|
|
self.extractor.sleep(self.sleep(), "download")
|
|
if self.archive is not None and self._archive_write_skip:
|
|
self.archive.add(kwdict)
|
|
self.out.skip(self.pathfmt.build_filename(kwdict))
|
|
|
|
def handle_directory(self, kwdict):
|
|
if self.pathfmt is None:
|
|
self.initialize()
|
|
|
|
|
|
class KeywordJob(Job):
|
|
"""Print available keywords"""
|
|
|
|
def __init__(self, url, parent=None):
|
|
Job.__init__(self, url, parent)
|
|
self.private = config.get(("output",), "private")
|
|
|
|
def handle_url(self, url, kwdict):
|
|
stdout_write("\nKeywords for filenames and --filter:\n"
|
|
"------------------------------------\n")
|
|
|
|
if self.metadata_http and url.startswith("http"):
|
|
kwdict[self.metadata_http] = util.extract_headers(
|
|
self.extractor.request(url, method="HEAD"))
|
|
|
|
self.print_kwdict(kwdict)
|
|
raise exception.StopExtraction()
|
|
|
|
def handle_directory(self, kwdict):
|
|
stdout_write("Keywords for directory names:\n"
|
|
"-----------------------------\n")
|
|
self.print_kwdict(kwdict)
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
extr = None
|
|
if "_extractor" in kwdict:
|
|
extr = kwdict["_extractor"].from_url(url)
|
|
|
|
if not util.filter_dict(kwdict):
|
|
self.extractor.log.info(
|
|
"This extractor only spawns other extractors "
|
|
"and does not provide any metadata on its own.")
|
|
|
|
if extr:
|
|
self.extractor.log.info(
|
|
"Showing results for '%s' instead:\n", url)
|
|
KeywordJob(extr, self).run()
|
|
else:
|
|
self.extractor.log.info(
|
|
"Try 'gallery-dl -K \"%s\"' instead.", url)
|
|
else:
|
|
stdout_write("Keywords for --chapter-filter:\n"
|
|
"------------------------------\n")
|
|
self.print_kwdict(kwdict)
|
|
if extr or self.extractor.categorytransfer:
|
|
stdout_write("\n")
|
|
KeywordJob(extr or url, self).run()
|
|
raise exception.StopExtraction()
|
|
|
|
def print_kwdict(self, kwdict, prefix="", markers=None):
|
|
"""Print key-value pairs in 'kwdict' with formatting"""
|
|
write = sys.stdout.write
|
|
suffix = "']" if prefix else ""
|
|
|
|
markerid = id(kwdict)
|
|
if markers is None:
|
|
markers = {markerid}
|
|
elif markerid in markers:
|
|
write(f"{prefix[:-2]}\n <circular reference>\n")
|
|
return # ignore circular reference
|
|
else:
|
|
markers.add(markerid)
|
|
|
|
for key, value in sorted(kwdict.items()):
|
|
if key[0] == "_" and not self.private:
|
|
continue
|
|
key = prefix + key + suffix
|
|
|
|
if isinstance(value, dict):
|
|
self.print_kwdict(value, key + "['", markers)
|
|
|
|
elif isinstance(value, list):
|
|
if not value:
|
|
pass
|
|
elif isinstance(value[0], dict):
|
|
self.print_kwdict(value[0], key + "[N]['", markers)
|
|
else:
|
|
fmt = (" {:>%s} {}\n" % len(str(len(value)))).format
|
|
write(key + "[N]\n")
|
|
for idx, val in enumerate(value, 0):
|
|
write(fmt(idx, val))
|
|
|
|
else:
|
|
# string or number
|
|
write(f"{key}\n {value}\n")
|
|
|
|
markers.remove(markerid)
|
|
|
|
|
|
class UrlJob(Job):
|
|
"""Print download urls"""
|
|
maxdepth = 1
|
|
|
|
def __init__(self, url, parent=None, depth=1):
|
|
Job.__init__(self, url, parent)
|
|
self.depth = depth
|
|
if depth >= self.maxdepth:
|
|
self.handle_queue = self.handle_url
|
|
|
|
def handle_url(self, url, _):
|
|
stdout_write(url + "\n")
|
|
|
|
def handle_url_fallback(self, url, kwdict):
|
|
stdout_write(url + "\n")
|
|
if "_fallback" in kwdict:
|
|
for url in kwdict["_fallback"]:
|
|
stdout_write(f"| {url}\n")
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
if cls := kwdict.get("_extractor"):
|
|
extr = cls.from_url(url)
|
|
else:
|
|
extr = extractor.find(url)
|
|
|
|
if extr:
|
|
self.status |= self.__class__(extr, self, self.depth + 1).run()
|
|
else:
|
|
self._write_unsupported(url)
|
|
|
|
|
|
class InfoJob(Job):
|
|
"""Print extractor defaults and settings"""
|
|
|
|
def run(self):
|
|
ex = self.extractor
|
|
pm = self._print_multi
|
|
pc = self._print_config
|
|
|
|
if ex.basecategory:
|
|
pm("Category / Subcategory / Basecategory",
|
|
ex.category, ex.subcategory, ex.basecategory)
|
|
else:
|
|
pm("Category / Subcategory", ex.category, ex.subcategory)
|
|
|
|
pc("Filename format", "filename", ex.filename_fmt)
|
|
pc("Directory format", "directory", ex.directory_fmt)
|
|
pc("Archive format", "archive-format", ex.archive_fmt)
|
|
pc("Request interval", "sleep-request", ex.request_interval)
|
|
|
|
return 0
|
|
|
|
def _print_multi(self, title, *values):
|
|
stdout_write(
|
|
f"{title}\n {' / '.join(map(util.json_dumps, values))}\n\n")
|
|
|
|
def _print_config(self, title, optname, value):
|
|
optval = self.extractor.config(optname, util.SENTINEL)
|
|
if optval is not util.SENTINEL:
|
|
stdout_write(
|
|
f"{title} (custom):\n {util.json_dumps(optval)}\n"
|
|
f"{title} (default):\n {util.json_dumps(value)}\n\n")
|
|
elif value:
|
|
stdout_write(
|
|
f"{title} (default):\n {util.json_dumps(value)}\n\n")
|
|
|
|
|
|
class DataJob(Job):
|
|
"""Collect extractor results and dump them"""
|
|
resolve = False
|
|
|
|
def __init__(self, url, parent=None, file=sys.stdout, ensure_ascii=True,
|
|
resolve=False):
|
|
Job.__init__(self, url, parent)
|
|
self.file = file
|
|
self.data = []
|
|
self.data_urls = []
|
|
self.data_post = []
|
|
self.data_meta = []
|
|
self.exception = None
|
|
self.ascii = config.get(("output",), "ascii", ensure_ascii)
|
|
self.jsonl = config.get(("output",), "jsonl", False)
|
|
self.resolve = 128 if resolve is True else (resolve or self.resolve)
|
|
|
|
private = config.get(("output",), "private")
|
|
self.filter = dict.copy if private else util.filter_dict
|
|
|
|
if self.resolve > 0:
|
|
self.handle_queue = self.handle_queue_resolve
|
|
if not self.jsonl:
|
|
self.out = util.noop
|
|
|
|
def run(self):
|
|
self._init()
|
|
|
|
extractor = self.extractor
|
|
sleep = util.build_duration_func(
|
|
extractor.config("sleep-extractor"))
|
|
if sleep is not None:
|
|
extractor.sleep(sleep(), "extractor")
|
|
|
|
# collect data
|
|
try:
|
|
self.dispatch(extractor)
|
|
except exception.StopExtraction:
|
|
pass
|
|
except Exception as exc:
|
|
self.exception = exc
|
|
self.data.append((-1, {
|
|
"error" : exc.__class__.__name__,
|
|
"message": str(exc),
|
|
}))
|
|
except BaseException:
|
|
pass
|
|
|
|
# convert numbers to string
|
|
if config.get(("output",), "num-to-str", False):
|
|
for msg in self.data:
|
|
util.transform_dict(msg[-1], util.number_to_string)
|
|
|
|
if self.file and not self.jsonl:
|
|
# dump to 'file'
|
|
try:
|
|
util.dump_json(self.data, self.file, self.ascii, 2)
|
|
self.file.flush()
|
|
except Exception:
|
|
pass
|
|
|
|
return 0
|
|
|
|
def out(self, msg):
|
|
self.file.write(util.json_dumps(msg))
|
|
self.file.write("\n")
|
|
self.file.flush()
|
|
|
|
def handle_url(self, url, kwdict):
|
|
kwdict = self.filter(kwdict)
|
|
self.out(msg := (Message.Url, url, kwdict))
|
|
self.data_urls.append(url)
|
|
self.data_meta.append(kwdict)
|
|
self.data.append(msg)
|
|
|
|
def handle_directory(self, kwdict):
|
|
kwdict = self.filter(kwdict)
|
|
self.out(msg := (Message.Directory, kwdict))
|
|
self.data_post.append(kwdict)
|
|
self.data.append(msg)
|
|
|
|
def handle_queue(self, url, kwdict):
|
|
kwdict = self.filter(kwdict)
|
|
self.out(msg := (Message.Queue, url, kwdict))
|
|
self.data_urls.append(url)
|
|
self.data_meta.append(kwdict)
|
|
self.data.append(msg)
|
|
|
|
def handle_queue_resolve(self, url, kwdict):
|
|
if cls := kwdict.get("_extractor"):
|
|
extr = cls.from_url(url)
|
|
else:
|
|
extr = extractor.find(url)
|
|
|
|
if not extr:
|
|
kwdict = self.filter(kwdict)
|
|
self.out(msg := (Message.Queue, url, kwdict))
|
|
self.data_urls.append(url)
|
|
self.data_meta.append(kwdict)
|
|
return self.data.append(msg)
|
|
|
|
job = self.__class__(extr, self, None, self.ascii, self.resolve-1)
|
|
job.data = self.data
|
|
job.data_urls = self.data_urls
|
|
job.data_post = self.data_post
|
|
job.data_meta = self.data_meta
|
|
job.run()
|