initial support for child extractor options
Using "parent-category>child-category" as extractor category in a config
file allows to set options for a child extractor when it was spawned by
that parent.
For example "reddit>gfycat" to set gfycat options for when it was found
in a reddit post.
{
"extractor": {
"gfycat": {
"filename": "regular filename"
},
"reddit>gfycat": {
"filename": "reddit-specific filename"
}
}
}
Note: This does currently not work for most imgur links due to how its
extractor hierarchy is structured.
This commit is contained in:
@@ -45,10 +45,6 @@ class Extractor():
|
||||
def __init__(self, match):
|
||||
self.log = logging.getLogger(self.category)
|
||||
self.url = match.string
|
||||
|
||||
if self.basecategory:
|
||||
self.config = self._config_shared
|
||||
self.config_accumulate = self._config_shared_accumulate
|
||||
self._cfgpath = ("extractor", self.category, self.subcategory)
|
||||
self._parentdir = ""
|
||||
|
||||
@@ -98,16 +94,22 @@ class Extractor():
|
||||
return config.accumulate(self._cfgpath, key)
|
||||
|
||||
def _config_shared(self, key, default=None):
|
||||
return config.interpolate_common(("extractor",), (
|
||||
(self.category, self.subcategory),
|
||||
(self.basecategory, self.subcategory),
|
||||
), key, default)
|
||||
return config.interpolate_common(
|
||||
("extractor",), self._cfgpath, key, default)
|
||||
|
||||
def _config_shared_accumulate(self, key):
|
||||
values = config.accumulate(self._cfgpath, key)
|
||||
conf = config.get(("extractor",), self.basecategory)
|
||||
if conf:
|
||||
values[:0] = config.accumulate((self.subcategory,), key, conf=conf)
|
||||
first = True
|
||||
extr = ("extractor",)
|
||||
|
||||
for path in self._cfgpath:
|
||||
if first:
|
||||
first = False
|
||||
values = config.accumulate(extr + path, key)
|
||||
else:
|
||||
conf = config.get(extr, path[0])
|
||||
if conf:
|
||||
values[:0] = config.accumulate(
|
||||
(self.subcategory,), key, conf=conf)
|
||||
return values
|
||||
|
||||
def request(self, url, method="GET", session=None,
|
||||
|
||||
@@ -32,6 +32,21 @@ class Job():
|
||||
self.kwdict = {}
|
||||
self.status = 0
|
||||
|
||||
cfgpath = []
|
||||
if parent and parent.extractor.category != extr.category:
|
||||
cat = "{}>{}".format(
|
||||
parent.extractor.category, extr.category)
|
||||
cfgpath.append((cat, extr.subcategory))
|
||||
cfgpath.append((extr.category, extr.subcategory))
|
||||
if extr.basecategory:
|
||||
if not cfgpath:
|
||||
cfgpath.append((extr.category, extr.subcategory))
|
||||
cfgpath.append((extr.basecategory, extr.subcategory))
|
||||
if cfgpath:
|
||||
extr._cfgpath = cfgpath
|
||||
extr.config = extr._config_shared
|
||||
extr.config_accumulate = extr._config_shared_accumulate
|
||||
|
||||
actions = extr.config("actions")
|
||||
if actions:
|
||||
from .actions import parse
|
||||
|
||||
Reference in New Issue
Block a user