initial support for child extractor options
Using "parent-category>child-category" as extractor category in a config
file allows to set options for a child extractor when it was spawned by
that parent.
For example "reddit>gfycat" to set gfycat options for when it was found
in a reddit post.
{
"extractor": {
"gfycat": {
"filename": "regular filename"
},
"reddit>gfycat": {
"filename": "reddit-specific filename"
}
}
}
Note: This does currently not work for most imgur links due to how its
extractor hierarchy is structured.
This commit is contained in:
@@ -45,10 +45,6 @@ class Extractor():
|
|||||||
def __init__(self, match):
|
def __init__(self, match):
|
||||||
self.log = logging.getLogger(self.category)
|
self.log = logging.getLogger(self.category)
|
||||||
self.url = match.string
|
self.url = match.string
|
||||||
|
|
||||||
if self.basecategory:
|
|
||||||
self.config = self._config_shared
|
|
||||||
self.config_accumulate = self._config_shared_accumulate
|
|
||||||
self._cfgpath = ("extractor", self.category, self.subcategory)
|
self._cfgpath = ("extractor", self.category, self.subcategory)
|
||||||
self._parentdir = ""
|
self._parentdir = ""
|
||||||
|
|
||||||
@@ -98,16 +94,22 @@ class Extractor():
|
|||||||
return config.accumulate(self._cfgpath, key)
|
return config.accumulate(self._cfgpath, key)
|
||||||
|
|
||||||
def _config_shared(self, key, default=None):
|
def _config_shared(self, key, default=None):
|
||||||
return config.interpolate_common(("extractor",), (
|
return config.interpolate_common(
|
||||||
(self.category, self.subcategory),
|
("extractor",), self._cfgpath, key, default)
|
||||||
(self.basecategory, self.subcategory),
|
|
||||||
), key, default)
|
|
||||||
|
|
||||||
def _config_shared_accumulate(self, key):
|
def _config_shared_accumulate(self, key):
|
||||||
values = config.accumulate(self._cfgpath, key)
|
first = True
|
||||||
conf = config.get(("extractor",), self.basecategory)
|
extr = ("extractor",)
|
||||||
|
|
||||||
|
for path in self._cfgpath:
|
||||||
|
if first:
|
||||||
|
first = False
|
||||||
|
values = config.accumulate(extr + path, key)
|
||||||
|
else:
|
||||||
|
conf = config.get(extr, path[0])
|
||||||
if conf:
|
if conf:
|
||||||
values[:0] = config.accumulate((self.subcategory,), key, conf=conf)
|
values[:0] = config.accumulate(
|
||||||
|
(self.subcategory,), key, conf=conf)
|
||||||
return values
|
return values
|
||||||
|
|
||||||
def request(self, url, method="GET", session=None,
|
def request(self, url, method="GET", session=None,
|
||||||
|
|||||||
@@ -32,6 +32,21 @@ class Job():
|
|||||||
self.kwdict = {}
|
self.kwdict = {}
|
||||||
self.status = 0
|
self.status = 0
|
||||||
|
|
||||||
|
cfgpath = []
|
||||||
|
if parent and parent.extractor.category != extr.category:
|
||||||
|
cat = "{}>{}".format(
|
||||||
|
parent.extractor.category, extr.category)
|
||||||
|
cfgpath.append((cat, extr.subcategory))
|
||||||
|
cfgpath.append((extr.category, extr.subcategory))
|
||||||
|
if extr.basecategory:
|
||||||
|
if not cfgpath:
|
||||||
|
cfgpath.append((extr.category, extr.subcategory))
|
||||||
|
cfgpath.append((extr.basecategory, extr.subcategory))
|
||||||
|
if cfgpath:
|
||||||
|
extr._cfgpath = cfgpath
|
||||||
|
extr.config = extr._config_shared
|
||||||
|
extr.config_accumulate = extr._config_shared_accumulate
|
||||||
|
|
||||||
actions = extr.config("actions")
|
actions = extr.config("actions")
|
||||||
if actions:
|
if actions:
|
||||||
from .actions import parse
|
from .actions import parse
|
||||||
|
|||||||
Reference in New Issue
Block a user