decouple extractor initialization

Introduce an 'initialize()' function that does the actual init
(session, cookies, config options) and can called separately from
the constructor __init__().

This allows, for example, to adjust config access inside a Job
before most of it already happened when calling 'extractor.find()'.
This commit is contained in:
Mike Fährmann
2023-07-25 20:09:44 +02:00
parent f0203b7559
commit a383eca7f6
71 changed files with 314 additions and 193 deletions

View File

@@ -38,14 +38,18 @@ class DeviantartExtractor(Extractor):
def __init__(self, match):
Extractor.__init__(self, match)
self.user = match.group(1) or match.group(2)
def _init(self):
self.flat = self.config("flat", True)
self.extra = self.config("extra", False)
self.original = self.config("original", True)
self.comments = self.config("comments", False)
self.user = match.group(1) or match.group(2)
self.api = DeviantartOAuthAPI(self)
self.group = False
self.offset = 0
self.api = None
self._premium_cache = {}
unwatch = self.config("auto-unwatch")
if unwatch:
@@ -60,11 +64,13 @@ class DeviantartExtractor(Extractor):
self._update_content = self._update_content_image
self.original = True
self._premium_cache = {}
self.commit_journal = {
"html": self._commit_journal_html,
"text": self._commit_journal_text,
}.get(self.config("journals", "html"))
journals = self.config("journals", "html")
if journals == "html":
self.commit_journal = self._commit_journal_html
elif journals == "text":
self.commit_journal = self._commit_journal_text
else:
self.commit_journal = None
def skip(self, num):
self.offset += num
@@ -80,8 +86,6 @@ class DeviantartExtractor(Extractor):
return True
def items(self):
self.api = DeviantartOAuthAPI(self)
if self.user and self.config("group", True):
profile = self.api.user_profile(self.user)
self.group = not profile
@@ -449,6 +453,9 @@ class DeviantartUserExtractor(DeviantartExtractor):
("https://shimoda7.deviantart.com/"),
)
def initialize(self):
pass
def items(self):
base = "{}/{}/".format(self.root, self.user)
return self._dispatch_extractors((