add missing extractor info when spawning new ones (fixes #1051)
Not having this information causes the blacklist/whitelist logic to trigger and prevents things from functioning as intended when using default settings. Fixes issues for 8muses, deviantart, exhentai, and mangoxo.
This commit is contained in:
@@ -94,12 +94,12 @@ class _8musesAlbumExtractor(Extractor):
|
|||||||
if albums:
|
if albums:
|
||||||
for album in albums:
|
for album in albums:
|
||||||
url = self.root + "/comics/album/" + album["permalink"]
|
url = self.root + "/comics/album/" + album["permalink"]
|
||||||
album = {
|
yield Message.Queue, url, {
|
||||||
"url" : url,
|
"url" : url,
|
||||||
"name" : album["name"],
|
"name" : album["name"],
|
||||||
"private": album["isPrivate"],
|
"private" : album["isPrivate"],
|
||||||
|
"_extractor": _8musesAlbumExtractor,
|
||||||
}
|
}
|
||||||
yield Message.Queue, url, album
|
|
||||||
|
|
||||||
if data["page"] >= data["pages"]:
|
if data["page"] >= data["pages"]:
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -262,9 +262,11 @@ class DeviantartExtractor(Extractor):
|
|||||||
return folder
|
return folder
|
||||||
raise exception.NotFoundError("folder")
|
raise exception.NotFoundError("folder")
|
||||||
|
|
||||||
def _folder_urls(self, folders, category):
|
def _folder_urls(self, folders, category, extractor):
|
||||||
url = "{}/{}/{}/0/".format(self.root, self.user, category)
|
base = "{}/{}/{}/0/".format(self.root, self.user, category)
|
||||||
return [(url + folder["name"], folder) for folder in folders]
|
for folder in folders:
|
||||||
|
folder["_extractor"] = extractor
|
||||||
|
yield base + folder["name"], folder
|
||||||
|
|
||||||
def _update_content_default(self, deviation, content):
|
def _update_content_default(self, deviation, content):
|
||||||
public = "premium_folder_data" not in deviation
|
public = "premium_folder_data" not in deviation
|
||||||
@@ -450,7 +452,7 @@ class DeviantartGalleryExtractor(DeviantartExtractor):
|
|||||||
if self.flat and not self.group:
|
if self.flat and not self.group:
|
||||||
return self.api.gallery_all(self.user, self.offset)
|
return self.api.gallery_all(self.user, self.offset)
|
||||||
folders = self.api.gallery_folders(self.user)
|
folders = self.api.gallery_folders(self.user)
|
||||||
return self._folder_urls(folders, "gallery")
|
return self._folder_urls(folders, "gallery", DeviantartFolderExtractor)
|
||||||
|
|
||||||
|
|
||||||
class DeviantartFolderExtractor(DeviantartExtractor):
|
class DeviantartFolderExtractor(DeviantartExtractor):
|
||||||
@@ -589,7 +591,8 @@ class DeviantartFavoriteExtractor(DeviantartExtractor):
|
|||||||
self.api.collections(self.user, folder["folderid"])
|
self.api.collections(self.user, folder["folderid"])
|
||||||
for folder in folders
|
for folder in folders
|
||||||
)
|
)
|
||||||
return self._folder_urls(folders, "favourites")
|
return self._folder_urls(
|
||||||
|
folders, "favourites", DeviantartCollectionExtractor)
|
||||||
|
|
||||||
|
|
||||||
class DeviantartCollectionExtractor(DeviantartExtractor):
|
class DeviantartCollectionExtractor(DeviantartExtractor):
|
||||||
|
|||||||
@@ -392,6 +392,7 @@ class ExhentaiSearchExtractor(ExhentaiExtractor):
|
|||||||
def items(self):
|
def items(self):
|
||||||
self.login()
|
self.login()
|
||||||
yield Message.Version, 1
|
yield Message.Version, 1
|
||||||
|
data = {"_extractor": ExhentaiGalleryExtractor}
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
last = None
|
last = None
|
||||||
@@ -402,7 +403,7 @@ class ExhentaiSearchExtractor(ExhentaiExtractor):
|
|||||||
if url == last:
|
if url == last:
|
||||||
continue
|
continue
|
||||||
last = url
|
last = url
|
||||||
yield Message.Queue, url, {}
|
yield Message.Queue, url, data
|
||||||
|
|
||||||
if 'class="ptdd">><' in page or ">No hits found</p>" in page:
|
if 'class="ptdd">><' in page or ">No hits found</p>" in page:
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -167,6 +167,8 @@ class MangoxoChannelExtractor(MangoxoExtractor):
|
|||||||
self.login()
|
self.login()
|
||||||
num = total = 1
|
num = total = 1
|
||||||
url = "{}/channel/{}/album/".format(self.root, self.channel_id)
|
url = "{}/channel/{}/album/".format(self.root, self.channel_id)
|
||||||
|
data = {"_extractor": MangoxoAlbumExtractor}
|
||||||
|
|
||||||
yield Message.Version, 1
|
yield Message.Version, 1
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
@@ -174,7 +176,7 @@ class MangoxoChannelExtractor(MangoxoExtractor):
|
|||||||
|
|
||||||
for album in text.extract_iter(
|
for album in text.extract_iter(
|
||||||
page, '<a class="link black" href="', '"'):
|
page, '<a class="link black" href="', '"'):
|
||||||
yield Message.Queue, album, {}
|
yield Message.Queue, album, data
|
||||||
|
|
||||||
if num == 1:
|
if num == 1:
|
||||||
total = self._total_pages(page)
|
total = self._total_pages(page)
|
||||||
|
|||||||
Reference in New Issue
Block a user