remove most 'yield Message.Version' statements

only leave them in oauth.py as noop results
This commit is contained in:
Mike Fährmann
2021-08-16 03:10:48 +02:00
parent bdfdabf498
commit bd08ee2859
59 changed files with 0 additions and 91 deletions

View File

@@ -35,7 +35,6 @@ class _2chanThreadExtractor(Extractor):
self.server, self.board, self.thread)
page = self.request(url).text
data = self.metadata(page)
yield Message.Version, 1
yield Message.Directory, data
for post in self.posts(page):
if "filename" not in post:

View File

@@ -22,7 +22,6 @@ class _35photoExtractor(Extractor):
def items(self):
first = True
data = self.metadata()
yield Message.Version, 1
for photo_id in self.photos():
for photo in self._photo_data(photo_id):

View File

@@ -49,7 +49,6 @@ class _4chanThreadExtractor(Extractor):
"title" : text.unescape(title)[:50],
}
yield Message.Version, 1
yield Message.Directory, data
for post in posts:
if "filename" in post:

View File

@@ -29,7 +29,6 @@ class _500pxExtractor(Extractor):
def items(self):
first = True
data = self.metadata()
yield Message.Version, 1
for photo in self.photos():
url = photo["images"][-1]["url"]

View File

@@ -49,7 +49,6 @@ class _8kunThreadExtractor(Extractor):
"num" : 0,
}
yield Message.Version, 1
yield Message.Directory, data
for post in posts:
if "filename" in post:

View File

@@ -19,7 +19,6 @@ class BehanceExtractor(Extractor):
root = "https://www.behance.net"
def items(self):
yield Message.Version, 1
for gallery in self.galleries():
gallery["_extractor"] = BehanceGalleryExtractor
yield Message.Queue, gallery["url"], self._update(gallery)
@@ -99,7 +98,6 @@ class BehanceGalleryExtractor(BehanceExtractor):
imgs = self.get_images(data)
data["count"] = len(imgs)
yield Message.Version, 1
yield Message.Directory, data
for data["num"], (url, module) in enumerate(imgs, 1):
data["module"] = module

View File

@@ -34,7 +34,6 @@ class BloggerExtractor(Extractor):
self.api = BloggerAPI(self)
def items(self):
yield Message.Version, 1
blog = self.api.blog_by_url("http://" + self.blog)
blog["pages"] = blog["pages"]["totalItems"]

View File

@@ -448,7 +448,6 @@ class GalleryExtractor(Extractor):
pass
images = enumerate(imgs, 1)
yield Message.Version, 1
yield Message.Directory, data
for data[self.enum], (url, imgdata) in images:
if imgdata:
@@ -504,7 +503,6 @@ class MangaExtractor(Extractor):
if self.reverse:
chapters.reverse()
yield Message.Version, 1
for chapter, data in chapters:
data["_extractor"] = self.chapterclass
yield Message.Queue, chapter, data

View File

@@ -78,7 +78,6 @@ class DeviantartExtractor(Extractor):
else:
self.user = profile["user"]["username"]
yield Message.Version, 1
for deviation in self.deviations():
if isinstance(deviation, tuple):
url, data = deviation
@@ -924,7 +923,6 @@ class DeviantartFollowingExtractor(DeviantartExtractor):
def items(self):
eclipse_api = DeviantartEclipseAPI(self)
yield Message.Version, 1
for user in eclipse_api.user_watching(self.user, self.offset):
url = "{}/{}".format(self.root, user["username"])
user["_extractor"] = DeviantartUserExtractor

View File

@@ -62,6 +62,5 @@ class DirectlinkExtractor(Extractor):
data["_http_headers"] = {
"Referer": self.url.encode("latin-1", "ignore")}
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, self.url, data

View File

@@ -112,7 +112,6 @@ class DynastyscansSearchExtractor(DynastyscansBase, Extractor):
self.query = match.group(1) or ""
def items(self):
yield Message.Version, 1
yield Message.Directory, {}
for image_id in self.images():
image = self._parse_image_page(image_id)

View File

@@ -31,7 +31,6 @@ class FanboxExtractor(Extractor):
self.embeds = self.config("embeds", True)
def items(self):
yield Message.Version, 1
if self._warning:
if "FANBOXSESSID" not in self.session.cookies:

View File

@@ -20,7 +20,6 @@ class FantiaExtractor(Extractor):
_warning = True
def items(self):
yield Message.Version, 1
if self._warning:
if "_session_id" not in self.session.cookies:

View File

@@ -155,7 +155,6 @@ class FlickrAlbumExtractor(FlickrExtractor):
return self._album_items()
def _album_items(self):
yield Message.Version, 1
data = FlickrExtractor.metadata(self)
data["_extractor"] = FlickrAlbumExtractor

View File

@@ -94,7 +94,6 @@ class FoolslideChapterExtractor(FoolslideExtractor):
data["count"] = len(imgs)
data["chapter_id"] = text.parse_int(imgs[0]["chapter_id"])
yield Message.Version, 1
yield Message.Directory, data
for data["page"], image in enumerate(imgs, 1):
try:

View File

@@ -129,7 +129,6 @@ class HentaifoxSearchExtractor(HentaifoxBase, Extractor):
self.path = match.group(1)
def items(self):
yield Message.Version, 1
for gallery in self.galleries():
yield Message.Queue, gallery["url"], gallery

View File

@@ -57,7 +57,6 @@ class ImagefapGalleryExtractor(ImagefapExtractor):
url = "{}/pictures/{}/".format(self.root, self.gid)
page = self.request(url).text
data = self.get_job_metadata(page)
yield Message.Version, 1
yield Message.Directory, data
for url, image in self.get_images():
data.update(image)
@@ -118,7 +117,6 @@ class ImagefapImageExtractor(ImagefapExtractor):
def items(self):
url, data = self.get_image()
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, url, data
@@ -169,7 +167,6 @@ class ImagefapUserExtractor(ImagefapExtractor):
self.user, self.user_id = match.groups()
def items(self):
yield Message.Version, 1
for gid, name in self.get_gallery_data():
url = "{}/gallery/{}".format(self.root, gid)
data = {

View File

@@ -57,7 +57,6 @@ class ImagehostImageExtractor(Extractor):
if self.https and url.startswith("http:"):
url = "https:" + url[5:]
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, url, data

View File

@@ -43,7 +43,6 @@ class ImgbbExtractor(Extractor):
data = self.metadata(page)
first = True
yield Message.Version, 1
for img in self.images(page):
image = {
"id" : img["url_viewer"].rpartition("/")[2],
@@ -230,6 +229,5 @@ class ImgbbImageExtractor(ImgbbExtractor):
}
image["extension"] = text.ext_from_url(image["url"])
yield Message.Version, 1
yield Message.Directory, image
yield Message.Url, image["url"], image

View File

@@ -20,7 +20,6 @@ class ImgboxExtractor(Extractor):
def items(self):
data = self.get_job_metadata()
yield Message.Version, 1
yield Message.Directory, data
for image_key in self.get_image_keys():

View File

@@ -33,7 +33,6 @@ class ImgthGalleryExtractor(Extractor):
def items(self):
page = self.request(self.url_base + "0").text
data = self.metadata(page)
yield Message.Version, 1
yield Message.Directory, data
for data["num"], url in enumerate(self.images(page), 1):
yield Message.Url, url, text.nameext_from_url(url, data)

View File

@@ -46,7 +46,6 @@ class ImgurExtractor(Extractor):
album_ex = ImgurAlbumExtractor
image_ex = ImgurImageExtractor
yield Message.Version, 1
for item in items:
item["_extractor"] = album_ex if item["is_album"] else image_ex
yield Message.Queue, item["link"], item
@@ -133,7 +132,6 @@ class ImgurImageExtractor(ImgurExtractor):
image.update(image["media"][0])
del image["media"]
url = self._prepare(image)
yield Message.Version, 1
yield Message.Directory, image
yield Message.Url, url, image
@@ -221,7 +219,6 @@ class ImgurAlbumExtractor(ImgurExtractor):
except KeyError:
pass
yield Message.Version, 1
for num, image in enumerate(images, 1):
url = self._prepare(image)
image["num"] = num

View File

@@ -94,7 +94,6 @@ class IssuuUserExtractor(IssuuBase, Extractor):
url = "{}/call/profile/v1/documents/{}".format(self.root, self.user)
params = {"offset": 0, "limit": "25"}
yield Message.Version, 1
while True:
data = self.request(url, params=params).json()

View File

@@ -55,7 +55,6 @@ class KeenspotComicExtractor(Extractor):
def items(self):
data = {"comic": self.comic}
yield Message.Version, 1
yield Message.Directory, data
with self.request(self.root + "/") as response:

View File

@@ -40,7 +40,6 @@ class KhinsiderSoundtrackExtractor(AsynchronousMixin, Extractor):
raise exception.NotFoundError("soundtrack")
data = self.metadata(page)
yield Message.Version, 1
yield Message.Directory, data
for track in self.tracks(page):
track.update(data)

View File

@@ -25,7 +25,6 @@ class LivedoorExtractor(Extractor):
self.user = match.group(1)
def items(self):
yield Message.Version, 1
for post in self.posts():
images = self._images(post)
if images:

View File

@@ -120,7 +120,6 @@ class LusciousAlbumExtractor(LusciousExtractor):
def items(self):
album = self.metadata()
yield Message.Version, 1
yield Message.Directory, {"album": album}
for num, image in enumerate(self.images(), 1):
image["num"] = num
@@ -443,7 +442,6 @@ fragment AlbumMinimal on Album {
}
}
"""
yield Message.Version, 1
while True:
data = self._graphql("AlbumListWithPeek", variables, query)

View File

@@ -106,7 +106,6 @@ class MangoxoAlbumExtractor(MangoxoExtractor):
data = self.metadata(page)
imgs = self.images(url, page)
yield Message.Version, 1
yield Message.Directory, data
data["extension"] = None
@@ -175,8 +174,6 @@ class MangoxoChannelExtractor(MangoxoExtractor):
url = "{}/channel/{}/album/".format(self.root, self.channel_id)
data = {"_extractor": MangoxoAlbumExtractor}
yield Message.Version, 1
while True:
page = self.request(url + str(num)).text

View File

@@ -102,7 +102,6 @@ class NaverBlogExtractor(NaverBase, Extractor):
self.blog_id = match.group(1) or match.group(2)
def items(self):
yield Message.Version, 1
# fetch first post number
url = "{}/PostList.nhn?blogId={}".format(self.root, self.blog_id)

View File

@@ -117,7 +117,6 @@ class NhentaiSearchExtractor(NhentaiBase, Extractor):
self.params = text.parse_query(match.group(1))
def items(self):
yield Message.Version, 1
data = {"_extractor": NhentaiGalleryExtractor}
for gallery_id in self._pagination(self.params):
url = "{}/g/{}/".format(self.root, gallery_id)

View File

@@ -36,7 +36,6 @@ class NijieExtractor(AsynchronousMixin, Extractor):
def items(self):
self.login()
yield Message.Version, 1
for image_id in self.image_ids():

View File

@@ -25,7 +25,6 @@ class NozomiExtractor(Extractor):
archive_fmt = "{dataid}"
def items(self):
yield Message.Version, 1
data = self.metadata()
self.session.headers["Origin"] = self.root

View File

@@ -27,7 +27,6 @@ class PatreonExtractor(Extractor):
_warning = True
def items(self):
yield Message.Version, 1
if self._warning:
if "session_id" not in self.session.cookies:

View File

@@ -54,7 +54,6 @@ class PhotobucketAlbumExtractor(Extractor):
self.session.headers["Referer"] = self.url
def items(self):
yield Message.Version, 1
for image in self.images():
image["titleOrFilename"] = text.unescape(image["titleOrFilename"])
image["title"] = text.unescape(image["title"])
@@ -172,6 +171,5 @@ class PhotobucketImageExtractor(Extractor):
mtype, _, mid = base64.b64decode(image["id"]).partition(b":")
image["pictureId"] = mid.decode() if mtype == b"mediaId" else ""
yield Message.Version, 1
yield Message.Directory, image
yield Message.Url, image["fileUrl"], image

View File

@@ -22,7 +22,6 @@ class PiczelExtractor(Extractor):
api_root = "https://tombstone.piczel.tv"
def items(self):
yield Message.Version, 1
for post in self.posts():
post["tags"] = [t["title"] for t in post["tags"] if t["title"]]
post["date"] = text.parse_datetime(

View File

@@ -213,7 +213,6 @@ class PixivMeExtractor(PixivExtractor):
data = {"_extractor": PixivUserExtractor}
response = self.request(
url, method="HEAD", allow_redirects=False, notfound="user")
yield Message.Version, 1
yield Message.Queue, response.headers["Location"], data

View File

@@ -89,7 +89,6 @@ class PixnetImageExtractor(PixnetExtractor):
data["blog"] = self.blog
data["user"] = data.pop("author_name")
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, data["url"], data
@@ -120,7 +119,6 @@ class PixnetSetExtractor(PixnetExtractor):
page = self.request(url, encoding="utf-8").text
data = self.metadata(page)
yield Message.Version, 1
yield Message.Directory, data
for num, info in enumerate(self._pagination(page), 1):
url, pos = text.extract(info, ' href="', '"')

View File

@@ -60,7 +60,6 @@ class PornhubGalleryExtractor(PornhubExtractor):
def items(self):
data = self.metadata()
yield Message.Version, 1
yield Message.Directory, data
for num, image in enumerate(self.images(), 1):
url = image["url"]
@@ -146,7 +145,6 @@ class PornhubUserExtractor(PornhubExtractor):
}
data = {"_extractor": PornhubGalleryExtractor}
yield Message.Version, 1
while True:
page = self.request(
url, method="POST", headers=headers, params=params).text

View File

@@ -37,7 +37,6 @@ class ReactorExtractor(Extractor):
def items(self):
data = self.metadata()
yield Message.Version, 1
yield Message.Directory, data
for post in self.posts():
for image in self._parse_post(post):
@@ -197,7 +196,6 @@ class ReactorPostExtractor(ReactorExtractor):
self.post_id = match.group(2)
def items(self):
yield Message.Version, 1
post = self.request(self.url).text
pos = post.find('class="uhead">')
for image in self._parse_post(post[pos:]):

View File

@@ -38,8 +38,6 @@ class RedditExtractor(Extractor):
visited = set()
depth = 0
yield Message.Version, 1
while True:
extra = []
@@ -257,7 +255,6 @@ class RedditImageExtractor(Extractor):
def items(self):
data = text.nameext_from_url(self.url)
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, self.url, data

View File

@@ -29,7 +29,6 @@ class SeigaExtractor(Extractor):
images = iter(self.get_images())
data = next(images)
yield Message.Version, 1
yield Message.Directory, data
for image in util.advance(images, self.start_image):
data.update(image)

View File

@@ -42,7 +42,6 @@ class SenmangaChapterExtractor(Extractor):
def items(self):
data = self.metadata()
yield Message.Version, 1
yield Message.Directory, data
for data["page"] in range(1, data["count"]+1):
data["extension"] = None

View File

@@ -21,7 +21,6 @@ class SexcomExtractor(Extractor):
root = "https://www.sex.com"
def items(self):
yield Message.Version, 1
yield Message.Directory, self.metadata()
for pin in map(self._parse_pin, self.pins()):
if pin:

View File

@@ -128,7 +128,6 @@ class SimplyhentaiImageExtractor(Extractor):
})
data["token"] = data["filename"].rpartition("_")[2]
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, url, data
@@ -192,6 +191,5 @@ class SimplyhentaiVideoExtractor(Extractor):
date), "%B %d, %Y %H:%M"),
})
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, video_url, data

View File

@@ -68,7 +68,6 @@ class SlickpicAlbumExtractor(SlickpicExtractor):
"count": len(imgs),
}
yield Message.Version, 1
yield Message.Directory, data
for num, img in enumerate(imgs, 1):
url = img["url_rsz"] + "/o/" + img["fname"]
@@ -137,6 +136,5 @@ class SlickpicUserExtractor(SlickpicExtractor):
data = {"_extractor": SlickpicAlbumExtractor}
base = self.root + "/albums/"
yield Message.Version, 1
for album in text.extract_iter(page, 'href="' + base, '"'):
yield Message.Queue, base + album, data

View File

@@ -49,7 +49,6 @@ class SlidesharePresentationExtractor(Extractor):
data = self.get_job_metadata(page)
imgs = self.get_image_urls(page)
data["count"] = len(imgs)
yield Message.Version, 1
yield Message.Directory, data
for data["num"], url in enumerate(imgs, 1):
yield Message.Url, url, text.nameext_from_url(url, data)

View File

@@ -95,7 +95,6 @@ class SmugmugAlbumExtractor(SmugmugExtractor):
del album["Uris"]
data = {"Album": album, "User": user}
yield Message.Version, 1
yield Message.Directory, data
for image in self.api.album_images(self.album_id, "ImageSizeDetails"):
@@ -133,7 +132,6 @@ class SmugmugImageExtractor(SmugmugExtractor):
data = {"Image": image}
text.nameext_from_url(url, data)
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, url, data
@@ -170,7 +168,6 @@ class SmugmugPathExtractor(SmugmugExtractor):
self.domain, self.user, self.path = match.groups()
def items(self):
yield Message.Version, 1
if not self.user:
self.user = self.api.site_user(self.domain)["NickName"]

View File

@@ -39,7 +39,6 @@ class SpeakerdeckPresentationExtractor(Extractor):
data = self.get_job_metadata()
imgs = self.get_image_urls()
data["count"] = len(imgs)
yield Message.Version, 1
yield Message.Directory, data
for data["num"], url in enumerate(imgs, 1):
yield Message.Url, url, text.nameext_from_url(url, data)

View File

@@ -71,7 +71,6 @@ class TestExtractor(Extractor):
if not tests:
raise exception.NotFoundError("test")
yield Message.Version, 1
for test in tests:
yield Message.Queue, test[0], {}

View File

@@ -152,7 +152,6 @@ class TsuminoSearchExtractor(TsuminoBase, Extractor):
self.query = match.group(1)
def items(self):
yield Message.Version, 1
for gallery in self.galleries():
url = "{}/entry/{}".format(self.root, gallery["id"])
gallery["_extractor"] = TsuminoGalleryExtractor

View File

@@ -56,7 +56,6 @@ class VanillarockPostExtractor(VanillarockExtractor):
'<div class="cat-tag">', '</div>'))[::2],
}
yield Message.Version, 1
yield Message.Directory, data
for data["num"], url in enumerate(imgs, 1):
yield Message.Url, url, text.nameext_from_url(url, data)
@@ -84,7 +83,6 @@ class VanillarockTagExtractor(VanillarockExtractor):
url = self.root + self.path
data = {"_extractor": VanillarockPostExtractor}
yield Message.Version, 1
while url:
extr = text.extract_from(self.request(url).text)
while True:

View File

@@ -30,7 +30,6 @@ class VscoExtractor(Extractor):
def items(self):
videos = self.config("videos", True)
yield Message.Version, 1
yield Message.Directory, {"user": self.user}
for img in self.images():

View File

@@ -47,7 +47,6 @@ class WarosuThreadExtractor(Extractor):
title = text.remove_html(posts[0]["com"])
data["title"] = text.unescape(title)[:50]
yield Message.Version, 1
yield Message.Directory, data
for post in posts:
if "image" in post:

View File

@@ -123,7 +123,6 @@ class WeasylSubmissionsExtractor(WeasylExtractor):
self.owner_login = match.group(1)
def items(self):
yield Message.Version, 1
yield Message.Directory, {"owner_login": self.owner_login}
yield from self.submissions(self.owner_login)
@@ -141,7 +140,6 @@ class WeasylFolderExtractor(WeasylExtractor):
self.owner_login, self.folderid = match.groups()
def items(self):
yield Message.Version, 1
iter = self.submissions(self.owner_login, self.folderid)
# Folder names are only on single submission api calls
msg, url, data = next(iter)
@@ -171,7 +169,6 @@ class WeasylJournalExtractor(WeasylExtractor):
def items(self):
data = self.retrieve_journal(self.journalid)
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, data["html"], data
@@ -190,7 +187,6 @@ class WeasylJournalsExtractor(WeasylExtractor):
self.owner_login = match.group(1)
def items(self):
yield Message.Version, 1
yield Message.Directory, {"owner_login": self.owner_login}
url = "{}/journals/{}".format(self.root, self.owner_login)

View File

@@ -27,7 +27,6 @@ class WikiartExtractor(Extractor):
def items(self):
data = self.metadata()
yield Message.Version, 1
yield Message.Directory, data
for painting in self.paintings():
url = painting["image"]

View File

@@ -43,7 +43,6 @@ class WikieatThreadExtractor(Extractor):
"num" : 0,
}
yield Message.Version, 1
yield Message.Directory, data
for post in posts:
if "filename" in post:

View File

@@ -90,7 +90,6 @@ class XhamsterGalleryExtractor(XhamsterExtractor):
def items(self):
data = self.metadata()
yield Message.Version, 1
yield Message.Directory, data
for num, image in enumerate(self.images(), 1):
url = image["imageURL"]
@@ -167,7 +166,6 @@ class XhamsterUserExtractor(XhamsterExtractor):
self.user = match.group(2)
def items(self):
yield Message.Version, 1
url = "{}/users/{}/photos".format(self.root, self.user)
data = {"_extractor": XhamsterGalleryExtractor}

View File

@@ -129,7 +129,6 @@ class XvideosUserExtractor(XvideosBase, Extractor):
]
galleries.sort(key=lambda x: x["id"])
yield Message.Version, 1
for gallery in galleries:
url = "https://www.xvideos.com/profiles/{}/photos/{}".format(
self.user, gallery["id"])

View File

@@ -134,13 +134,6 @@ class Job():
if self.pred_queue(url, kwdict):
self.handle_queue(url, kwdict)
elif msg[0] == Message.Version:
if msg[1] != 1:
raise "unsupported message-version ({}, {})".format(
self.extractor.category, msg[1]
)
# TODO: support for multiple message versions
def handle_url(self, url, kwdict):
"""Handle Message.Url"""