remove most 'yield Message.Version' statements
only leave them in oauth.py as noop results
This commit is contained in:
@@ -35,7 +35,6 @@ class _2chanThreadExtractor(Extractor):
|
||||
self.server, self.board, self.thread)
|
||||
page = self.request(url).text
|
||||
data = self.metadata(page)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for post in self.posts(page):
|
||||
if "filename" not in post:
|
||||
|
||||
@@ -22,7 +22,6 @@ class _35photoExtractor(Extractor):
|
||||
def items(self):
|
||||
first = True
|
||||
data = self.metadata()
|
||||
yield Message.Version, 1
|
||||
|
||||
for photo_id in self.photos():
|
||||
for photo in self._photo_data(photo_id):
|
||||
|
||||
@@ -49,7 +49,6 @@ class _4chanThreadExtractor(Extractor):
|
||||
"title" : text.unescape(title)[:50],
|
||||
}
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for post in posts:
|
||||
if "filename" in post:
|
||||
|
||||
@@ -29,7 +29,6 @@ class _500pxExtractor(Extractor):
|
||||
def items(self):
|
||||
first = True
|
||||
data = self.metadata()
|
||||
yield Message.Version, 1
|
||||
|
||||
for photo in self.photos():
|
||||
url = photo["images"][-1]["url"]
|
||||
|
||||
@@ -49,7 +49,6 @@ class _8kunThreadExtractor(Extractor):
|
||||
"num" : 0,
|
||||
}
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for post in posts:
|
||||
if "filename" in post:
|
||||
|
||||
@@ -19,7 +19,6 @@ class BehanceExtractor(Extractor):
|
||||
root = "https://www.behance.net"
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
for gallery in self.galleries():
|
||||
gallery["_extractor"] = BehanceGalleryExtractor
|
||||
yield Message.Queue, gallery["url"], self._update(gallery)
|
||||
@@ -99,7 +98,6 @@ class BehanceGalleryExtractor(BehanceExtractor):
|
||||
imgs = self.get_images(data)
|
||||
data["count"] = len(imgs)
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for data["num"], (url, module) in enumerate(imgs, 1):
|
||||
data["module"] = module
|
||||
|
||||
@@ -34,7 +34,6 @@ class BloggerExtractor(Extractor):
|
||||
self.api = BloggerAPI(self)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
|
||||
blog = self.api.blog_by_url("http://" + self.blog)
|
||||
blog["pages"] = blog["pages"]["totalItems"]
|
||||
|
||||
@@ -448,7 +448,6 @@ class GalleryExtractor(Extractor):
|
||||
pass
|
||||
images = enumerate(imgs, 1)
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for data[self.enum], (url, imgdata) in images:
|
||||
if imgdata:
|
||||
@@ -504,7 +503,6 @@ class MangaExtractor(Extractor):
|
||||
if self.reverse:
|
||||
chapters.reverse()
|
||||
|
||||
yield Message.Version, 1
|
||||
for chapter, data in chapters:
|
||||
data["_extractor"] = self.chapterclass
|
||||
yield Message.Queue, chapter, data
|
||||
|
||||
@@ -78,7 +78,6 @@ class DeviantartExtractor(Extractor):
|
||||
else:
|
||||
self.user = profile["user"]["username"]
|
||||
|
||||
yield Message.Version, 1
|
||||
for deviation in self.deviations():
|
||||
if isinstance(deviation, tuple):
|
||||
url, data = deviation
|
||||
@@ -924,7 +923,6 @@ class DeviantartFollowingExtractor(DeviantartExtractor):
|
||||
def items(self):
|
||||
eclipse_api = DeviantartEclipseAPI(self)
|
||||
|
||||
yield Message.Version, 1
|
||||
for user in eclipse_api.user_watching(self.user, self.offset):
|
||||
url = "{}/{}".format(self.root, user["username"])
|
||||
user["_extractor"] = DeviantartUserExtractor
|
||||
|
||||
@@ -62,6 +62,5 @@ class DirectlinkExtractor(Extractor):
|
||||
data["_http_headers"] = {
|
||||
"Referer": self.url.encode("latin-1", "ignore")}
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, self.url, data
|
||||
|
||||
@@ -112,7 +112,6 @@ class DynastyscansSearchExtractor(DynastyscansBase, Extractor):
|
||||
self.query = match.group(1) or ""
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, {}
|
||||
for image_id in self.images():
|
||||
image = self._parse_image_page(image_id)
|
||||
|
||||
@@ -31,7 +31,6 @@ class FanboxExtractor(Extractor):
|
||||
self.embeds = self.config("embeds", True)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
|
||||
if self._warning:
|
||||
if "FANBOXSESSID" not in self.session.cookies:
|
||||
|
||||
@@ -20,7 +20,6 @@ class FantiaExtractor(Extractor):
|
||||
_warning = True
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
|
||||
if self._warning:
|
||||
if "_session_id" not in self.session.cookies:
|
||||
|
||||
@@ -155,7 +155,6 @@ class FlickrAlbumExtractor(FlickrExtractor):
|
||||
return self._album_items()
|
||||
|
||||
def _album_items(self):
|
||||
yield Message.Version, 1
|
||||
data = FlickrExtractor.metadata(self)
|
||||
data["_extractor"] = FlickrAlbumExtractor
|
||||
|
||||
|
||||
@@ -94,7 +94,6 @@ class FoolslideChapterExtractor(FoolslideExtractor):
|
||||
data["count"] = len(imgs)
|
||||
data["chapter_id"] = text.parse_int(imgs[0]["chapter_id"])
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for data["page"], image in enumerate(imgs, 1):
|
||||
try:
|
||||
|
||||
@@ -129,7 +129,6 @@ class HentaifoxSearchExtractor(HentaifoxBase, Extractor):
|
||||
self.path = match.group(1)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
for gallery in self.galleries():
|
||||
yield Message.Queue, gallery["url"], gallery
|
||||
|
||||
|
||||
@@ -57,7 +57,6 @@ class ImagefapGalleryExtractor(ImagefapExtractor):
|
||||
url = "{}/pictures/{}/".format(self.root, self.gid)
|
||||
page = self.request(url).text
|
||||
data = self.get_job_metadata(page)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for url, image in self.get_images():
|
||||
data.update(image)
|
||||
@@ -118,7 +117,6 @@ class ImagefapImageExtractor(ImagefapExtractor):
|
||||
|
||||
def items(self):
|
||||
url, data = self.get_image()
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, url, data
|
||||
|
||||
@@ -169,7 +167,6 @@ class ImagefapUserExtractor(ImagefapExtractor):
|
||||
self.user, self.user_id = match.groups()
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
for gid, name in self.get_gallery_data():
|
||||
url = "{}/gallery/{}".format(self.root, gid)
|
||||
data = {
|
||||
|
||||
@@ -57,7 +57,6 @@ class ImagehostImageExtractor(Extractor):
|
||||
if self.https and url.startswith("http:"):
|
||||
url = "https:" + url[5:]
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, url, data
|
||||
|
||||
|
||||
@@ -43,7 +43,6 @@ class ImgbbExtractor(Extractor):
|
||||
data = self.metadata(page)
|
||||
first = True
|
||||
|
||||
yield Message.Version, 1
|
||||
for img in self.images(page):
|
||||
image = {
|
||||
"id" : img["url_viewer"].rpartition("/")[2],
|
||||
@@ -230,6 +229,5 @@ class ImgbbImageExtractor(ImgbbExtractor):
|
||||
}
|
||||
image["extension"] = text.ext_from_url(image["url"])
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, image
|
||||
yield Message.Url, image["url"], image
|
||||
|
||||
@@ -20,7 +20,6 @@ class ImgboxExtractor(Extractor):
|
||||
|
||||
def items(self):
|
||||
data = self.get_job_metadata()
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
|
||||
for image_key in self.get_image_keys():
|
||||
|
||||
@@ -33,7 +33,6 @@ class ImgthGalleryExtractor(Extractor):
|
||||
def items(self):
|
||||
page = self.request(self.url_base + "0").text
|
||||
data = self.metadata(page)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for data["num"], url in enumerate(self.images(page), 1):
|
||||
yield Message.Url, url, text.nameext_from_url(url, data)
|
||||
|
||||
@@ -46,7 +46,6 @@ class ImgurExtractor(Extractor):
|
||||
album_ex = ImgurAlbumExtractor
|
||||
image_ex = ImgurImageExtractor
|
||||
|
||||
yield Message.Version, 1
|
||||
for item in items:
|
||||
item["_extractor"] = album_ex if item["is_album"] else image_ex
|
||||
yield Message.Queue, item["link"], item
|
||||
@@ -133,7 +132,6 @@ class ImgurImageExtractor(ImgurExtractor):
|
||||
image.update(image["media"][0])
|
||||
del image["media"]
|
||||
url = self._prepare(image)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, image
|
||||
yield Message.Url, url, image
|
||||
|
||||
@@ -221,7 +219,6 @@ class ImgurAlbumExtractor(ImgurExtractor):
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
yield Message.Version, 1
|
||||
for num, image in enumerate(images, 1):
|
||||
url = self._prepare(image)
|
||||
image["num"] = num
|
||||
|
||||
@@ -94,7 +94,6 @@ class IssuuUserExtractor(IssuuBase, Extractor):
|
||||
url = "{}/call/profile/v1/documents/{}".format(self.root, self.user)
|
||||
params = {"offset": 0, "limit": "25"}
|
||||
|
||||
yield Message.Version, 1
|
||||
while True:
|
||||
data = self.request(url, params=params).json()
|
||||
|
||||
|
||||
@@ -55,7 +55,6 @@ class KeenspotComicExtractor(Extractor):
|
||||
|
||||
def items(self):
|
||||
data = {"comic": self.comic}
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
|
||||
with self.request(self.root + "/") as response:
|
||||
|
||||
@@ -40,7 +40,6 @@ class KhinsiderSoundtrackExtractor(AsynchronousMixin, Extractor):
|
||||
raise exception.NotFoundError("soundtrack")
|
||||
|
||||
data = self.metadata(page)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for track in self.tracks(page):
|
||||
track.update(data)
|
||||
|
||||
@@ -25,7 +25,6 @@ class LivedoorExtractor(Extractor):
|
||||
self.user = match.group(1)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
for post in self.posts():
|
||||
images = self._images(post)
|
||||
if images:
|
||||
|
||||
@@ -120,7 +120,6 @@ class LusciousAlbumExtractor(LusciousExtractor):
|
||||
|
||||
def items(self):
|
||||
album = self.metadata()
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, {"album": album}
|
||||
for num, image in enumerate(self.images(), 1):
|
||||
image["num"] = num
|
||||
@@ -443,7 +442,6 @@ fragment AlbumMinimal on Album {
|
||||
}
|
||||
}
|
||||
"""
|
||||
yield Message.Version, 1
|
||||
while True:
|
||||
data = self._graphql("AlbumListWithPeek", variables, query)
|
||||
|
||||
|
||||
@@ -106,7 +106,6 @@ class MangoxoAlbumExtractor(MangoxoExtractor):
|
||||
data = self.metadata(page)
|
||||
imgs = self.images(url, page)
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
|
||||
data["extension"] = None
|
||||
@@ -175,8 +174,6 @@ class MangoxoChannelExtractor(MangoxoExtractor):
|
||||
url = "{}/channel/{}/album/".format(self.root, self.channel_id)
|
||||
data = {"_extractor": MangoxoAlbumExtractor}
|
||||
|
||||
yield Message.Version, 1
|
||||
|
||||
while True:
|
||||
page = self.request(url + str(num)).text
|
||||
|
||||
|
||||
@@ -102,7 +102,6 @@ class NaverBlogExtractor(NaverBase, Extractor):
|
||||
self.blog_id = match.group(1) or match.group(2)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
|
||||
# fetch first post number
|
||||
url = "{}/PostList.nhn?blogId={}".format(self.root, self.blog_id)
|
||||
|
||||
@@ -117,7 +117,6 @@ class NhentaiSearchExtractor(NhentaiBase, Extractor):
|
||||
self.params = text.parse_query(match.group(1))
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
data = {"_extractor": NhentaiGalleryExtractor}
|
||||
for gallery_id in self._pagination(self.params):
|
||||
url = "{}/g/{}/".format(self.root, gallery_id)
|
||||
|
||||
@@ -36,7 +36,6 @@ class NijieExtractor(AsynchronousMixin, Extractor):
|
||||
|
||||
def items(self):
|
||||
self.login()
|
||||
yield Message.Version, 1
|
||||
|
||||
for image_id in self.image_ids():
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ class NozomiExtractor(Extractor):
|
||||
archive_fmt = "{dataid}"
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
|
||||
data = self.metadata()
|
||||
self.session.headers["Origin"] = self.root
|
||||
|
||||
@@ -27,7 +27,6 @@ class PatreonExtractor(Extractor):
|
||||
_warning = True
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
|
||||
if self._warning:
|
||||
if "session_id" not in self.session.cookies:
|
||||
|
||||
@@ -54,7 +54,6 @@ class PhotobucketAlbumExtractor(Extractor):
|
||||
self.session.headers["Referer"] = self.url
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
for image in self.images():
|
||||
image["titleOrFilename"] = text.unescape(image["titleOrFilename"])
|
||||
image["title"] = text.unescape(image["title"])
|
||||
@@ -172,6 +171,5 @@ class PhotobucketImageExtractor(Extractor):
|
||||
mtype, _, mid = base64.b64decode(image["id"]).partition(b":")
|
||||
image["pictureId"] = mid.decode() if mtype == b"mediaId" else ""
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, image
|
||||
yield Message.Url, image["fileUrl"], image
|
||||
|
||||
@@ -22,7 +22,6 @@ class PiczelExtractor(Extractor):
|
||||
api_root = "https://tombstone.piczel.tv"
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
for post in self.posts():
|
||||
post["tags"] = [t["title"] for t in post["tags"] if t["title"]]
|
||||
post["date"] = text.parse_datetime(
|
||||
|
||||
@@ -213,7 +213,6 @@ class PixivMeExtractor(PixivExtractor):
|
||||
data = {"_extractor": PixivUserExtractor}
|
||||
response = self.request(
|
||||
url, method="HEAD", allow_redirects=False, notfound="user")
|
||||
yield Message.Version, 1
|
||||
yield Message.Queue, response.headers["Location"], data
|
||||
|
||||
|
||||
|
||||
@@ -89,7 +89,6 @@ class PixnetImageExtractor(PixnetExtractor):
|
||||
data["blog"] = self.blog
|
||||
data["user"] = data.pop("author_name")
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, data["url"], data
|
||||
|
||||
@@ -120,7 +119,6 @@ class PixnetSetExtractor(PixnetExtractor):
|
||||
page = self.request(url, encoding="utf-8").text
|
||||
data = self.metadata(page)
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for num, info in enumerate(self._pagination(page), 1):
|
||||
url, pos = text.extract(info, ' href="', '"')
|
||||
|
||||
@@ -60,7 +60,6 @@ class PornhubGalleryExtractor(PornhubExtractor):
|
||||
|
||||
def items(self):
|
||||
data = self.metadata()
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for num, image in enumerate(self.images(), 1):
|
||||
url = image["url"]
|
||||
@@ -146,7 +145,6 @@ class PornhubUserExtractor(PornhubExtractor):
|
||||
}
|
||||
|
||||
data = {"_extractor": PornhubGalleryExtractor}
|
||||
yield Message.Version, 1
|
||||
while True:
|
||||
page = self.request(
|
||||
url, method="POST", headers=headers, params=params).text
|
||||
|
||||
@@ -37,7 +37,6 @@ class ReactorExtractor(Extractor):
|
||||
|
||||
def items(self):
|
||||
data = self.metadata()
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for post in self.posts():
|
||||
for image in self._parse_post(post):
|
||||
@@ -197,7 +196,6 @@ class ReactorPostExtractor(ReactorExtractor):
|
||||
self.post_id = match.group(2)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
post = self.request(self.url).text
|
||||
pos = post.find('class="uhead">')
|
||||
for image in self._parse_post(post[pos:]):
|
||||
|
||||
@@ -38,8 +38,6 @@ class RedditExtractor(Extractor):
|
||||
visited = set()
|
||||
depth = 0
|
||||
|
||||
yield Message.Version, 1
|
||||
|
||||
while True:
|
||||
extra = []
|
||||
|
||||
@@ -257,7 +255,6 @@ class RedditImageExtractor(Extractor):
|
||||
|
||||
def items(self):
|
||||
data = text.nameext_from_url(self.url)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, self.url, data
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ class SeigaExtractor(Extractor):
|
||||
images = iter(self.get_images())
|
||||
data = next(images)
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for image in util.advance(images, self.start_image):
|
||||
data.update(image)
|
||||
|
||||
@@ -42,7 +42,6 @@ class SenmangaChapterExtractor(Extractor):
|
||||
|
||||
def items(self):
|
||||
data = self.metadata()
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for data["page"] in range(1, data["count"]+1):
|
||||
data["extension"] = None
|
||||
|
||||
@@ -21,7 +21,6 @@ class SexcomExtractor(Extractor):
|
||||
root = "https://www.sex.com"
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, self.metadata()
|
||||
for pin in map(self._parse_pin, self.pins()):
|
||||
if pin:
|
||||
|
||||
@@ -128,7 +128,6 @@ class SimplyhentaiImageExtractor(Extractor):
|
||||
})
|
||||
data["token"] = data["filename"].rpartition("_")[2]
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, url, data
|
||||
|
||||
@@ -192,6 +191,5 @@ class SimplyhentaiVideoExtractor(Extractor):
|
||||
date), "%B %d, %Y %H:%M"),
|
||||
})
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, video_url, data
|
||||
|
||||
@@ -68,7 +68,6 @@ class SlickpicAlbumExtractor(SlickpicExtractor):
|
||||
"count": len(imgs),
|
||||
}
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for num, img in enumerate(imgs, 1):
|
||||
url = img["url_rsz"] + "/o/" + img["fname"]
|
||||
@@ -137,6 +136,5 @@ class SlickpicUserExtractor(SlickpicExtractor):
|
||||
data = {"_extractor": SlickpicAlbumExtractor}
|
||||
base = self.root + "/albums/"
|
||||
|
||||
yield Message.Version, 1
|
||||
for album in text.extract_iter(page, 'href="' + base, '"'):
|
||||
yield Message.Queue, base + album, data
|
||||
|
||||
@@ -49,7 +49,6 @@ class SlidesharePresentationExtractor(Extractor):
|
||||
data = self.get_job_metadata(page)
|
||||
imgs = self.get_image_urls(page)
|
||||
data["count"] = len(imgs)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for data["num"], url in enumerate(imgs, 1):
|
||||
yield Message.Url, url, text.nameext_from_url(url, data)
|
||||
|
||||
@@ -95,7 +95,6 @@ class SmugmugAlbumExtractor(SmugmugExtractor):
|
||||
del album["Uris"]
|
||||
data = {"Album": album, "User": user}
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
|
||||
for image in self.api.album_images(self.album_id, "ImageSizeDetails"):
|
||||
@@ -133,7 +132,6 @@ class SmugmugImageExtractor(SmugmugExtractor):
|
||||
data = {"Image": image}
|
||||
text.nameext_from_url(url, data)
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, url, data
|
||||
|
||||
@@ -170,7 +168,6 @@ class SmugmugPathExtractor(SmugmugExtractor):
|
||||
self.domain, self.user, self.path = match.groups()
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
|
||||
if not self.user:
|
||||
self.user = self.api.site_user(self.domain)["NickName"]
|
||||
|
||||
@@ -39,7 +39,6 @@ class SpeakerdeckPresentationExtractor(Extractor):
|
||||
data = self.get_job_metadata()
|
||||
imgs = self.get_image_urls()
|
||||
data["count"] = len(imgs)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for data["num"], url in enumerate(imgs, 1):
|
||||
yield Message.Url, url, text.nameext_from_url(url, data)
|
||||
|
||||
@@ -71,7 +71,6 @@ class TestExtractor(Extractor):
|
||||
if not tests:
|
||||
raise exception.NotFoundError("test")
|
||||
|
||||
yield Message.Version, 1
|
||||
for test in tests:
|
||||
yield Message.Queue, test[0], {}
|
||||
|
||||
|
||||
@@ -152,7 +152,6 @@ class TsuminoSearchExtractor(TsuminoBase, Extractor):
|
||||
self.query = match.group(1)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
for gallery in self.galleries():
|
||||
url = "{}/entry/{}".format(self.root, gallery["id"])
|
||||
gallery["_extractor"] = TsuminoGalleryExtractor
|
||||
|
||||
@@ -56,7 +56,6 @@ class VanillarockPostExtractor(VanillarockExtractor):
|
||||
'<div class="cat-tag">', '</div>'))[::2],
|
||||
}
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for data["num"], url in enumerate(imgs, 1):
|
||||
yield Message.Url, url, text.nameext_from_url(url, data)
|
||||
@@ -84,7 +83,6 @@ class VanillarockTagExtractor(VanillarockExtractor):
|
||||
url = self.root + self.path
|
||||
data = {"_extractor": VanillarockPostExtractor}
|
||||
|
||||
yield Message.Version, 1
|
||||
while url:
|
||||
extr = text.extract_from(self.request(url).text)
|
||||
while True:
|
||||
|
||||
@@ -30,7 +30,6 @@ class VscoExtractor(Extractor):
|
||||
|
||||
def items(self):
|
||||
videos = self.config("videos", True)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, {"user": self.user}
|
||||
for img in self.images():
|
||||
|
||||
|
||||
@@ -47,7 +47,6 @@ class WarosuThreadExtractor(Extractor):
|
||||
title = text.remove_html(posts[0]["com"])
|
||||
data["title"] = text.unescape(title)[:50]
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for post in posts:
|
||||
if "image" in post:
|
||||
|
||||
@@ -123,7 +123,6 @@ class WeasylSubmissionsExtractor(WeasylExtractor):
|
||||
self.owner_login = match.group(1)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, {"owner_login": self.owner_login}
|
||||
yield from self.submissions(self.owner_login)
|
||||
|
||||
@@ -141,7 +140,6 @@ class WeasylFolderExtractor(WeasylExtractor):
|
||||
self.owner_login, self.folderid = match.groups()
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
iter = self.submissions(self.owner_login, self.folderid)
|
||||
# Folder names are only on single submission api calls
|
||||
msg, url, data = next(iter)
|
||||
@@ -171,7 +169,6 @@ class WeasylJournalExtractor(WeasylExtractor):
|
||||
|
||||
def items(self):
|
||||
data = self.retrieve_journal(self.journalid)
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
yield Message.Url, data["html"], data
|
||||
|
||||
@@ -190,7 +187,6 @@ class WeasylJournalsExtractor(WeasylExtractor):
|
||||
self.owner_login = match.group(1)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, {"owner_login": self.owner_login}
|
||||
|
||||
url = "{}/journals/{}".format(self.root, self.owner_login)
|
||||
|
||||
@@ -27,7 +27,6 @@ class WikiartExtractor(Extractor):
|
||||
|
||||
def items(self):
|
||||
data = self.metadata()
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for painting in self.paintings():
|
||||
url = painting["image"]
|
||||
|
||||
@@ -43,7 +43,6 @@ class WikieatThreadExtractor(Extractor):
|
||||
"num" : 0,
|
||||
}
|
||||
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for post in posts:
|
||||
if "filename" in post:
|
||||
|
||||
@@ -90,7 +90,6 @@ class XhamsterGalleryExtractor(XhamsterExtractor):
|
||||
|
||||
def items(self):
|
||||
data = self.metadata()
|
||||
yield Message.Version, 1
|
||||
yield Message.Directory, data
|
||||
for num, image in enumerate(self.images(), 1):
|
||||
url = image["imageURL"]
|
||||
@@ -167,7 +166,6 @@ class XhamsterUserExtractor(XhamsterExtractor):
|
||||
self.user = match.group(2)
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
url = "{}/users/{}/photos".format(self.root, self.user)
|
||||
data = {"_extractor": XhamsterGalleryExtractor}
|
||||
|
||||
|
||||
@@ -129,7 +129,6 @@ class XvideosUserExtractor(XvideosBase, Extractor):
|
||||
]
|
||||
galleries.sort(key=lambda x: x["id"])
|
||||
|
||||
yield Message.Version, 1
|
||||
for gallery in galleries:
|
||||
url = "https://www.xvideos.com/profiles/{}/photos/{}".format(
|
||||
self.user, gallery["id"])
|
||||
|
||||
@@ -134,13 +134,6 @@ class Job():
|
||||
if self.pred_queue(url, kwdict):
|
||||
self.handle_queue(url, kwdict)
|
||||
|
||||
elif msg[0] == Message.Version:
|
||||
if msg[1] != 1:
|
||||
raise "unsupported message-version ({}, {})".format(
|
||||
self.extractor.category, msg[1]
|
||||
)
|
||||
# TODO: support for multiple message versions
|
||||
|
||||
def handle_url(self, url, kwdict):
|
||||
"""Handle Message.Url"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user