remove explicit (sub)category keywords

This commit is contained in:
Mike Fährmann
2016-09-25 14:22:07 +02:00
parent a347d50ef5
commit 19c2d4ff6f
43 changed files with 26 additions and 104 deletions

View File

@@ -44,17 +44,12 @@ class ImagebamGalleryExtractor(AsynchronousExtractor):
"""Collect metadata for extractor-job"""
url = self.url_base + "/gallery/" + self.gkey
page = self.request(url, encoding="utf-8").text
data = {
"category": self.category,
"gallery-key": self.gkey,
}
data, _ = text.extract_all(page, (
return text.extract_all(page, (
(None , "<img src='/img/icons/photos.png'", ""),
("title" , "'> ", " <"),
("count" , "'>", " images"),
("first-url", "<a href='http://www.imagebam.com", "'"),
), values=data)
return data
), values={"gallery-key": self.gkey})[0]
def get_images(self, url):
"""Yield all image-urls and -ids for a gallery"""
@@ -71,7 +66,6 @@ class ImagebamGalleryExtractor(AsynchronousExtractor):
yield image_url, image_id
class ImagebamImageExtractor(Extractor):
"""Extractor for single images from imagebam.com"""
category = "imagebam"
@@ -90,10 +84,9 @@ class ImagebamImageExtractor(Extractor):
self.token = match.group(1)
def items(self):
data = {"category": self.category, "token": self.token}
page = self.request("http://www.imagebam.com/image/" + self.token).text
url = text.extract(page, 'property="og:image" content="', '"')[0]
text.nameext_from_url(url, data)
url = text.extract(page, 'property="og:image" content="', '"')[0]
data = text.nameext_from_url(url, {"token": self.token})
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, url, data