[chevereto:album] extract 'count' & 'num' metadata (#8604)
This commit is contained in:
@@ -158,21 +158,25 @@ class CheveretoAlbumExtractor(CheveretoExtractor):
|
||||
else:
|
||||
albums = (url,)
|
||||
|
||||
kwdict = self.kwdict
|
||||
for album in albums:
|
||||
for item_url in self._pagination(
|
||||
album, self._extract_metadata_album):
|
||||
for kwdict["num"], item_url in enumerate(self._pagination(
|
||||
album, self._extract_metadata_album), 1):
|
||||
data = data_video if "/video/" in item_url else data_image
|
||||
yield Message.Queue, item_url, data
|
||||
|
||||
def _extract_metadata_album(self, page):
|
||||
url, pos = text.extract(
|
||||
page, 'property="og:url" content="', '"')
|
||||
title, pos = text.extract(
|
||||
page, 'property="og:title" content="', '"', pos)
|
||||
|
||||
kwdict = self.kwdict
|
||||
kwdict["album_slug"], _, kwdict["album_id"] = \
|
||||
url[url.rfind("/")+1:].rpartition(".")
|
||||
kwdict["album"] = text.unescape(text.extract(
|
||||
page, 'property="og:title" content="', '"', pos)[0])
|
||||
kwdict["album"] = text.unescape(title)
|
||||
kwdict["count"] = text.parse_int(text.extract(
|
||||
page, 'data-text="image-count">', "<", pos)[0])
|
||||
|
||||
|
||||
class CheveretoCategoryExtractor(CheveretoExtractor):
|
||||
|
||||
@@ -122,6 +122,8 @@ __tests__ = (
|
||||
"album" : "funny meme album",
|
||||
"album_id" : "CDilP",
|
||||
"album_slug": "funny-meme-album",
|
||||
"count" : 2,
|
||||
"num" : range(1, 2),
|
||||
},
|
||||
|
||||
{
|
||||
@@ -133,6 +135,8 @@ __tests__ = (
|
||||
"album" : "Gunggingnsk OF",
|
||||
"album_id" : "N9OOI",
|
||||
"album_slug": "gunggingnsk",
|
||||
"count" : 114,
|
||||
"num" : range(1, 114),
|
||||
},
|
||||
|
||||
{
|
||||
@@ -144,6 +148,8 @@ __tests__ = (
|
||||
"album" : "101-200",
|
||||
"album_id" : "aNJ6A",
|
||||
"album_slug": "101-200",
|
||||
"count" : 100,
|
||||
"num" : range(1, 100),
|
||||
},
|
||||
|
||||
{
|
||||
@@ -152,9 +158,11 @@ __tests__ = (
|
||||
"#class" : chevereto.CheveretoAlbumExtractor,
|
||||
"#count" : 606,
|
||||
|
||||
"album" : "re:([12345]0)?1-[123456]00",
|
||||
"album" : "re:([1-5]0)?1-[1-6]00",
|
||||
"album_id" : str,
|
||||
"album_slug": str,
|
||||
"count" : {100, 106},
|
||||
"num" : range(1, 106),
|
||||
},
|
||||
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user