[yaplog] update metadata structure (#190)
Put all blog post related fields in its own dict. 'image_id' -> 'id' 'post_id' -> 'post[id]' 'title' -> 'post[title]' etc ...
This commit is contained in:
@@ -16,9 +16,9 @@ class YaplogExtractor(AsynchronousMixin, Extractor):
|
||||
"""Base class for yaplog extractors"""
|
||||
category = "yaplog"
|
||||
root = "https://yaplog.jp"
|
||||
filename_fmt = "{post_id}_{image_id}_{title}.{extension}"
|
||||
directory_fmt = ("{category}", "{user}")
|
||||
archive_fmt = "{user}_{image_id}"
|
||||
filename_fmt = "{post[id]}_{post[title]}_{id}.{extension}"
|
||||
directory_fmt = ("{category}", "{post[user]}")
|
||||
archive_fmt = "{post[user]}_{id}"
|
||||
|
||||
def __init__(self, match):
|
||||
Extractor.__init__(self, match)
|
||||
@@ -26,8 +26,8 @@ class YaplogExtractor(AsynchronousMixin, Extractor):
|
||||
|
||||
def items(self):
|
||||
yield Message.Version, 1
|
||||
for urls, data in self.posts():
|
||||
yield Message.Directory, data
|
||||
for post, urls in self.posts():
|
||||
yield Message.Directory, {"post": post}
|
||||
for num, url in enumerate(urls, 1):
|
||||
page = self.request(url).text if num > 1 else url
|
||||
iurl = text.extract(page, '<img src="', '"')[0]
|
||||
@@ -35,14 +35,14 @@ class YaplogExtractor(AsynchronousMixin, Extractor):
|
||||
image = {
|
||||
"url" : iurl,
|
||||
"num" : num,
|
||||
"image_id" : text.parse_int(iid.partition("_")[0]),
|
||||
"id" : text.parse_int(iid.partition("_")[0]),
|
||||
"extension": ext,
|
||||
"post" : post,
|
||||
}
|
||||
image.update(data)
|
||||
yield Message.Url, iurl, image
|
||||
|
||||
def posts(self):
|
||||
"""Return an iterable with (data, image URLs) tuples"""
|
||||
"""Return an iterable with (data, image page URLs) tuples"""
|
||||
|
||||
def _parse_post(self, url):
|
||||
page = self.request(url).text
|
||||
@@ -66,16 +66,16 @@ class YaplogExtractor(AsynchronousMixin, Extractor):
|
||||
urls.append(base + part)
|
||||
|
||||
return prev, urls, {
|
||||
"post_id": text.parse_int(pid),
|
||||
"title" : text.unescape(title[:-3]),
|
||||
"user" : self.user,
|
||||
"date" : date,
|
||||
"id" : text.parse_int(pid),
|
||||
"title": text.unescape(title[:-3]),
|
||||
"user" : self.user,
|
||||
"date" : date,
|
||||
}
|
||||
|
||||
|
||||
class YaplogUserExtractor(YaplogExtractor):
|
||||
class YaplogBlogExtractor(YaplogExtractor):
|
||||
"""Extractor for a user's blog on yaplog.jp"""
|
||||
subcategory = "user"
|
||||
subcategory = "blog"
|
||||
pattern = r"(?:https?://)?(?:www\.)?yaplog\.jp/(\w+)/?(?:$|[?&#])"
|
||||
test = ("https://yaplog.jp/omitakashi3", {
|
||||
"pattern": r"https://img.yaplog.jp/img/18/pc/o/m/i/omitakashi3/0/",
|
||||
@@ -86,7 +86,7 @@ class YaplogUserExtractor(YaplogExtractor):
|
||||
url = "{}/{}/image/".format(self.root, self.user)
|
||||
while url:
|
||||
url, images, data = self._parse_post(url)
|
||||
yield images, data
|
||||
yield data, images
|
||||
|
||||
|
||||
class YaplogPostExtractor(YaplogExtractor):
|
||||
@@ -96,7 +96,7 @@ class YaplogPostExtractor(YaplogExtractor):
|
||||
r"/(\w+)/(?:archive|image)/(\d+)")
|
||||
test = ("https://yaplog.jp/imamiami0726/image/1299", {
|
||||
"url": "896cae20fa718735a57e723c48544e830ff31345",
|
||||
"keyword": "9bb89e959bb518a8797242aa3e03e17caa00beba",
|
||||
"keyword": "f8d8781e61c4c38238a7622d6df6c905f864e5d3",
|
||||
})
|
||||
|
||||
def __init__(self, match):
|
||||
@@ -106,4 +106,4 @@ class YaplogPostExtractor(YaplogExtractor):
|
||||
def posts(self):
|
||||
url = "{}/{}/image/{}".format(self.root, self.user, self.post_id)
|
||||
_, images, data = self._parse_post(url)
|
||||
return ((images, data),)
|
||||
return ((data, images),)
|
||||
|
||||
Reference in New Issue
Block a user