[yaplog] update metadata structure (#190)

Put all blog post related fields in its own dict.

'image_id' -> 'id'
'post_id'  -> 'post[id]'
'title'    -> 'post[title]'
etc ...
This commit is contained in:
Mike Fährmann
2019-04-06 16:40:07 +02:00
parent 35919a9bb8
commit 2c32dc76cb

View File

@@ -16,9 +16,9 @@ class YaplogExtractor(AsynchronousMixin, Extractor):
"""Base class for yaplog extractors"""
category = "yaplog"
root = "https://yaplog.jp"
filename_fmt = "{post_id}_{image_id}_{title}.{extension}"
directory_fmt = ("{category}", "{user}")
archive_fmt = "{user}_{image_id}"
filename_fmt = "{post[id]}_{post[title]}_{id}.{extension}"
directory_fmt = ("{category}", "{post[user]}")
archive_fmt = "{post[user]}_{id}"
def __init__(self, match):
Extractor.__init__(self, match)
@@ -26,8 +26,8 @@ class YaplogExtractor(AsynchronousMixin, Extractor):
def items(self):
yield Message.Version, 1
for urls, data in self.posts():
yield Message.Directory, data
for post, urls in self.posts():
yield Message.Directory, {"post": post}
for num, url in enumerate(urls, 1):
page = self.request(url).text if num > 1 else url
iurl = text.extract(page, '<img src="', '"')[0]
@@ -35,14 +35,14 @@ class YaplogExtractor(AsynchronousMixin, Extractor):
image = {
"url" : iurl,
"num" : num,
"image_id" : text.parse_int(iid.partition("_")[0]),
"id" : text.parse_int(iid.partition("_")[0]),
"extension": ext,
"post" : post,
}
image.update(data)
yield Message.Url, iurl, image
def posts(self):
"""Return an iterable with (data, image URLs) tuples"""
"""Return an iterable with (data, image page URLs) tuples"""
def _parse_post(self, url):
page = self.request(url).text
@@ -66,16 +66,16 @@ class YaplogExtractor(AsynchronousMixin, Extractor):
urls.append(base + part)
return prev, urls, {
"post_id": text.parse_int(pid),
"title" : text.unescape(title[:-3]),
"user" : self.user,
"date" : date,
"id" : text.parse_int(pid),
"title": text.unescape(title[:-3]),
"user" : self.user,
"date" : date,
}
class YaplogUserExtractor(YaplogExtractor):
class YaplogBlogExtractor(YaplogExtractor):
"""Extractor for a user's blog on yaplog.jp"""
subcategory = "user"
subcategory = "blog"
pattern = r"(?:https?://)?(?:www\.)?yaplog\.jp/(\w+)/?(?:$|[?&#])"
test = ("https://yaplog.jp/omitakashi3", {
"pattern": r"https://img.yaplog.jp/img/18/pc/o/m/i/omitakashi3/0/",
@@ -86,7 +86,7 @@ class YaplogUserExtractor(YaplogExtractor):
url = "{}/{}/image/".format(self.root, self.user)
while url:
url, images, data = self._parse_post(url)
yield images, data
yield data, images
class YaplogPostExtractor(YaplogExtractor):
@@ -96,7 +96,7 @@ class YaplogPostExtractor(YaplogExtractor):
r"/(\w+)/(?:archive|image)/(\d+)")
test = ("https://yaplog.jp/imamiami0726/image/1299", {
"url": "896cae20fa718735a57e723c48544e830ff31345",
"keyword": "9bb89e959bb518a8797242aa3e03e17caa00beba",
"keyword": "f8d8781e61c4c38238a7622d6df6c905f864e5d3",
})
def __init__(self, match):
@@ -106,4 +106,4 @@ class YaplogPostExtractor(YaplogExtractor):
def posts(self):
url = "{}/{}/image/{}".format(self.root, self.user, self.post_id)
_, images, data = self._parse_post(url)
return ((images, data),)
return ((data, images),)