[patreon] small fixes and adjustments (#226)

- fix datetime parsing
- rename 'user' to 'creator'
- convert 'id' to integer
- improve tests
This commit is contained in:
Mike Fährmann
2019-05-17 13:13:11 +02:00
parent fb09dd962a
commit ca3bad1779

View File

@@ -17,7 +17,7 @@ class PatreonExtractor(Extractor):
"""Base class for patreon extractors""" """Base class for patreon extractors"""
category = "patreon" category = "patreon"
root = "https://www.patreon.com" root = "https://www.patreon.com"
directory_fmt = ("{category}", "{user[full_name]}") directory_fmt = ("{category}", "{creator[full_name]}")
filename_fmt = "{id}_{title}_{num:>02}.{extension}" filename_fmt = "{id}_{title}_{num:>02}.{extension}"
archive_fmt = "{id}_{num}" archive_fmt = "{id}_{num}"
_warning = True _warning = True
@@ -25,8 +25,9 @@ class PatreonExtractor(Extractor):
def items(self): def items(self):
yield Message.Version, 1 yield Message.Version, 1
if self._warning and "session_id" not in self.session.cookies: if self._warning:
self.log.warning("no 'session_id' cookie set") if "session_id" not in self.session.cookies:
self.log.warning("no 'session_id' cookie set")
PatreonExtractor._warning = False PatreonExtractor._warning = False
for post in self.posts(): for post in self.posts():
@@ -73,9 +74,10 @@ class PatreonExtractor(Extractor):
# update posts # update posts
for post in posts["data"]: for post in posts["data"]:
attr = post["attributes"] attr = post["attributes"]
attr["id"] = post["id"] attr["id"] = text.parse_int(post["id"])
attr["date"] = text.parse_timestamp(attr["published_at"]) attr["date"] = text.parse_datetime(
attr["user"] = self._user( attr["published_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
attr["creator"] = self._user(
post["relationships"]["user"]["links"]["related"]) post["relationships"]["user"]["links"]["related"])
# add attachments to post attributes # add attachments to post attributes
@@ -99,6 +101,8 @@ class PatreonExtractor(Extractor):
user = self.request(url).json()["data"] user = self.request(url).json()["data"]
attr = user["attributes"] attr = user["attributes"]
attr["id"] = user["id"] attr["id"] = user["id"]
attr["date"] = text.parse_datetime(
attr["created"], "%Y-%m-%dT%H:%M:%S.%f%z")
return attr return attr
@staticmethod @staticmethod
@@ -134,6 +138,18 @@ class PatreonCreatorExtractor(PatreonExtractor):
test = ("https://www.patreon.com/koveliana", { test = ("https://www.patreon.com/koveliana", {
"range": "1-25", "range": "1-25",
"count": ">= 25", "count": ">= 25",
"keyword": {
"attachments": list,
"comment_count": int,
"content": str,
"creator": dict,
"date": "type:datetime",
"id": int,
"like_count": int,
"post_type": str,
"published_at": str,
"title": str,
},
}) })
def __init__(self, match): def __init__(self, match):