[patreon] small fixes and adjustments (#226)
- fix datetime parsing - rename 'user' to 'creator' - convert 'id' to integer - improve tests
This commit is contained in:
@@ -17,7 +17,7 @@ class PatreonExtractor(Extractor):
|
|||||||
"""Base class for patreon extractors"""
|
"""Base class for patreon extractors"""
|
||||||
category = "patreon"
|
category = "patreon"
|
||||||
root = "https://www.patreon.com"
|
root = "https://www.patreon.com"
|
||||||
directory_fmt = ("{category}", "{user[full_name]}")
|
directory_fmt = ("{category}", "{creator[full_name]}")
|
||||||
filename_fmt = "{id}_{title}_{num:>02}.{extension}"
|
filename_fmt = "{id}_{title}_{num:>02}.{extension}"
|
||||||
archive_fmt = "{id}_{num}"
|
archive_fmt = "{id}_{num}"
|
||||||
_warning = True
|
_warning = True
|
||||||
@@ -25,8 +25,9 @@ class PatreonExtractor(Extractor):
|
|||||||
def items(self):
|
def items(self):
|
||||||
yield Message.Version, 1
|
yield Message.Version, 1
|
||||||
|
|
||||||
if self._warning and "session_id" not in self.session.cookies:
|
if self._warning:
|
||||||
self.log.warning("no 'session_id' cookie set")
|
if "session_id" not in self.session.cookies:
|
||||||
|
self.log.warning("no 'session_id' cookie set")
|
||||||
PatreonExtractor._warning = False
|
PatreonExtractor._warning = False
|
||||||
|
|
||||||
for post in self.posts():
|
for post in self.posts():
|
||||||
@@ -73,9 +74,10 @@ class PatreonExtractor(Extractor):
|
|||||||
# update posts
|
# update posts
|
||||||
for post in posts["data"]:
|
for post in posts["data"]:
|
||||||
attr = post["attributes"]
|
attr = post["attributes"]
|
||||||
attr["id"] = post["id"]
|
attr["id"] = text.parse_int(post["id"])
|
||||||
attr["date"] = text.parse_timestamp(attr["published_at"])
|
attr["date"] = text.parse_datetime(
|
||||||
attr["user"] = self._user(
|
attr["published_at"], "%Y-%m-%dT%H:%M:%S.%f%z")
|
||||||
|
attr["creator"] = self._user(
|
||||||
post["relationships"]["user"]["links"]["related"])
|
post["relationships"]["user"]["links"]["related"])
|
||||||
|
|
||||||
# add attachments to post attributes
|
# add attachments to post attributes
|
||||||
@@ -99,6 +101,8 @@ class PatreonExtractor(Extractor):
|
|||||||
user = self.request(url).json()["data"]
|
user = self.request(url).json()["data"]
|
||||||
attr = user["attributes"]
|
attr = user["attributes"]
|
||||||
attr["id"] = user["id"]
|
attr["id"] = user["id"]
|
||||||
|
attr["date"] = text.parse_datetime(
|
||||||
|
attr["created"], "%Y-%m-%dT%H:%M:%S.%f%z")
|
||||||
return attr
|
return attr
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -134,6 +138,18 @@ class PatreonCreatorExtractor(PatreonExtractor):
|
|||||||
test = ("https://www.patreon.com/koveliana", {
|
test = ("https://www.patreon.com/koveliana", {
|
||||||
"range": "1-25",
|
"range": "1-25",
|
||||||
"count": ">= 25",
|
"count": ">= 25",
|
||||||
|
"keyword": {
|
||||||
|
"attachments": list,
|
||||||
|
"comment_count": int,
|
||||||
|
"content": str,
|
||||||
|
"creator": dict,
|
||||||
|
"date": "type:datetime",
|
||||||
|
"id": int,
|
||||||
|
"like_count": int,
|
||||||
|
"post_type": str,
|
||||||
|
"published_at": str,
|
||||||
|
"title": str,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
def __init__(self, match):
|
def __init__(self, match):
|
||||||
|
|||||||
Reference in New Issue
Block a user