[cohost] add 'avatar' and 'background' options (#6656)
This commit is contained in:
@@ -1755,6 +1755,26 @@ Description
|
||||
Extract ``ask`` posts.
|
||||
|
||||
|
||||
extractor.cohost.avatar
|
||||
-----------------------
|
||||
Type
|
||||
``bool``
|
||||
Default
|
||||
``false``
|
||||
Description
|
||||
Download ``avatar`` images.
|
||||
|
||||
|
||||
extractor.cohost.background
|
||||
---------------------------
|
||||
Type
|
||||
``bool``
|
||||
Default
|
||||
``false``
|
||||
Description
|
||||
Download ``background``/``banner``/``header`` images.
|
||||
|
||||
|
||||
extractor.cohost.pinned
|
||||
-----------------------
|
||||
Type
|
||||
|
||||
@@ -176,6 +176,8 @@
|
||||
"cohost":
|
||||
{
|
||||
"asks" : true,
|
||||
"avatar" : false,
|
||||
"background": false,
|
||||
"pinned" : false,
|
||||
"replies": true,
|
||||
"shares" : true
|
||||
|
||||
@@ -19,7 +19,7 @@ class CohostExtractor(Extractor):
|
||||
category = "cohost"
|
||||
root = "https://cohost.org"
|
||||
directory_fmt = ("{category}", "{postingProject[handle]}")
|
||||
filename_fmt = ("{postId}_{headline:?/_/[b:200]}{num}.{extension}")
|
||||
filename_fmt = ("{postId}{headline:?_//[b:200]}{num:?_//}.{extension}")
|
||||
archive_fmt = "{postId}_{num}"
|
||||
|
||||
def _init(self):
|
||||
@@ -28,6 +28,14 @@ class CohostExtractor(Extractor):
|
||||
self.shares = self.config("shares", False)
|
||||
self.asks = self.config("asks", True)
|
||||
|
||||
self.avatar = self.config("avatar", False)
|
||||
if self.avatar:
|
||||
self._urls_avatar = {None, ""}
|
||||
|
||||
self.background = self.config("background", False)
|
||||
if self.background:
|
||||
self._urls_background = {None, ""}
|
||||
|
||||
def items(self):
|
||||
for post in self.posts():
|
||||
reason = post.get("limitedVisibilityReason")
|
||||
@@ -43,6 +51,26 @@ class CohostExtractor(Extractor):
|
||||
post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
yield Message.Directory, post
|
||||
|
||||
project = post["postingProject"]
|
||||
if self.avatar:
|
||||
url = project.get("avatarURL")
|
||||
if url not in self._urls_avatar:
|
||||
self._urls_avatar.add(url)
|
||||
p = post.copy()
|
||||
p["postId"] = p["kind"] = "avatar"
|
||||
p["headline"] = p["num"] = ""
|
||||
yield Message.Url, url, text.nameext_from_url(url, p)
|
||||
|
||||
if self.background:
|
||||
url = project.get("headerURL")
|
||||
if url not in self._urls_background:
|
||||
self._urls_background.add(url)
|
||||
p = post.copy()
|
||||
p["postId"] = p["kind"] = "background"
|
||||
p["headline"] = p["num"] = ""
|
||||
yield Message.Url, url, text.nameext_from_url(url, p)
|
||||
|
||||
for post["num"], file in enumerate(files, 1):
|
||||
url = file["fileURL"]
|
||||
post.update(file)
|
||||
|
||||
@@ -16,6 +16,18 @@ __tests__ = (
|
||||
"#count" : 20,
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://cohost.org/infinitebrians",
|
||||
"#category": ("", "cohost", "user"),
|
||||
"#class" : cohost.CohostUserExtractor,
|
||||
"#options" : {"avatar": True, "background": True},
|
||||
"#range" : "1-2",
|
||||
"#urls" : (
|
||||
"https://staging.cohostcdn.org/avatar/3281-abb43502-4c48-407d-9778-2bed7722d3d7-profile.gif",
|
||||
"https://staging.cohostcdn.org/header/3281-b29dbf4d-45b2-417b-b03b-0f7f07595e66-profile.png",
|
||||
),
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://cohost.org/infinitebrians/post/4957017-thank-you-akira-tori",
|
||||
"#category": ("", "cohost", "post"),
|
||||
|
||||
Reference in New Issue
Block a user