github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[cohost] remove module

Browse Source 
cohost.org  now redirects to archive.org
This commit is contained in:
Mike Fährmann
2025-01-13 14:38:35 +01:00
parent 42070240ae
commit bde99cc6ce
6 changed files with 0 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
docs/configuration.rst
View File

@@ -1766,66 +1766,6 @@ Description
to download images in JPEG format at their original resolution.
extractor.cohost.asks
---------------------
Type
``bool``
Default
``true``
Description
Extract ``ask`` posts.
extractor.cohost.avatar
-----------------------
Type
``bool``
Default
``false``
Description
Download ``avatar`` images.
extractor.cohost.background
---------------------------
Type
``bool``
Default
``false``
Description
Download ``background``/``banner``/``header`` images.
extractor.cohost.pinned
-----------------------
Type
``bool``
Default
``false``
Description
Extract pinned posts.
extractor.cohost.replies
------------------------
Type
``bool``
Default
``true``
Description
Extract reply posts.
extractor.cohost.shares
-----------------------
Type
``bool``
Default
``false``
Description
Extract shared posts.
extractor.cyberdrop.domain
--------------------------
Type

9
docs/gallery-dl.conf
View File

@@ -173,15 +173,6 @@
"nsfw" : true,
"quality" : "original=true"
},
"cohost":
{
"asks" : true,
"avatar" : false,
"background": false,
"pinned" : false,
"replies": true,
"shares" : true
},
"coomerparty":
{
"username": "",

6
docs/supportedsites.md
View File

@@ -175,12 +175,6 @@ Consider all listed sites to potentially be NSFW.
<td>individual Images, Image Listings, Models, Model Listings, Posts, Search Results, Tag Searches, User Profiles, User Images, User Models, User Posts, User Videos</td>
<td></td>
</tr>
<tr>
<td>cohost!</td>
<td>https://cohost.org/</td>
<td>Likes, Posts, Tag Searches, User Profiles</td>
<td></td>
</tr>
<tr>
<td>Comic Vine</td>
<td>https://comicvine.gamespot.com/</td>

1
gallery_dl/extractor/__init__.py
View File

@@ -39,7 +39,6 @@ modules = [
"chevereto",
"cien",
"civitai",
"cohost",
"comicvine",
"cyberdrop",
"danbooru",

250
gallery_dl/extractor/cohost.py
View File

@@ -1,250 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2024 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://cohost.org/"""
from .common import Extractor, Message
from .. import text, util
BASE_PATTERN = r"(?:https?://)?(?:www\.)?cohost\.org"
class CohostExtractor(Extractor):
"""Base class for cohost extractors"""
category = "cohost"
root = "https://cohost.org"
directory_fmt = ("{category}", "{postingProject[handle]}")
filename_fmt = ("{postId}{headline:?_//[b:200]}{num:?_//}.{extension}")
archive_fmt = "{postId}_{num}"
def _init(self):
self.replies = self.config("replies", True)
self.pinned = self.config("pinned", False)
self.shares = self.config("shares", False)
self.asks = self.config("asks", True)
self.avatar = self.config("avatar", False)
if self.avatar:
self._urls_avatar = {None, ""}
self.background = self.config("background", False)
if self.background:
self._urls_background = {None, ""}
def items(self):
for post in self.posts():
reason = post.get("limitedVisibilityReason")
if reason and reason != "none":
if reason == "log-in-first":
reason = ("This page's posts are visible only to users "
"who are logged in.")
self.log.warning('%s: "%s"', post["postId"], reason)
files = self._extract_files(post)
post["count"] = len(files)
post["date"] = text.parse_datetime(
post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ")
yield Message.Directory, post
project = post["postingProject"]
if self.avatar:
url = project.get("avatarURL")
if url not in self._urls_avatar:
self._urls_avatar.add(url)
p = post.copy()
p["postId"] = p["kind"] = "avatar"
p["headline"] = p["num"] = ""
yield Message.Url, url, text.nameext_from_url(url, p)
if self.background:
url = project.get("headerURL")
if url not in self._urls_background:
self._urls_background.add(url)
p = post.copy()
p["postId"] = p["kind"] = "background"
p["headline"] = p["num"] = ""
yield Message.Url, url, text.nameext_from_url(url, p)
for post["num"], file in enumerate(files, 1):
url = file["fileURL"]
post.update(file)
text.nameext_from_url(url, post)
yield Message.Url, url, post
def posts(self):
return ()
def _request_api(self, endpoint, input):
url = "{}/api/v1/trpc/{}".format(self.root, endpoint)
params = {"batch": "1", "input": util.json_dumps({"0": input})}
headers = {"content-type": "application/json"}
data = self.request(url, params=params, headers=headers).json()
return data[0]["result"]["data"]
def _extract_files(self, post):
files = []
self._extract_blocks(post, files)
if self.shares and post.get("shareTree"):
for share in post["shareTree"]:
self._extract_blocks(share, files, share)
del post["shareTree"]
return files
def _extract_blocks(self, post, files, shared=None):
post["content"] = content = []
for block in post.pop("blocks") or ():
try:
type = block["type"]
if type == "attachment":
file = block["attachment"].copy()
file["shared"] = shared
files.append(file)
elif type == "attachment-row":
for att in block["attachments"]:
file = att["attachment"].copy()
file["shared"] = shared
files.append(file)
elif type == "markdown":
content.append(block["markdown"]["content"])
elif type == "ask":
post["ask"] = block["ask"]
else:
self.log.debug("%s: Unsupported block type '%s'",
post["postId"], type)
except Exception as exc:
self.log.debug("%s: %s", exc.__class__.__name__, exc)
class CohostUserExtractor(CohostExtractor):
"""Extractor for media from a cohost user"""
subcategory = "user"
pattern = BASE_PATTERN + r"/([^/?#]+)/?(?:$|\?|#)"
example = "https://cohost.org/USER"
def posts(self):
empty = 0
params = {
"projectHandle": self.groups[0],
"page": 0,
"options": {
"pinnedPostsAtTop" : True if self.pinned else False,
"hideReplies" : not self.replies,
"hideShares" : not self.shares,
"hideAsks" : not self.asks,
"viewingOnProjectPage": True,
},
}
while True:
data = self._request_api("posts.profilePosts", params)
posts = data["posts"]
if posts:
empty = 0
yield from posts
else:
empty += 1
pagination = data["pagination"]
if not pagination.get("morePagesForward"):
return
if empty >= 3:
return self.log.debug("Empty API results")
params["page"] = pagination["nextPage"]
class CohostPostExtractor(CohostExtractor):
"""Extractor for media from a single cohost post"""
subcategory = "post"
pattern = BASE_PATTERN + r"/([^/?#]+)/post/(\d+)"
example = "https://cohost.org/USER/post/12345"
def posts(self):
endpoint = "posts.singlePost"
params = {
"handle": self.groups[0],
"postId": int(self.groups[1]),
}
data = self._request_api(endpoint, params)
post = data["post"]
try:
post["comments"] = data["comments"][self.groups[1]]
except LookupError:
post["comments"] = ()
return (post,)
class CohostTagExtractor(CohostExtractor):
"""Extractor for tagged posts"""
subcategory = "tag"
pattern = BASE_PATTERN + r"/([^/?#]+)/tagged/([^/?#]+)(?:\?([^#]+))?"
example = "https://cohost.org/USER/tagged/TAG"
def posts(self):
user, tag, query = self.groups
url = "{}/{}/tagged/{}".format(self.root, user, tag)
params = text.parse_query(query)
post_feed_key = ("tagged-post-feed" if user == "rc" else
"project-tagged-post-feed")
while True:
page = self.request(url, params=params).text
data = util.json_loads(text.extr(
page, 'id="__COHOST_LOADER_STATE__">', '</script>'))
try:
feed = data[post_feed_key]
except KeyError:
feed = data.popitem()[1]
yield from feed["posts"]
pagination = feed["paginationMode"]
if not pagination.get("morePagesForward"):
return
params["refTimestamp"] = pagination["refTimestamp"]
params["skipPosts"] = \
pagination["currentSkip"] + pagination["idealPageStride"]
class CohostLikesExtractor(CohostExtractor):
"""Extractor for liked posts"""
subcategory = "likes"
pattern = BASE_PATTERN + r"/rc/liked-posts"
example = "https://cohost.org/rc/liked-posts"
def posts(self):
url = "{}/rc/liked-posts".format(self.root)
params = {}
while True:
page = self.request(url, params=params).text
data = util.json_loads(text.extr(
page, 'id="__COHOST_LOADER_STATE__">', '</script>'))
try:
feed = data["liked-posts-feed"]
except KeyError:
feed = data.popitem()[1]
yield from feed["posts"]
pagination = feed["paginationMode"]
if not pagination.get("morePagesForward"):
return
params["refTimestamp"] = pagination["refTimestamp"]
params["skipPosts"] = \
pagination["currentSkip"] + pagination["idealPageStride"]

38
test/results/cohost.py
View File

@@ -1,38 +0,0 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
from gallery_dl.extractor import cohost
__tests__ = (
{
"#url" : "https://cohost.org/infinitebrians",
"#category": ("", "cohost", "user"),
"#class" : cohost.CohostUserExtractor,
"#range" : "1-20",
"#count" : 20,
},
{
"#url" : "https://cohost.org/infinitebrians",
"#category": ("", "cohost", "user"),
"#class" : cohost.CohostUserExtractor,
"#options" : {"avatar": True, "background": True},
"#range" : "1-2",
"#urls" : (
"https://staging.cohostcdn.org/avatar/3281-abb43502-4c48-407d-9778-2bed7722d3d7-profile.gif",
"https://staging.cohostcdn.org/header/3281-b29dbf4d-45b2-417b-b03b-0f7f07595e66-profile.png",
),
},
{
"#url" : "https://cohost.org/infinitebrians/post/4957017-thank-you-akira-tori",
"#category": ("", "cohost", "post"),
"#class" : cohost.CohostPostExtractor,
"#urls" : "https://staging.cohostcdn.org/attachment/58f9aa96-d2b2-4838-b81c-9aa8bac0bea0/march%204%202024.png",
},
)