# -*- coding: utf-8 -*- # Copyright 2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://kemono.party/""" from .common import Extractor, Message from .. import text, exception from ..cache import cache import itertools import re BASE_PATTERN = r"(?:https?://)?kemono\.party/([^/?#]+)/user/([^/?#]+)" class KemonopartyExtractor(Extractor): """Base class for kemonoparty extractors""" category = "kemonoparty" root = "https://kemono.party" directory_fmt = ("{category}", "{service}", "{user}") filename_fmt = "{id}_{title}_{num:>02}_{filename}.{extension}" archive_fmt = "{service}_{user}_{id}_{num}" cookiedomain = ".kemono.party" def items(self): self._prepare_ddosguard_cookies() find_inline = re.compile(r'src="(/inline/[^"]+)').findall skip_service = \ "patreon" if self.config("patreon-skip-file", True) else None if self.config("metadata"): username = text.unescape(text.extract( self.request(self.user_url).text, ' data.kemono.party ("https://kemono.party/gumroad/user/trylsc/post/IURjT", { "pattern": r"https://kemono\.party/data/(file|attachment)s" r"/gumroad/trylsc/IURjT/", }), # username (#1548, #1652) ("https://kemono.party/gumroad/user/3252870377455/post/aJnAH", { "options": (("metadata", True),), "keyword": {"username": "Kudalyn's Creations"}, }), # skip patreon main file (#1667, #1689) ("https://kemono.party/patreon/user/4158582/post/32099982", { "count": 2, "keyword": {"type": "attachment"}, }), ("https://kemono.party/subscribestar/user/alcorart/post/184330"), ) def __init__(self, match): KemonopartyExtractor.__init__(self, match) service, user_id, post_id = match.groups() self.api_url = "{}/api/{}/user/{}/post/{}".format( self.root, service, user_id, post_id) self.user_url = "{}/{}/user/{}".format(self.root, service, user_id) def posts(self): posts = self.request(self.api_url).json() return (posts[0],) if len(posts) > 1 else posts class KemonopartyDiscordExtractor(KemonopartyExtractor): """Extractor for kemono.party discord servers""" subcategory = "discord" directory_fmt = ("{category}", "discord", "{server}", "{channel}") filename_fmt = "{id}_{num:>02}_{filename}.{extension}" archive_fmt = "discord_{server}_{id}_{num}" pattern = r"(?:https?://)?kemono\.party/discord/server/(\d+)" test = ("https://kemono.party/discord/server/256559665620451329", { "pattern": r"https://kemono\.party/data/attachments/discord" r"/256559665620451329/\d+/\d+/.+", "count": ">= 2", }) def __init__(self, match): KemonopartyExtractor.__init__(self, match) self.server = match.group(1) def items(self): self._prepare_ddosguard_cookies() for post in self.posts(): post["date"] = text.parse_datetime( post["published"], "%a, %d %b %Y %H:%M:%S %Z") yield Message.Directory, post for post["num"], file in enumerate(post["attachments"], 1): post["type"] = "attachment" url = file["path"] if url[0] == "/": url = self.root + "/data" + url elif url.startswith("https://kemono.party"): url = self.root + "/data" + url[20:] text.nameext_from_url(file["name"], post) yield Message.Url, url, post def posts(self): url = "{}/api/discord/channels/lookup?q={}".format( self.root, self.server) for channel in self.request(url).json(): url = "{}/api/discord/channel/{}".format(self.root, channel["id"]) params = {"skip": 0} while True: posts = self.request(url, params=params).json() yield from posts if len(posts) < 25: return params["skip"] += 25 class KemonopartyFavoriteExtractor(KemonopartyExtractor): """Extractor for kemono.party favorites""" subcategory = "favorite" pattern = r"(?:https?://)?kemono\.party/favorites" test = ("https://kemono.party/favorites", { "pattern": KemonopartyUserExtractor.pattern, "url": "f4b5b796979bcba824af84206578c79101c7f0e1", "count": 3, }) def items(self): self._prepare_ddosguard_cookies() self.login() users = self.request(self.root + "/api/favorites").json() for user in users: user["_extractor"] = KemonopartyUserExtractor url = "{}/{}/user/{}".format( self.root, user["service"], user["id"]) yield Message.Queue, url, user