[itaku] add 'posts' extractors (#7707)
This commit is contained in:
Mike Fährmann
@@ -3203,6 +3203,7 @@ Description
|
||||
Supported values are
|
||||
|
||||
* ``gallery``
|
||||
* ``posts``
|
||||
* ``followers``
|
||||
* ``following``
|
||||
* ``stars``
|
||||
|
||||
@@ -514,7 +514,7 @@ Consider all listed sites to potentially be NSFW.
|
||||
<tr>
|
||||
<td>Itaku</td>
|
||||
<td>https://itaku.ee/</td>
|
||||
<td>Followers, Followed Users, Galleries, individual Images, Search Results, Stars, User Profiles</td>
|
||||
<td>Followers, Followed Users, Galleries, individual Images, Posts, Search Results, Stars, User Profiles</td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
||||
@@ -32,7 +32,6 @@ class ItakuExtractor(Extractor):
|
||||
def items(self):
|
||||
if images := self.images():
|
||||
for image in images:
|
||||
|
||||
image["date"] = text.parse_datetime(
|
||||
image["date_added"], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
for category, tags in image.pop("categorized_tags").items():
|
||||
@@ -59,7 +58,20 @@ class ItakuExtractor(Extractor):
|
||||
|
||||
if posts := self.posts():
|
||||
for post in posts:
|
||||
...
|
||||
images = post.pop("gallery_images") or ()
|
||||
post["count"] = len(images)
|
||||
post["date"] = text.parse_datetime(
|
||||
post["date_added"], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
post["tags"] = [t["name"] for t in post["tags"]]
|
||||
|
||||
yield Message.Directory, post
|
||||
for post["num"], image in enumerate(images, 1):
|
||||
post["file"] = image
|
||||
image["date"] = text.parse_datetime(
|
||||
image["date_added"], "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
url = image["image"]
|
||||
yield Message.Url, url, text.nameext_from_url(url, post)
|
||||
return
|
||||
|
||||
if users := self.users():
|
||||
@@ -74,7 +86,7 @@ class ItakuExtractor(Extractor):
|
||||
|
||||
|
||||
class ItakuGalleryExtractor(ItakuExtractor):
|
||||
"""Extractor for posts from an itaku user gallery"""
|
||||
"""Extractor for images from an itaku user gallery"""
|
||||
subcategory = "gallery"
|
||||
pattern = USER_PATTERN + r"/gallery(?:/(\d+))?"
|
||||
example = "https://itaku.ee/profile/USER/gallery"
|
||||
@@ -87,6 +99,24 @@ class ItakuGalleryExtractor(ItakuExtractor):
|
||||
})
|
||||
|
||||
|
||||
class ItakuPostsExtractor(ItakuExtractor):
|
||||
"""Extractor for an itaku user's posts"""
|
||||
subcategory = "posts"
|
||||
directory_fmt = ("{category}", "{owner_username}", "Posts",
|
||||
"{id}{title:? //}")
|
||||
filename_fmt = "{file[id]}{file[title]:? //}.{extension}"
|
||||
archive_fmt = "{id}_{file[id]}"
|
||||
pattern = USER_PATTERN + r"/posts(?:/(\d+))?"
|
||||
example = "https://itaku.ee/profile/USER/posts"
|
||||
|
||||
def posts(self):
|
||||
user, folder = self.groups
|
||||
return self.api.posts({
|
||||
"owner" : self.api.user_id(user),
|
||||
"folders": folder,
|
||||
})
|
||||
|
||||
|
||||
class ItakuStarsExtractor(ItakuExtractor):
|
||||
subcategory = "stars"
|
||||
pattern = USER_PATTERN + r"/stars(?:/(\d+))?"
|
||||
@@ -132,6 +162,7 @@ class ItakuUserExtractor(Dispatch, ItakuExtractor):
|
||||
base = f"{self.root}/profile/{self.groups[0]}/"
|
||||
return self._dispatch_extractors((
|
||||
(ItakuGalleryExtractor , base + "gallery"),
|
||||
(ItakuPostsExtractor , base + "posts"),
|
||||
(ItakuFollowersExtractor, base + "followers"),
|
||||
(ItakuFollowingExtractor, base + "following"),
|
||||
(ItakuStarsExtractor , base + "stara"),
|
||||
@@ -147,6 +178,19 @@ class ItakuImageExtractor(ItakuExtractor):
|
||||
return (self.api.image(self.groups[0]),)
|
||||
|
||||
|
||||
class ItakuPostExtractor(ItakuExtractor):
|
||||
subcategory = "post"
|
||||
directory_fmt = ("{category}", "{owner_username}", "Posts",
|
||||
"{id}{title:? //}")
|
||||
filename_fmt = "{file[id]}{file[title]:? //}.{extension}"
|
||||
archive_fmt = "{id}_{file[id]}"
|
||||
pattern = BASE_PATTERN + r"/posts/(\d+)"
|
||||
example = "https://itaku.ee/posts/12345"
|
||||
|
||||
def posts(self):
|
||||
return (self.api.post(self.groups[0]),)
|
||||
|
||||
|
||||
class ItakuSearchExtractor(ItakuExtractor):
|
||||
subcategory = "search"
|
||||
pattern = BASE_PATTERN + r"/home/images/?\?([^#]+)"
|
||||
@@ -200,6 +244,19 @@ class ItakuAPI():
|
||||
}
|
||||
return self._pagination(endpoint, params, self.image)
|
||||
|
||||
def posts(self, params):
|
||||
endpoint = "/posts/"
|
||||
params = {
|
||||
"cursor" : None,
|
||||
"date_range": "",
|
||||
"maturity_rating": ("SFW", "Questionable", "NSFW"),
|
||||
"ordering" : "-date_added",
|
||||
"page" : "1",
|
||||
"page_size" : "30",
|
||||
**params,
|
||||
}
|
||||
return self._pagination(endpoint, params)
|
||||
|
||||
def user_profiles(self, params):
|
||||
endpoint = "/user_profiles/"
|
||||
params = {
|
||||
@@ -216,6 +273,10 @@ class ItakuAPI():
|
||||
endpoint = f"/galleries/images/{image_id}/"
|
||||
return self._call(endpoint)
|
||||
|
||||
def post(self, post_id):
|
||||
endpoint = f"/posts/{post_id}/"
|
||||
return self._call(endpoint)
|
||||
|
||||
@memcache(keyarg=1)
|
||||
def user(self, username):
|
||||
return self._call(f"/user_profiles/{username}/")
|
||||
|
||||
@@ -293,6 +293,9 @@ SUBCATEGORY_MAP = {
|
||||
"saved": "Saved Posts",
|
||||
"tagged": "Tagged Posts",
|
||||
},
|
||||
"itaku": {
|
||||
"posts": "",
|
||||
},
|
||||
"iwara": {
|
||||
"user-images": "User Images",
|
||||
"user-videos": "User Videos",
|
||||
|
||||
@@ -36,6 +36,36 @@ __tests__ = (
|
||||
"sections" : ["Fanart/Pokemon"],
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://itaku.ee/profile/piku/posts",
|
||||
"#class" : itaku.ItakuPostsExtractor,
|
||||
"#results" : (
|
||||
"https://itaku.ee/api/media/gallery_imgs/220415_xEFUVR6.png",
|
||||
"https://itaku.ee/api/media/gallery_imgs/220308_J0mgJ24.png",
|
||||
"https://itaku.ee/api/media/gallery_imgs/220511_rdGpatf.png",
|
||||
"https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB.png",
|
||||
),
|
||||
|
||||
"id" : {23762, 16422},
|
||||
"count": {3, 1},
|
||||
"num" : range(1, 3),
|
||||
"date" : "type:datetime",
|
||||
"title": {"Maids", ""},
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://itaku.ee/profile/starluxioad/posts/2008",
|
||||
"#comment" : "posts folder",
|
||||
"#class" : itaku.ItakuPostsExtractor,
|
||||
"#count" : 12,
|
||||
|
||||
"id" : {160779, 160163, 151859, 151851, 150443},
|
||||
"count": {2, 3},
|
||||
"num" : range(1, 3),
|
||||
"date" : "type:datetime",
|
||||
"title": str,
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://itaku.ee/profile/piku/stars",
|
||||
"#class" : itaku.ItakuStarsExtractor,
|
||||
@@ -119,6 +149,75 @@ __tests__ = (
|
||||
"#results" : "https://itaku.ee/api/media/gallery_vids/sleepy_af_OY5GHWw.mp4",
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://itaku.ee/posts/16422",
|
||||
"#class" : itaku.ItakuPostExtractor,
|
||||
"#results" : "https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB.png",
|
||||
|
||||
"already_pinned" : None,
|
||||
"can_reshare" : True,
|
||||
"category" : "itaku",
|
||||
"content" : "",
|
||||
"content_warning": "",
|
||||
"count" : 1,
|
||||
"created_by_images": True,
|
||||
"date" : "dt:2022-04-26 16:06:30",
|
||||
"date_added" : "2022-04-26T16:06:30.352389Z",
|
||||
"date_edited" : "2022-05-10T21:32:44.017311Z",
|
||||
"extension" : "png",
|
||||
"file" : {
|
||||
"already_pinned" : None,
|
||||
"animated" : False,
|
||||
"blacklisted" : {
|
||||
"blacklisted_tags": [],
|
||||
"is_blacklisted" : False,
|
||||
},
|
||||
"bookmarked_by_you": False,
|
||||
"content_warning": "",
|
||||
"date" : "dt:2022-04-26 16:06:28",
|
||||
"date_added" : "2022-04-26T16:06:28.272442Z",
|
||||
"date_edited" : "2022-06-30T09:43:58.816192Z",
|
||||
"id" : 77775,
|
||||
"image" : "https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB.png",
|
||||
"image_lg" : "https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB/lg.jpg",
|
||||
"image_xl" : "https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB/lg.jpg",
|
||||
"is_thumbnail_for_video": False,
|
||||
"liked_by_you" : False,
|
||||
"maturity_rating": "SFW",
|
||||
"num_comments" : 0,
|
||||
"num_likes" : range(60, 90),
|
||||
"num_reshares" : 0,
|
||||
"owner" : 16775,
|
||||
"owner_displayname": "Piku",
|
||||
"show_content_warning": False,
|
||||
"title" : "Felicia",
|
||||
"too_mature" : False,
|
||||
"visibility" : "PUBLIC",
|
||||
},
|
||||
"filename" : "220420b_4Lrk6gB",
|
||||
"folders" : [],
|
||||
"id" : 16422,
|
||||
"liked_by_you" : False,
|
||||
"maturity_rating": "SFW",
|
||||
"num" : 1,
|
||||
"num_comments" : 0,
|
||||
"num_images" : 1,
|
||||
"num_likes" : range(40, 70),
|
||||
"num_reshares" : 0,
|
||||
"obj_tags" : 99052,
|
||||
"owner" : 16775,
|
||||
"owner_avatar" : "https://itaku.ee/api/media/profile_pics/av2022r_vKYVywc/md.jpg",
|
||||
"owner_displayname": "Piku",
|
||||
"owner_username" : "piku",
|
||||
"poll" : None,
|
||||
"reshared_by_you": False,
|
||||
"subcategory" : "post",
|
||||
"tags" : [],
|
||||
"title" : "",
|
||||
"too_mature" : False,
|
||||
"visibility" : "PUBLIC",
|
||||
},
|
||||
|
||||
{
|
||||
"#url" : "https://itaku.ee/home/images?tags=cute",
|
||||
"#comment" : "simple search",
|
||||
|
||||
Reference in New Issue
Block a user