diff --git a/docs/configuration.rst b/docs/configuration.rst index 0cbc1340..5d77d9ee 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -3203,6 +3203,7 @@ Description Supported values are * ``gallery`` + * ``posts`` * ``followers`` * ``following`` * ``stars`` diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 87e8e6d2..c0019210 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -514,7 +514,7 @@ Consider all listed sites to potentially be NSFW. Itaku https://itaku.ee/ - Followers, Followed Users, Galleries, individual Images, Search Results, Stars, User Profiles + Followers, Followed Users, Galleries, individual Images, Posts, Search Results, Stars, User Profiles diff --git a/gallery_dl/extractor/itaku.py b/gallery_dl/extractor/itaku.py index 0ea22526..624fcce1 100644 --- a/gallery_dl/extractor/itaku.py +++ b/gallery_dl/extractor/itaku.py @@ -32,7 +32,6 @@ class ItakuExtractor(Extractor): def items(self): if images := self.images(): for image in images: - image["date"] = text.parse_datetime( image["date_added"], "%Y-%m-%dT%H:%M:%S.%fZ") for category, tags in image.pop("categorized_tags").items(): @@ -59,7 +58,20 @@ class ItakuExtractor(Extractor): if posts := self.posts(): for post in posts: - ... + images = post.pop("gallery_images") or () + post["count"] = len(images) + post["date"] = text.parse_datetime( + post["date_added"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["tags"] = [t["name"] for t in post["tags"]] + + yield Message.Directory, post + for post["num"], image in enumerate(images, 1): + post["file"] = image + image["date"] = text.parse_datetime( + image["date_added"], "%Y-%m-%dT%H:%M:%S.%fZ") + + url = image["image"] + yield Message.Url, url, text.nameext_from_url(url, post) return if users := self.users(): @@ -74,7 +86,7 @@ class ItakuExtractor(Extractor): class ItakuGalleryExtractor(ItakuExtractor): - """Extractor for posts from an itaku user gallery""" + """Extractor for images from an itaku user gallery""" subcategory = "gallery" pattern = USER_PATTERN + r"/gallery(?:/(\d+))?" example = "https://itaku.ee/profile/USER/gallery" @@ -87,6 +99,24 @@ class ItakuGalleryExtractor(ItakuExtractor): }) +class ItakuPostsExtractor(ItakuExtractor): + """Extractor for an itaku user's posts""" + subcategory = "posts" + directory_fmt = ("{category}", "{owner_username}", "Posts", + "{id}{title:? //}") + filename_fmt = "{file[id]}{file[title]:? //}.{extension}" + archive_fmt = "{id}_{file[id]}" + pattern = USER_PATTERN + r"/posts(?:/(\d+))?" + example = "https://itaku.ee/profile/USER/posts" + + def posts(self): + user, folder = self.groups + return self.api.posts({ + "owner" : self.api.user_id(user), + "folders": folder, + }) + + class ItakuStarsExtractor(ItakuExtractor): subcategory = "stars" pattern = USER_PATTERN + r"/stars(?:/(\d+))?" @@ -132,6 +162,7 @@ class ItakuUserExtractor(Dispatch, ItakuExtractor): base = f"{self.root}/profile/{self.groups[0]}/" return self._dispatch_extractors(( (ItakuGalleryExtractor , base + "gallery"), + (ItakuPostsExtractor , base + "posts"), (ItakuFollowersExtractor, base + "followers"), (ItakuFollowingExtractor, base + "following"), (ItakuStarsExtractor , base + "stara"), @@ -147,6 +178,19 @@ class ItakuImageExtractor(ItakuExtractor): return (self.api.image(self.groups[0]),) +class ItakuPostExtractor(ItakuExtractor): + subcategory = "post" + directory_fmt = ("{category}", "{owner_username}", "Posts", + "{id}{title:? //}") + filename_fmt = "{file[id]}{file[title]:? //}.{extension}" + archive_fmt = "{id}_{file[id]}" + pattern = BASE_PATTERN + r"/posts/(\d+)" + example = "https://itaku.ee/posts/12345" + + def posts(self): + return (self.api.post(self.groups[0]),) + + class ItakuSearchExtractor(ItakuExtractor): subcategory = "search" pattern = BASE_PATTERN + r"/home/images/?\?([^#]+)" @@ -200,6 +244,19 @@ class ItakuAPI(): } return self._pagination(endpoint, params, self.image) + def posts(self, params): + endpoint = "/posts/" + params = { + "cursor" : None, + "date_range": "", + "maturity_rating": ("SFW", "Questionable", "NSFW"), + "ordering" : "-date_added", + "page" : "1", + "page_size" : "30", + **params, + } + return self._pagination(endpoint, params) + def user_profiles(self, params): endpoint = "/user_profiles/" params = { @@ -216,6 +273,10 @@ class ItakuAPI(): endpoint = f"/galleries/images/{image_id}/" return self._call(endpoint) + def post(self, post_id): + endpoint = f"/posts/{post_id}/" + return self._call(endpoint) + @memcache(keyarg=1) def user(self, username): return self._call(f"/user_profiles/{username}/") diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 3b0ffa3c..50afbd98 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -293,6 +293,9 @@ SUBCATEGORY_MAP = { "saved": "Saved Posts", "tagged": "Tagged Posts", }, + "itaku": { + "posts": "", + }, "iwara": { "user-images": "User Images", "user-videos": "User Videos", diff --git a/test/results/itaku.py b/test/results/itaku.py index 325f8b6a..39c8ac1e 100644 --- a/test/results/itaku.py +++ b/test/results/itaku.py @@ -36,6 +36,36 @@ __tests__ = ( "sections" : ["Fanart/Pokemon"], }, +{ + "#url" : "https://itaku.ee/profile/piku/posts", + "#class" : itaku.ItakuPostsExtractor, + "#results" : ( + "https://itaku.ee/api/media/gallery_imgs/220415_xEFUVR6.png", + "https://itaku.ee/api/media/gallery_imgs/220308_J0mgJ24.png", + "https://itaku.ee/api/media/gallery_imgs/220511_rdGpatf.png", + "https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB.png", + ), + + "id" : {23762, 16422}, + "count": {3, 1}, + "num" : range(1, 3), + "date" : "type:datetime", + "title": {"Maids", ""}, +}, + +{ + "#url" : "https://itaku.ee/profile/starluxioad/posts/2008", + "#comment" : "posts folder", + "#class" : itaku.ItakuPostsExtractor, + "#count" : 12, + + "id" : {160779, 160163, 151859, 151851, 150443}, + "count": {2, 3}, + "num" : range(1, 3), + "date" : "type:datetime", + "title": str, +}, + { "#url" : "https://itaku.ee/profile/piku/stars", "#class" : itaku.ItakuStarsExtractor, @@ -119,6 +149,75 @@ __tests__ = ( "#results" : "https://itaku.ee/api/media/gallery_vids/sleepy_af_OY5GHWw.mp4", }, +{ + "#url" : "https://itaku.ee/posts/16422", + "#class" : itaku.ItakuPostExtractor, + "#results" : "https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB.png", + + "already_pinned" : None, + "can_reshare" : True, + "category" : "itaku", + "content" : "", + "content_warning": "", + "count" : 1, + "created_by_images": True, + "date" : "dt:2022-04-26 16:06:30", + "date_added" : "2022-04-26T16:06:30.352389Z", + "date_edited" : "2022-05-10T21:32:44.017311Z", + "extension" : "png", + "file" : { + "already_pinned" : None, + "animated" : False, + "blacklisted" : { + "blacklisted_tags": [], + "is_blacklisted" : False, + }, + "bookmarked_by_you": False, + "content_warning": "", + "date" : "dt:2022-04-26 16:06:28", + "date_added" : "2022-04-26T16:06:28.272442Z", + "date_edited" : "2022-06-30T09:43:58.816192Z", + "id" : 77775, + "image" : "https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB.png", + "image_lg" : "https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB/lg.jpg", + "image_xl" : "https://itaku.ee/api/media/gallery_imgs/220420b_4Lrk6gB/lg.jpg", + "is_thumbnail_for_video": False, + "liked_by_you" : False, + "maturity_rating": "SFW", + "num_comments" : 0, + "num_likes" : range(60, 90), + "num_reshares" : 0, + "owner" : 16775, + "owner_displayname": "Piku", + "show_content_warning": False, + "title" : "Felicia", + "too_mature" : False, + "visibility" : "PUBLIC", + }, + "filename" : "220420b_4Lrk6gB", + "folders" : [], + "id" : 16422, + "liked_by_you" : False, + "maturity_rating": "SFW", + "num" : 1, + "num_comments" : 0, + "num_images" : 1, + "num_likes" : range(40, 70), + "num_reshares" : 0, + "obj_tags" : 99052, + "owner" : 16775, + "owner_avatar" : "https://itaku.ee/api/media/profile_pics/av2022r_vKYVywc/md.jpg", + "owner_displayname": "Piku", + "owner_username" : "piku", + "poll" : None, + "reshared_by_you": False, + "subcategory" : "post", + "tags" : [], + "title" : "", + "too_mature" : False, + "visibility" : "PUBLIC", +}, + { "#url" : "https://itaku.ee/home/images?tags=cute", "#comment" : "simple search",