From 24bbcbcfa3fa5e7a18b431611dbce9f3f70385aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 26 Mar 2025 12:48:59 +0100 Subject: [PATCH] [danbooru] add 'favgroup' extractor --- docs/configuration.rst | 4 +- docs/gallery-dl.conf | 3 + docs/supportedsites.md | 8 +-- gallery_dl/extractor/danbooru.py | 101 +++++++++++++++++++------------ scripts/supportedsites.py | 1 + test/results/danbooru.py | 17 ++++++ 6 files changed, 90 insertions(+), 44 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index c4a31cc1..0f5e7386 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1886,12 +1886,14 @@ Description extractor.[Danbooru].pool.order-posts ------------------------------------- +extractor.[Danbooru].favgroup.order-posts +----------------------------------------- Type ``string`` Default ``"pool"`` Description - Controls the order in which pool posts are returned. + Controls the order in which ``pool``/``favgroup`` posts are returned. ``"pool"`` | ``"pool_asc"`` | ``"asc"`` | ``"asc_pool"`` Pool order diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 3df119e4..8ede5683 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -792,6 +792,9 @@ "threshold": "auto", "ugoira" : false, + "favgroup": { + "order-posts": "pool" + }, "pool": { "order-posts": "pool" } diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 25a05b60..b70c7935 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1190,25 +1190,25 @@ Consider all listed sites to potentially be NSFW. Danbooru https://danbooru.donmai.us/ - Artists, Artist Searches, Pools, Popular Images, Posts, Tag Searches + Artists, Artist Searches, Favorite Groups, Pools, Popular Images, Posts, Tag Searches Supported ATFBooru https://booru.allthefallen.moe/ - Artists, Artist Searches, Pools, Popular Images, Posts, Tag Searches + Artists, Artist Searches, Favorite Groups, Pools, Popular Images, Posts, Tag Searches Supported Aibooru https://aibooru.online/ - Artists, Artist Searches, Pools, Popular Images, Posts, Tag Searches + Artists, Artist Searches, Favorite Groups, Pools, Popular Images, Posts, Tag Searches Supported Booruvar https://booru.borvar.art/ - Artists, Artist Searches, Pools, Popular Images, Posts, Tag Searches + Artists, Artist Searches, Favorite Groups, Pools, Popular Images, Posts, Tag Searches Supported diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index 8d007283..741800cf 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -175,6 +175,51 @@ class DanbooruExtractor(BaseExtractor): return [{"file": fmt(index), "delay": delay} for index, delay in enumerate(delays)] + def _collection_posts(self, cid, ctype): + reverse = prefix = None + + order = self.config("order-posts") + if not order or order in {"asc", "pool", "pool_asc", "asc_pool"}: + params = {"tags": "ord{}:{}".format(ctype, cid)} + elif order in {"id", "desc_id", "id_desc"}: + params = {"tags": "{}:{}".format(ctype, cid)} + prefix = "b" + elif order in {"desc", "desc_pool", "pool_desc"}: + params = {"tags": "ord{}:{}".format(ctype, cid)} + reverse = True + elif order in {"asc_id", "id_asc"}: + params = {"tags": "{}:{}".format(ctype, cid)} + reverse = True + + posts = self._pagination("/posts.json", params, prefix) + if reverse: + self.log.info("Collecting posts of %s %s", ctype, cid) + return self._collection_enumerate_reverse(posts) + else: + return self._collection_enumerate(posts) + + def _collection_metadata(self, cid, ctype, cname=None): + url = "{}/{}s/{}.json".format(self.root, cname or ctype, cid) + collection = self.request(url).json() + collection["name"] = collection["name"].replace("_", " ") + self.post_ids = collection.pop("post_ids", ()) + return {ctype: collection} + + def _collection_enumerate(self, posts): + pid_to_num = {pid: num for num, pid in enumerate(self.post_ids, 1)} + for post in posts: + post["num"] = pid_to_num[post["id"]] + yield post + + def _collection_enumerate_reverse(self, posts): + posts = list(posts) + posts.reverse() + + pid_to_num = {pid: num for num, pid in enumerate(self.post_ids, 1)} + for post in posts: + post["num"] = pid_to_num[post["id"]] + return posts + BASE_PATTERN = DanbooruExtractor.update({ "danbooru": { @@ -228,7 +273,7 @@ class DanbooruTagExtractor(DanbooruExtractor): class DanbooruPoolExtractor(DanbooruExtractor): - """Extractor for posts from danbooru pools""" + """Extractor for Danbooru pools""" subcategory = "pool" directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name]}") filename_fmt = "{num:>04}_{id}_{filename}.{extension}" @@ -237,50 +282,28 @@ class DanbooruPoolExtractor(DanbooruExtractor): example = "https://danbooru.donmai.us/pools/12345" def metadata(self): - self.pool_id = self.groups[-1] - url = "{}/pools/{}.json".format(self.root, self.pool_id) - pool = self.request(url).json() - pool["name"] = pool["name"].replace("_", " ") - self.post_ids = pool.pop("post_ids", ()) - return {"pool": pool} + return self._collection_metadata(self.groups[-1], "pool") def posts(self): - reverse = prefix = None + return self._collection_posts(self.groups[-1], "pool") - order = self.config("order-posts") - if not order or order in ("asc", "pool", "pool_asc", "asc_pool"): - params = {"tags": "ordpool:" + self.pool_id} - elif order in ("id", "desc_id", "id_desc"): - params = {"tags": "pool:" + self.pool_id} - prefix = "b" - elif order in ("desc", "desc_pool", "pool_desc"): - params = {"tags": "ordpool:" + self.pool_id} - reverse = True - elif order in ("asc_id", "id_asc"): - params = {"tags": "pool:" + self.pool_id} - reverse = True - posts = self._pagination("/posts.json", params, prefix) - if reverse: - return self._enumerate_posts_reverse(posts) - else: - return self._enumerate_posts(posts) +class DanbooruFavgroupExtractor(DanbooruExtractor): + """Extractor for Danbooru favorite groups""" + subcategory = "favgroup" + directory_fmt = ("{category}", "Favorite Groups", + "{favgroup[id]} {favgroup[name]}") + filename_fmt = "{num:>04}_{id}_{filename}.{extension}" + archive_fmt = "fg_{favgroup[id]}_{id}" + pattern = BASE_PATTERN + r"/favorite_group(?:s|/show)/(\d+)" + example = "https://danbooru.donmai.us/favorite_groups/12345" - def _enumerate_posts(self, posts): - pid_to_num = {pid: num+1 for num, pid in enumerate(self.post_ids)} - for post in posts: - post["num"] = pid_to_num[post["id"]] - yield post + def metadata(self): + return self._collection_metadata( + self.groups[-1], "favgroup", "favorite_group") - def _enumerate_posts_reverse(self, posts): - self.log.info("Collecting posts of pool %s", self.pool_id) - posts = list(posts) - posts.reverse() - - pid_to_num = {pid: num+1 for num, pid in enumerate(self.post_ids)} - for post in posts: - post["num"] = pid_to_num[post["id"]] - return posts + def posts(self): + return self._collection_posts(self.groups[-1], "favgroup") class DanbooruPostExtractor(DanbooruExtractor): diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 9f16b506..5b2bd94e 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -236,6 +236,7 @@ SUBCATEGORY_MAP = { }, "Danbooru": { "artist-search": "Artist Searches", + "favgroup": "Favorite Groups", }, "desktopography": { "site": "", diff --git a/test/results/danbooru.py b/test/results/danbooru.py index df540d22..1742bd1c 100644 --- a/test/results/danbooru.py +++ b/test/results/danbooru.py @@ -146,6 +146,23 @@ __tests__ = ( "#class" : danbooru.DanbooruPoolExtractor, }, +{ + "#url" : "https://danbooru.donmai.us/favorite_groups/14", + "#category": ("Danbooru", "danbooru", "favgroup"), + "#class" : danbooru.DanbooruFavgroupExtractor, + "#auth" : False, + "#count" : 24, + + "favgroup": { + "created_at": "2015-06-29T13:59:10.808-04:00", + "creator_id": 65304, + "id" : 14, + "is_public" : True, + "name" : "Mecha", + "updated_at": "2019-07-07T07:21:50.677-04:00", + }, +}, + { "#url" : "https://danbooru.donmai.us/posts/294929", "#category": ("Danbooru", "danbooru", "post"),