From f232a07faf676bd3bb6f8956b9bb282cf4a8a4f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 3 Mar 2025 14:55:02 +0100 Subject: [PATCH] [danbooru:pool] download posts in pool order (#7091) - add 'order-posts' option - add 'num' metadata field for pool position - update default filenames to order by pool position --- docs/configuration.rst | 19 ++++++++ docs/gallery-dl.conf | 6 ++- gallery_dl/extractor/danbooru.py | 39 +++++++++++++++- test/results/danbooru.py | 79 +++++++++++++++++++++++++++++++- 4 files changed, 139 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index dec115a4..2ce6eada 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1829,6 +1829,25 @@ Description follow the ``source`` and download from there if possible. +extractor.[Danbooru].pool.order-posts +------------------------------------- +Type + ``string`` +Default + ``"pool"`` +Description + Controls the order in which pool posts are returned. + + ``"pool"`` | ``"pool_asc"`` | ``"asc"`` | ``"asc_pool"`` + Pool order + ``"pool_desc"`` | ``"desc_pool"`` | ``"desc"`` + Reverse Pool order + ``"id"`` | ``"id_desc"`` | ``"desc_id"`` + Descending Post ID order + ``"id_asc"`` | ``"asc_id"`` + Ascending Post ID order + + extractor.[Danbooru].ugoira --------------------------- Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 5d1f931d..3564e86a 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -769,7 +769,11 @@ "external" : false, "metadata" : false, "threshold": "auto", - "ugoira" : false + "ugoira" : false, + + "pool": { + "order-posts": "pool" + } }, "danbooru": { diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index 76ac134f..8d007283 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -231,6 +231,7 @@ class DanbooruPoolExtractor(DanbooruExtractor): """Extractor for posts from danbooru pools""" subcategory = "pool" directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name]}") + filename_fmt = "{num:>04}_{id}_{filename}.{extension}" archive_fmt = "p_{pool[id]}_{id}" pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)" example = "https://danbooru.donmai.us/pools/12345" @@ -244,8 +245,42 @@ class DanbooruPoolExtractor(DanbooruExtractor): return {"pool": pool} def posts(self): - params = {"tags": "pool:" + self.pool_id} - return self._pagination("/posts.json", params, "b") + reverse = prefix = None + + order = self.config("order-posts") + if not order or order in ("asc", "pool", "pool_asc", "asc_pool"): + params = {"tags": "ordpool:" + self.pool_id} + elif order in ("id", "desc_id", "id_desc"): + params = {"tags": "pool:" + self.pool_id} + prefix = "b" + elif order in ("desc", "desc_pool", "pool_desc"): + params = {"tags": "ordpool:" + self.pool_id} + reverse = True + elif order in ("asc_id", "id_asc"): + params = {"tags": "pool:" + self.pool_id} + reverse = True + + posts = self._pagination("/posts.json", params, prefix) + if reverse: + return self._enumerate_posts_reverse(posts) + else: + return self._enumerate_posts(posts) + + def _enumerate_posts(self, posts): + pid_to_num = {pid: num+1 for num, pid in enumerate(self.post_ids)} + for post in posts: + post["num"] = pid_to_num[post["id"]] + yield post + + def _enumerate_posts_reverse(self, posts): + self.log.info("Collecting posts of pool %s", self.pool_id) + posts = list(posts) + posts.reverse() + + pid_to_num = {pid: num+1 for num, pid in enumerate(self.post_ids)} + for post in posts: + post["num"] = pid_to_num[post["id"]] + return posts class DanbooruPostExtractor(DanbooruExtractor): diff --git a/test/results/danbooru.py b/test/results/danbooru.py index 7f6cb0cb..df540d22 100644 --- a/test/results/danbooru.py +++ b/test/results/danbooru.py @@ -60,7 +60,84 @@ __tests__ = ( "#url" : "https://danbooru.donmai.us/pools/7659", "#category": ("Danbooru", "danbooru", "pool"), "#class" : danbooru.DanbooruPoolExtractor, - "#sha1_content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", + "#auth" : False, + "#sha1_content": "15226ba183579bc2cdd67260445b5c97959a3e82", + "#urls" : ( + "https://cdn.donmai.us/original/d8/3d/d83df4af8c0fa6d6069f8d3cf0b7dd56.jpg", + "https://cdn.donmai.us/original/70/82/7082572fa74650f3c1f39152550cd724.jpg", + "https://cdn.donmai.us/original/7a/b5/7ab551d011b89155a743d81308de653a.jpg", + "https://cdn.donmai.us/original/fd/d3/fdd31d7feb62e2a90de950765aa556b9.jpg", + "https://cdn.donmai.us/original/e7/11/e71119c20e1f7f7be829c168786ba807.jpg", + "https://cdn.donmai.us/original/65/ae/65ae1865ebfe490a00039832cd0ce16a.jpg", + "https://cdn.donmai.us/original/c3/e8/c3e82e213e6fa83fbe9c00659114c2d6.jpg", + "https://cdn.donmai.us/original/58/64/586438b110b88aef5f2bf14c19e7102d.jpg", + ), + + "num" : range(1, 8), + "pool": { + "category" : "series", + "created_at" : "2013-11-08T21:44:54.588-05:00", + "description": "A comic by [[ken (koala)]] about a girl. Seems to be somewhat melancholy.\r\n\r\n(Also, this will probably be renamed if it gets translated, since the title doesn't make much sense as-is)", + "id" : 7659, + "is_active" : False, + "is_deleted" : False, + "name" : "Original - ある○人の島 (Ken (Koala))", + "post_count" : 8, + "updated_at" : "2017-08-10T20:16:04.564-04:00", + }, +}, + +{ + "#url" : "https://danbooru.donmai.us/pools/24413", + "#category": ("Danbooru", "danbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#options" : {"order-posts": "asc"}, + "#urls" : ( + "https://cdn.donmai.us/original/c5/7b/c57b045fee282199277d0f94e298b9dc.jpg", + "https://cdn.donmai.us/original/44/6d/446d8b5db9b78694936408049745ee42.jpg", + "https://cdn.donmai.us/original/34/0c/340c721ceb7fce6892a234adf0bea811.jpg", + "https://cdn.donmai.us/original/47/c2/47c2c1ba1f7b83e0a487dbc7e722059f.jpg", + "https://cdn.donmai.us/original/30/3b/303bdefb719b54253aa7731cf11ef91f.jpg", + ), +}, +{ + "#url" : "https://danbooru.donmai.us/pools/24413", + "#category": ("Danbooru", "danbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#options" : {"order-posts": "pool_desc"}, + "#urls" : ( + "https://cdn.donmai.us/original/30/3b/303bdefb719b54253aa7731cf11ef91f.jpg", + "https://cdn.donmai.us/original/47/c2/47c2c1ba1f7b83e0a487dbc7e722059f.jpg", + "https://cdn.donmai.us/original/34/0c/340c721ceb7fce6892a234adf0bea811.jpg", + "https://cdn.donmai.us/original/44/6d/446d8b5db9b78694936408049745ee42.jpg", + "https://cdn.donmai.us/original/c5/7b/c57b045fee282199277d0f94e298b9dc.jpg", + ), +}, +{ + "#url" : "https://danbooru.donmai.us/pools/24413", + "#category": ("Danbooru", "danbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#options" : {"order-posts": "id"}, + "#urls" : ( + "https://cdn.donmai.us/original/30/3b/303bdefb719b54253aa7731cf11ef91f.jpg", + "https://cdn.donmai.us/original/44/6d/446d8b5db9b78694936408049745ee42.jpg", + "https://cdn.donmai.us/original/34/0c/340c721ceb7fce6892a234adf0bea811.jpg", + "https://cdn.donmai.us/original/47/c2/47c2c1ba1f7b83e0a487dbc7e722059f.jpg", + "https://cdn.donmai.us/original/c5/7b/c57b045fee282199277d0f94e298b9dc.jpg", + ), +}, +{ + "#url" : "https://danbooru.donmai.us/pools/24413", + "#category": ("Danbooru", "danbooru", "pool"), + "#class" : danbooru.DanbooruPoolExtractor, + "#options" : {"order-posts": "asc_id"}, + "#urls" : ( + "https://cdn.donmai.us/original/c5/7b/c57b045fee282199277d0f94e298b9dc.jpg", + "https://cdn.donmai.us/original/47/c2/47c2c1ba1f7b83e0a487dbc7e722059f.jpg", + "https://cdn.donmai.us/original/34/0c/340c721ceb7fce6892a234adf0bea811.jpg", + "https://cdn.donmai.us/original/44/6d/446d8b5db9b78694936408049745ee42.jpg", + "https://cdn.donmai.us/original/30/3b/303bdefb719b54253aa7731cf11ef91f.jpg", + ), }, {