From f117e32910e80e73e81ffbf0438feb6410f0bdfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 29 Feb 2020 23:37:53 +0100 Subject: [PATCH] [danbooru] restore 'popular' functionality --- gallery_dl/extractor/booru.py | 29 ++++------------ gallery_dl/extractor/danbooru.py | 57 +++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/gallery_dl/extractor/booru.py b/gallery_dl/extractor/booru.py index a48b48b1..162e9cca 100644 --- a/gallery_dl/extractor/booru.py +++ b/gallery_dl/extractor/booru.py @@ -203,8 +203,8 @@ class PostMixin(): self.params["tags"] = "id:" + self.post -class PopularMixin(): - """Extraction and metadata handling for Danbooru v2""" +class MoebooruPopularMixin(): + """Extraction and metadata handling for Moebooru and Danbooru v1""" subcategory = "popular" directory_fmt = ("{category}", "popular", "{scale}", "{date}") archive_fmt = "P_{scale[0]}_{date}_{id}" @@ -214,37 +214,20 @@ class PopularMixin(): def __init__(self, match): super().__init__(match) self.params.update(text.parse_query(match.group("query"))) + self.scale = match.group("scale") def get_metadata(self, fmt="%Y-%m-%d"): - date = self.get_date() or datetime.datetime.utcnow().strftime(fmt) + date = self.get_date() or datetime.date.today().isoformat() scale = self.get_scale() or "day" if scale == "week": - dt = datetime.datetime.strptime(date, fmt) - dt -= datetime.timedelta(days=dt.weekday()) - date = dt.strftime(fmt) + date = datetime.date.fromisoformat(date) + date = (date - datetime.timedelta(days=date.weekday())).isoformat() elif scale == "month": date = date[:-3] return {"date": date, "scale": scale} - def get_scale(self): - if "scale" in self.params: - return self.params["scale"] - return None - - def get_date(self): - if "date" in self.params: - return self.params["date"][:10] - return None - - -class MoebooruPopularMixin(PopularMixin): - """Extraction and metadata handling for Moebooru and Danbooru v1""" - def __init__(self, match): - super().__init__(match) - self.scale = match.group("scale") - def get_date(self): if "year" in self.params: return "{:>04}-{:>02}-{:>02}".format( diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index 3d7f78f2..3fdeaf92 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -10,6 +10,7 @@ from .common import Extractor, Message, SharedConfigMixin from .. import text +import datetime BASE_PATTERN = ( @@ -31,7 +32,6 @@ class DanbooruExtractor(SharedConfigMixin, Extractor): def __init__(self, match): Extractor.__init__(self, match) self.root = "https://{}.donmai.us".format(match.group(1)) - self.api_url = self.root + "/posts.json" self.ugoira = self.config("ugoira", True) self.params = {} @@ -74,17 +74,24 @@ class DanbooruExtractor(SharedConfigMixin, Extractor): return {} def posts(self): + return self._pagination(self.root + "/posts.json") + + def _pagination(self, url, pagenum=False): params = self.params.copy() params["limit"] = self.per_page params["page"] = self.page_start while True: - posts = self.request(self.api_url, params=params).json() + posts = self.request(url, params=params).json() yield from posts if len(posts) < self.per_page: return - params["page"] = "b{}".format(posts[-1]["id"]) + + if pagenum: + params["page"] += 1 + else: + params["page"] = "b{}".format(posts[-1]["id"]) class DanbooruTagExtractor(DanbooruExtractor): @@ -92,14 +99,14 @@ class DanbooruTagExtractor(DanbooruExtractor): subcategory = "tag" directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" - pattern = BASE_PATTERN + r"/posts\?(?:[^&#]*&)*tags=(?P[^&#]+)" + pattern = BASE_PATTERN + r"/posts\?(?:[^&#]*&)*tags=([^&#]+)" test = ( ("https://danbooru.donmai.us/posts?tags=bonocho", { "content": "b196fb9f1668109d7774a0a82efea3ffdda07746", }), # test page transitions - ("https://danbooru.donmai.us/posts?tags=canvas_%28cocktail_soft%29", { - "count": ">= 50", + ("https://danbooru.donmai.us/posts?tags=mushishi", { + "count": ">= 300", }), ("https://hijiribe.donmai.us/posts?tags=bonocho"), ("https://sonohara.donmai.us/posts?tags=bonocho"), @@ -132,6 +139,7 @@ class DanbooruPoolExtractor(DanbooruExtractor): def metadata(self): url = "{}/pools/{}.json".format(self.root, self.pool_id) pool = self.request(url).json() + pool["name"] = pool["name"].replace("_", " ") del pool["post_ids"] return {"pool": pool} @@ -160,21 +168,38 @@ class DanbooruPostExtractor(DanbooruExtractor): return (self.request(url).json(),) -r''' -class DanbooruPopularExtractor(booru.PopularMixin, DanbooruExtractor): +class DanbooruPopularExtractor(DanbooruExtractor): """Extractor for popular images from danbooru""" - pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?(?P[^#]*))?" + subcategory = "popular" + directory_fmt = ("{category}", "popular", "{scale}", "{date}") + archive_fmt = "P_{scale[0]}_{date}_{id}" + pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?([^#]*))?" test = ( ("https://danbooru.donmai.us/explore/posts/popular"), (("https://danbooru.donmai.us/explore/posts/popular" - "?date=2013-06-06+03%3A34%3A22+-0400&scale=week"), { - "count": ">= 1", + "?date=2013-06-06&scale=week"), { + "range": "1-120", + "count": 120, }), ) def __init__(self, match): - super().__init__(match) - urlfmt = "{scheme}://{subdomain}.donmai.us/explore/posts/popular.json" - self.api_url = urlfmt.format( - scheme=self.scheme, subdomain=self.subdomain) -''' + DanbooruExtractor.__init__(self, match) + self.params.update(text.parse_query(match.group(2))) + + def metadata(self): + self.page_start = self.page_start or 1 + scale = self.params.get("scale", "day") + date = self.params.get("date") or datetime.date.today().isoformat() + + if scale == "week": + date = datetime.date.fromisoformat(date) + date = (date - datetime.timedelta(days=date.weekday())).isoformat() + elif scale == "month": + date = date[:-3] + + return {"date": date, "scale": scale} + + def posts(self): + url = self.root + "/explore/posts/popular.json" + return self._pagination(url, True)