From e2be199124cae165dcd57c3589c8d1abfcc0d696 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Wed, 26 Jan 2022 19:43:14 +0100
Subject: [PATCH] [gelbooru] improve and fix pagination (#2230, #2232)

Use 'id:<POSTID' as a tag instead of going through pages with 'pid'.

Something similar was already implemented in 93cef784,
but that got broken again in 3085aac4.
---
 CHANGELOG.md                     |  4 +++-
 gallery_dl/extractor/gelbooru.py | 31 +++++++++++++------------------
 gallery_dl/version.py            |  2 +-
 3 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0cefbb2e..c2a108b1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,11 @@
 # Changelog
 
+## Unreleased
+
 ## 1.20.3 - 2022-01-26
 ### Fixes
 - [kemonoparty] fix DMs extraction ([#2008](https://github.com/mikf/gallery-dl/issues/2008))
-- [twitter] fix crash ob Tweets with deleted quotes ([#2225](https://github.com/mikf/gallery-dl/issues/2225))
+- [twitter] fix crash on Tweets with deleted quotes ([#2225](https://github.com/mikf/gallery-dl/issues/2225))
 - [twitter] fix crash on suspended Tweets without `legacy` entry ([#2216](https://github.com/mikf/gallery-dl/issues/2216))
 - [twitter] fix crash on unified cards without `type`
 - [twitter] prevent crash on invalid/deleted Retweets ([#2225](https://github.com/mikf/gallery-dl/issues/2225))
diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py
index fd261922..e8bee378 100644
--- a/gallery_dl/extractor/gelbooru.py
+++ b/gallery_dl/extractor/gelbooru.py
@@ -33,28 +33,20 @@ class GelbooruBase():
     def _pagination(self, params):
         params["pid"] = self.page_start
         params["limit"] = self.per_page
+        limit = self.per_page // 2
 
-        post = None
         while True:
-            try:
-                posts = self._api_request(params)
-            except ValueError:
-                if "tags" not in params or post is None:
-                    raise
-                taglist = [tag for tag in params["tags"].split()
-                           if not tag.startswith("id:<")]
-                taglist.append("id:<" + str(post.attrib["id"]))
-                params["tags"] = " ".join(taglist)
-                params["pid"] = 0
-                continue
+            posts = self._api_request(params)
 
-            post = None
             for post in posts:
                 yield post
 
-            if len(posts) < self.per_page:
+            if len(posts) < limit:
                 return
-            params["pid"] += 1
+
+            if "pid" in params:
+                del params["pid"]
+            params["tags"] = "{} id:<{}".format(self.tags, post["id"])
 
     @staticmethod
     def _file_url(post):
@@ -81,9 +73,12 @@ class GelbooruTagExtractor(GelbooruBase,
         ("https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", {
             "count": 5,
         }),
-        ("https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", {
-            "options": (("api", False),),
-            "count": 5,
+        ("https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", {
+            "range": "196-204",
+            "url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf",
+            "pattern": r"https://img\d\.gelbooru\.com"
+                       r"/images/../../[0-9a-f]{32}\.jpg",
+            "count": 9,
         }),
     )
 
diff --git a/gallery_dl/version.py b/gallery_dl/version.py
index 1a399fa3..b9c21273 100644
--- a/gallery_dl/version.py
+++ b/gallery_dl/version.py
@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.20.3"
+__version__ = "1.20.4-dev"