Merge pull request #3413 from ClosedPort22/e621-manual-pagination

[e621] implement manual pagination

docs/configuration.rst

@@ -1004,6 +1004,23 @@ Description
     Note: This requires 1 additional HTTP request for each post.
 
 
+extractor.danbooru.threshold
+----------------------------
+Type
+    ``string`` or ``int``
+Default
+    ``"auto"``
+Description
+    Stop paginating over API results if the length of a batch of returned
+    posts is less than the specified number. Defaults to the per-page limit
+    of the current instance, which is 320 for ``e621`` and 200 for
+    everything else.
+
+    Note: Changing this setting is normally not necessary. When the value is
+    greater than the per-page limit, gallery-dl will stop after the first
+    batch. The value cannot be less than 1.
+
+
 extractor.danbooru.ugoira
 -------------------------
 Type

gallery_dl/extractor/danbooru.py

@@ -41,6 +41,11 @@ class DanbooruExtractor(BaseExtractor):
         self.ugoira = self.config("ugoira", False)
         self.external = self.config("external", False)
         self.extended_metadata = self.config("metadata", False)
+        threshold = self.config("threshold")
+        if isinstance(threshold, int):
+            self.threshold = 1 if threshold < 1 else threshold
+        else:
+            self.threshold = self.per_page
 
         username, api_key = self._get_auth_info()
         if username:
@@ -126,7 +131,7 @@ class DanbooruExtractor(BaseExtractor):
                 posts = posts["posts"]
             yield from posts
 
-            if len(posts) < self.per_page:
+            if len(posts) < self.threshold:
                 return
 
             if pagenum: