github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[e621] softcode the pagination threshold

Browse Source
This commit is contained in:
ClosedPort22
2022-12-17 21:06:47 +08:00
parent 9faa4ed738
commit dd4a4a3fa6
2 changed files with 17 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
docs/configuration.rst
View File

@@ -1004,18 +1004,21 @@ Description
Note: This requires 1 additional HTTP request for each post. Note: This requires 1 additional HTTP request for each post.
extractor.danbooru.pagination extractor.danbooru.threshold
----------------------------- ----------------------------
Type Type
``string`` ``string`` or ``int``
Default Default
``"length"`` ``"auto"``
Description Description
Controls when to stop paginating over API results. Stop paginating over API results if the length of a batch of returned
posts is less than the specified number. Defaults to the per-page limit
of the current instance, which is 320 for ``e621`` and 200 for
everything else.
* ``"length"``: Stop when the length of a batch of results is less than Note: Changing this setting is normally not necessary. When the value is
the page limit. greater than the per-page limit, gallery-dl will stop after the first
* ``"manual"``: Only stop when a batch of results is empty. batch. The value cannot be less than 1.
extractor.danbooru.ugoira extractor.danbooru.ugoira

9
gallery_dl/extractor/danbooru.py
View File

@@ -41,7 +41,11 @@ class DanbooruExtractor(BaseExtractor):
self.ugoira = self.config("ugoira", False) self.ugoira = self.config("ugoira", False)
self.external = self.config("external", False) self.external = self.config("external", False)
self.extended_metadata = self.config("metadata", False) self.extended_metadata = self.config("metadata", False)
self.strategy = self.config("pagination", "length") threshold = self.config("threshold")
if isinstance(threshold, int):
self.threshold = 1 if threshold < 1 else threshold
else:
self.threshold = self.per_page
username, api_key = self._get_auth_info() username, api_key = self._get_auth_info()
if username: if username:
@@ -118,7 +122,6 @@ class DanbooruExtractor(BaseExtractor):
def _pagination(self, endpoint, params, pagenum=False): def _pagination(self, endpoint, params, pagenum=False):
url = self.root + endpoint url = self.root + endpoint
limit = self.per_page if self.strategy == "length" else 1
params["limit"] = self.per_page params["limit"] = self.per_page
params["page"] = self.page_start params["page"] = self.page_start
@@ -128,7 +131,7 @@ class DanbooruExtractor(BaseExtractor):
posts = posts["posts"] posts = posts["posts"]
yield from posts yield from posts
if len(posts) < limit: if len(posts) < self.threshold:
return return
if pagenum: if pagenum: