[deviantart] implement 'pagination' option (#2488)

docs/configuration.rst

@@ -1077,6 +1077,19 @@ Description
     everything else (archives, etc.).
 
 
+extractor.deviantart.pagination
+-------------------------------
+Type
+    ``string``
+Default
+    ``"api"``
+Description
+    Controls when to stop paginating over API results.
+
+    * ``"api"``: Trust the API and stop when ``has_more`` is ``false``.
+    * ``"manual"``: Disregard ``has_more`` and only stop when a batch of results is empty.
+
+
 extractor.deviantart.refresh-token
 ----------------------------------
 Type

gallery_dl/extractor/deviantart.py

@@ -1018,6 +1018,7 @@ class DeviantartOAuthAPI():
 
         self.folders = extractor.config("folders", False)
         self.metadata = extractor.extra or extractor.config("metadata", False)
+        self.strategy = extractor.config("pagination")
 
         self.client_id = extractor.config("client-id")
         if self.client_id:
@@ -1309,14 +1310,20 @@ class DeviantartOAuthAPI():
                     self._folders(results)
             yield from results
 
-            if not data["has_more"]:
+            if not data["has_more"] and (
+                    self.strategy != "manual" or not results):
                 return
+
             if "next_cursor" in data:
                 params["offset"] = None
                 params["cursor"] = data["next_cursor"]
-            else:
+            elif data["next_offset"] is not None:
                 params["offset"] = data["next_offset"]
                 params["cursor"] = None
+            else:
+                if params.get("offset") is None:
+                    return
+                params["offset"] = int(params["offset"]) + len(results)
 
     def _pagination_list(self, endpoint, params, key="results"):
         result = []