From c60ebc6519b1c4ddcd21d9f79d288d6f004a01a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Sun, 25 Feb 2024 03:36:00 +0100
Subject: [PATCH] [deviantart] improve fetching extended metadata (#5175)

use multiple metadata API calls per chunk of deviations if necessary
---
 gallery_dl/extractor/deviantart.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py
index 5a0b2c87..c0c0f9a3 100644
--- a/gallery_dl/extractor/deviantart.py
+++ b/gallery_dl/extractor/deviantart.py
@@ -1079,7 +1079,6 @@ class DeviantartOAuthAPI():
         self.headers = {"dA-minor-version": "20200519"}
         self._warn_429 = True
 
-        self.limit = None
         self.delay = extractor.config("wait-min", 0)
         self.delay_min = max(2, self.delay)
 
@@ -1140,6 +1139,7 @@ class DeviantartOAuthAPI():
                 self.limit = 50
         else:
             self.metadata = False
+            self.limit = None
 
         self.log.debug(
             "Using %s API credentials (client-id %s)",
@@ -1286,7 +1286,7 @@ class DeviantartOAuthAPI():
         )
         return self._call(
             endpoint,
-            params=self._metadata_params.copy(),
+            params=self._metadata_params,
             public=self._metadata_public,
         )["metadata"]
 
@@ -1449,8 +1449,9 @@ class DeviantartOAuthAPI():
         warn = True
         if public is None:
             public = self.public
+
         if self.limit and params["limit"] > self.limit:
-            params["limit"] = self.limit
+            params["limit"] = (params["limit"] // self.limit) * self.limit
 
         while True:
             data = self._call(endpoint, params=params, public=public)
@@ -1523,6 +1524,15 @@ class DeviantartOAuthAPI():
 
     def _metadata(self, deviations):
         """Add extended metadata to each deviation object"""
+        if len(deviations) <= self.limit:
+            self._metadata_batch(deviations)
+        else:
+            n = self.limit
+            for index in range(0, len(deviations), n):
+                self._metadata_batch(deviations[index:index+n])
+
+    def _metadata_batch(self, deviations):
+        """Fetch extended metadata for a single batch of deviations"""
         for deviation, metadata in zip(
                 deviations, self.deviation_metadata(deviations)):
             deviation.update(metadata)