[deviantart] fix pagination for Eclipse results (fixes #1444)

- don't crash on missing keys
- use fallback for invalid 'nextOffset' values
This commit is contained in:
Mike Fährmann
2021-04-09 15:16:56 +02:00
parent dee540050f
commit 457abf0e71

View File

@@ -1230,7 +1230,7 @@ class DeviantartEclipseAPI():
params = { params = {
"username" : user, "username" : user,
"offset" : offset, "offset" : offset,
"limit" : "24", "limit" : 24,
"scraps_folder": "true", "scraps_folder": "true",
} }
return self._pagination(endpoint, params) return self._pagination(endpoint, params)
@@ -1240,8 +1240,8 @@ class DeviantartEclipseAPI():
params = { params = {
"username": user, "username": user,
"moduleid": self._module_id_watching(user), "moduleid": self._module_id_watching(user),
"offset" : None, "offset" : offset,
"limit" : "24", "limit" : 24,
} }
return self._pagination(endpoint, params) return self._pagination(endpoint, params)
@@ -1260,14 +1260,23 @@ class DeviantartEclipseAPI():
except Exception: except Exception:
return {"error": response.text} return {"error": response.text}
def _pagination(self, endpoint, params=None): def _pagination(self, endpoint, params):
while True: while True:
data = self._call(endpoint, params) data = self._call(endpoint, params)
yield from data["results"]
if not data["hasMore"]: results = data.get("results")
if results is None:
return return
params["offset"] = data["nextOffset"] yield from results
if not data.get("hasMore"):
return
next_offset = data.get("nextOffset")
if next_offset:
params["offset"] = next_offset
else:
params["offset"] += params["limit"]
def _module_id_watching(self, user): def _module_id_watching(self, user):
url = "{}/{}/about".format(self.extractor.root, user) url = "{}/{}/about".format(self.extractor.root, user)