From 9683d79bb77527ff0dddc38da97c9bde563a62e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 16 Jan 2023 14:58:30 +0100 Subject: [PATCH] [twitter] "fix" search pagination (#3536, #3534) - properly process instructions - do not expect a predetermined instruction order --- gallery_dl/extractor/twitter.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index ef30c24e..e8a1754b 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -1245,17 +1245,28 @@ class TwitterAPI(): while True: data = self._call(endpoint, params) - instr = data["timeline"]["instructions"] - if not instr: + instructions = data["timeline"]["instructions"] + if not instructions: return tweets = data["globalObjects"]["tweets"] users = data["globalObjects"]["users"] tweet_id = cursor = None tweet_ids = [] + entries = () + + # process instructions + for instr in instructions: + if "addEntries" in instr: + entries = instr["addEntries"]["entries"] + elif "replaceEntry" in instr: + entry = instr["replaceEntry"]["entry"] + if entry["entryId"].startswith("cursor-bottom-"): + cursor = (entry["content"]["operation"] + ["cursor"]["value"]) # collect tweet IDs and cursor value - for entry in instr[0]["addEntries"]["entries"]: + for entry in entries: entry_startswith = entry["entryId"].startswith if entry_startswith(("tweet-", "sq-I-t-")): @@ -1315,11 +1326,7 @@ class TwitterAPI(): quoted["quoted_by_id_str"] = tweet["id_str"] yield quoted - # update cursor value - if "replaceEntry" in instr[-1] : - cursor = (instr[-1]["replaceEntry"]["entry"] - ["content"]["operation"]["cursor"]["value"]) - + # stop on empty response if not cursor or (not tweets and not tweet_id): return params["cursor"] = cursor