diff --git a/docs/configuration.rst b/docs/configuration.rst index 3a194aa6..65c9f049 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -6328,12 +6328,21 @@ Description extractor.twitter.search-limit ------------------------------ Type - ``integer`` + * ``integer`` + * ``list`` of ``integers`` Default ``20`` +Example + ``[50, 20, 10, 5, 2]`` Description Number of requested results per search query. + When given as a ``list``, + start with the first element as ``count`` parameter + and switch to the next element when + `search-stop `__ + is reached. + extractor.twitter.search-pagination ----------------------------------- diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 41685d7e..0e5c2224 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -1712,7 +1712,7 @@ class TwitterAPI(): self.client_transaction.generate_transaction_id(method, path) def _call(self, endpoint, params, method="GET", auth=True, root=None): - url = (root or self.root) + endpoint + url = (self.root if root is None else root) + endpoint while True: if auth: @@ -1893,6 +1893,14 @@ class TwitterAPI(): stop_tweets_max = stop_tweets api_retries = None + if isinstance(count := variables.get("count"), list): + count = count.copy() + count.reverse() + self.log.debug("Using 'count: %s'", count[-1]) + variables["count"] = count.pop() + else: + count = False + params = {"variables": None} if cursor := extr._init_cursor(): variables["cursor"] = cursor @@ -2126,15 +2134,19 @@ class TwitterAPI(): if tweet: stop_tweets = stop_tweets_max last_tweet = tweet - else: - if stop_tweets <= 0: + elif stop_tweets <= 0: + if not count: return extr._update_cursor(None) + self.log.debug("Switching to 'count: %s'", count[-1]) + variables["count"] = count.pop() + else: self.log.debug( "No Tweet results (%s/%s)", stop_tweets_max - stop_tweets + 1, stop_tweets_max) stop_tweets -= 1 if not cursor or cursor == variables.get("cursor"): + self.log.debug("No continuation cursor") return extr._update_cursor(None) if update_variables is None: