[twitter] add 'search-stop' option (#8173)
and rename 'pagination-search' to 'search-pagination'
This commit is contained in:
@@ -5631,22 +5631,6 @@ Description
|
||||
Logout and retry as guest when access to another user's Tweets is blocked.
|
||||
|
||||
|
||||
extractor.twitter.pagination-search
|
||||
-----------------------------------
|
||||
Type
|
||||
``string``
|
||||
Default
|
||||
``"cursor"``
|
||||
Description
|
||||
Selects how to paginate over search results.
|
||||
|
||||
``"cursor"``
|
||||
Use ``cursor`` values provided by the API
|
||||
``"id"`` | ``"max_id"`` | ``"tweet_id"``
|
||||
Update the ``max_id`` search query parameter
|
||||
to the Tweet ID value of the last retrieved Tweet.
|
||||
|
||||
|
||||
extractor.twitter.pinned
|
||||
------------------------
|
||||
Type
|
||||
@@ -5743,6 +5727,34 @@ Description
|
||||
will be taken from the original Tweets, not the Retweets.
|
||||
|
||||
|
||||
extractor.twitter.search-pagination
|
||||
-----------------------------------
|
||||
Type
|
||||
``string``
|
||||
Default
|
||||
``"cursor"``
|
||||
Description
|
||||
Selects how to paginate over search results.
|
||||
|
||||
``"cursor"``
|
||||
Use ``cursor`` values provided by the API
|
||||
``"max_id"`` | ``"maxid"`` | ``"id"``
|
||||
Update the ``max_id`` search query parameter
|
||||
to the Tweet ID value of the last retrieved Tweet.
|
||||
|
||||
|
||||
extractor.twitter.search-stop
|
||||
-----------------------------
|
||||
Type
|
||||
``integer``
|
||||
Default
|
||||
* ``3`` if `search-pagination <extractor.twitter.search-pagination_>`__ is set to ``"cursor"``
|
||||
* ``0`` otherwise
|
||||
Description
|
||||
Selects how many empty search result batches
|
||||
to receive before stopping.
|
||||
|
||||
|
||||
extractor.twitter.timeline.strategy
|
||||
-----------------------------------
|
||||
Type
|
||||
|
||||
@@ -812,13 +812,14 @@
|
||||
"include" : ["timeline"],
|
||||
"locked" : "abort",
|
||||
"logout" : true,
|
||||
"pagination-search": "cursor",
|
||||
"pinned" : false,
|
||||
"quoted" : false,
|
||||
"ratelimit" : "wait",
|
||||
"relogin" : true,
|
||||
"replies" : true,
|
||||
"retweets" : false,
|
||||
"search-pagination": "cursor",
|
||||
"search-stop" : "auto",
|
||||
"size" : ["orig", "4096x4096", "large", "medium", "small"],
|
||||
"text-tweets" : false,
|
||||
"tweet-endpoint": "auto",
|
||||
|
||||
@@ -1460,16 +1460,20 @@ class TwitterAPI():
|
||||
"withGrokTranslatedBio": False,
|
||||
}
|
||||
|
||||
if self.extractor.config("pagination-search") in (
|
||||
"id", "tweet_id", "max_id"):
|
||||
if self.extractor.config("search-pagination") in (
|
||||
"max_id", "maxid", "id"):
|
||||
update_variables = self._update_variables_search
|
||||
else:
|
||||
update_variables = None
|
||||
|
||||
stop_tweets = self.extractor.config("search-stop")
|
||||
if stop_tweets is None or stop_tweets == "auto":
|
||||
stop_tweets = 3 if update_variables is None else 0
|
||||
|
||||
return self._pagination_tweets(
|
||||
endpoint, variables,
|
||||
("search_by_raw_query", "search_timeline", "timeline"),
|
||||
stop_tweets=3, update_variables=update_variables)
|
||||
stop_tweets=stop_tweets, update_variables=update_variables)
|
||||
|
||||
def community_query(self, community_id):
|
||||
endpoint = "/graphql/2W09l7nD7ZbxGQHXvfB22w/CommunityQuery"
|
||||
|
||||
Reference in New Issue
Block a user