[twitter] fix pagination end condition
Some timelines would cause an endless loop because 'has_more_items' is always True, even if it would return the same list of tweets over and over again.
This commit is contained in:
@@ -124,8 +124,12 @@ class TwitterExtractor(Extractor):
|
||||
|
||||
if not data["has_more_items"]:
|
||||
return
|
||||
params["max_position"] = text.extract(
|
||||
tweet, 'data-tweet-id="', '"')[0]
|
||||
|
||||
position = text.parse_int(text.extract(
|
||||
tweet, 'data-tweet-id="', '"')[0])
|
||||
if "max_position" in params and position >= params["max_position"]:
|
||||
return
|
||||
params["max_position"] = position
|
||||
|
||||
|
||||
class TwitterTimelineExtractor(TwitterExtractor):
|
||||
|
||||
Reference in New Issue
Block a user