[twitter] fix pagination end condition

Some timelines would cause an endless loop because 'has_more_items' is
always True, even if it would return the same list of tweets over and
over again.
This commit is contained in:
Mike Fährmann
2019-05-05 22:46:52 +02:00
parent 51e0e92429
commit 049e9fd6ce

View File

@@ -124,8 +124,12 @@ class TwitterExtractor(Extractor):
if not data["has_more_items"]:
return
params["max_position"] = text.extract(
tweet, 'data-tweet-id="', '"')[0]
position = text.parse_int(text.extract(
tweet, 'data-tweet-id="', '"')[0])
if "max_position" in params and position >= params["max_position"]:
return
params["max_position"] = position
class TwitterTimelineExtractor(TwitterExtractor):