[twitter] prioritize tweet type checks (#3439)
Do not consider a tweet seen before applying 'retweet', 'quote' and 'reply' checks. Otherwise the original tweets will also be skipped if the "derivative" tweets and the original tweets are from the same user.
This commit is contained in:
@@ -75,11 +75,6 @@ class TwitterExtractor(Extractor):
|
||||
else:
|
||||
data = tweet
|
||||
|
||||
if seen_tweets is not None:
|
||||
if data["id_str"] in seen_tweets:
|
||||
continue
|
||||
seen_tweets.add(data["id_str"])
|
||||
|
||||
if not self.retweets and "retweeted_status_id_str" in data:
|
||||
self.log.debug("Skipping %s (retweet)", data["id_str"])
|
||||
continue
|
||||
@@ -97,6 +92,13 @@ class TwitterExtractor(Extractor):
|
||||
self.log.debug("Skipping %s (reply)", data["id_str"])
|
||||
continue
|
||||
|
||||
if seen_tweets is not None:
|
||||
if data["id_str"] in seen_tweets:
|
||||
self.log.debug(
|
||||
"Skipping %s (previously seen)", data["id_str"])
|
||||
continue
|
||||
seen_tweets.add(data["id_str"])
|
||||
|
||||
files = []
|
||||
if "extended_entities" in data:
|
||||
self._extract_media(
|
||||
|
||||
Reference in New Issue
Block a user