[twitter] prioritize tweet type checks (#3439)

Do not consider a tweet seen before applying 'retweet', 'quote' and
'reply' checks. Otherwise the original tweets will also be skipped if
the "derivative" tweets and the original tweets are from the same user.
This commit is contained in:
ClosedPort22
2022-12-24 12:35:44 +08:00
parent 2d7d80d302
commit 250d35107c

View File

@@ -75,11 +75,6 @@ class TwitterExtractor(Extractor):
else:
data = tweet
if seen_tweets is not None:
if data["id_str"] in seen_tweets:
continue
seen_tweets.add(data["id_str"])
if not self.retweets and "retweeted_status_id_str" in data:
self.log.debug("Skipping %s (retweet)", data["id_str"])
continue
@@ -97,6 +92,13 @@ class TwitterExtractor(Extractor):
self.log.debug("Skipping %s (reply)", data["id_str"])
continue
if seen_tweets is not None:
if data["id_str"] in seen_tweets:
self.log.debug(
"Skipping %s (previously seen)", data["id_str"])
continue
seen_tweets.add(data["id_str"])
files = []
if "extended_entities" in data:
self._extract_media(