[twitter] improve strategy for user URLs (#2665)

- use '/with_replies' when appropriate
- consider 'text-tweets'
- build search query as necessary
This commit is contained in:
Mike Fährmann
2022-06-11 18:07:07 +02:00
parent 45c980daf0
commit 0add1fc090

View File

@@ -418,12 +418,15 @@ class TwitterTimelineExtractor(TwitterExtractor):
self.user = "id:" + user_id self.user = "id:" + user_id
def tweets(self): def tweets(self):
tweets = (self.api.user_tweets(self.user) if self.retweets else if self.retweets or self.textonly:
self.api.user_media(self.user)) tweets = (self.api.user_tweets_and_replies if self.replies else
self.api.user_tweets)
else:
tweets = self.api.user_media
# yield initial batch of (media) tweets # yield initial batch of (media) tweets
tweet = None tweet = None
for tweet in tweets: for tweet in tweets(self.user):
yield tweet yield tweet
if tweet is None: if tweet is None:
@@ -442,12 +445,17 @@ class TwitterTimelineExtractor(TwitterExtractor):
if "legacy" in tweet: if "legacy" in tweet:
tweet = tweet["legacy"] tweet = tweet["legacy"]
# build search query
query = "from:{} max_id:{}".format(username, tweet["id_str"])
if self.retweets:
query += " include:retweets include:nativeretweets"
if not self.textonly:
query += (" (filter:images OR"
" filter:native_video OR"
" card_name:animated_gif)")
# yield search results starting from last tweet id # yield search results starting from last tweet id
yield from self.api.search_adaptive( yield from self.api.search_adaptive(query)
"from:{} include:retweets include:nativeretweets max_id:{} "
"filter:images OR card_name:animated_gif OR filter:native_video"
.format(username, tweet["id_str"])
)
class TwitterTweetsExtractor(TwitterExtractor): class TwitterTweetsExtractor(TwitterExtractor):