[twitter] implement 'users' option (#1337)

This commit is contained in:
Mike Fährmann
2021-03-15 22:55:24 +01:00
parent 847e9b0ed7
commit 3378b39719
3 changed files with 29 additions and 8 deletions

View File

@@ -1650,6 +1650,20 @@ Description
Extract `TwitPic <https://twitpic.com/>`__ embeds. Extract `TwitPic <https://twitpic.com/>`__ embeds.
extractor.twitter.users
-----------------------
Type
``string``
Default
``"timeline"``
Description
Output format for user results from
``following`` and ``list-members`` queries.
* ``"timeline"``: ``https://twitter.com/i/user/USERID``
* ``"media"``: ``https://twitter.com/id:USERID/media``
extractor.twitter.videos extractor.twitter.videos
------------------------ ------------------------
Type Type

View File

@@ -254,6 +254,7 @@
"replies": true, "replies": true,
"retweets": true, "retweets": true,
"twitpic": false, "twitpic": false,
"users": "timeline",
"videos": true "videos": true
}, },
"unsplash": "unsplash":

View File

@@ -224,6 +224,18 @@ class TwitterExtractor(Extractor):
} }
return cache[uid] return cache[uid]
def _users_result(self, users):
if self.config("users") == "media":
cls = TwitterMediaExtractor
fmt = "{}/id:{}/media".format
else:
cls = TwitterTimelineExtractor
fmt = "{}/i/user/{}".format
for user in users:
user["_extractor"] = cls
yield Message.Queue, fmt(self.root, user["rest_id"]), user
def metadata(self): def metadata(self):
"""Return general metadata""" """Return general metadata"""
return {} return {}
@@ -356,10 +368,7 @@ class TwitterListMembersExtractor(TwitterExtractor):
def items(self): def items(self):
self.login() self.login()
for user in TwitterAPI(self).list_members(self.user): return self._users_result(TwitterAPI(self).list_members(self.user))
user["_extractor"] = TwitterTimelineExtractor
url = "{}/i/user/{}".format(self.root, user["rest_id"])
yield Message.Queue, url, user
class TwitterFollowingExtractor(TwitterExtractor): class TwitterFollowingExtractor(TwitterExtractor):
@@ -373,10 +382,7 @@ class TwitterFollowingExtractor(TwitterExtractor):
def items(self): def items(self):
self.login() self.login()
for user in TwitterAPI(self).user_following(self.user): return self._users_result(TwitterAPI(self).user_following(self.user))
user["_extractor"] = TwitterTimelineExtractor
url = "{}/i/user/{}".format(self.root, user["rest_id"])
yield Message.Queue, url, user
class TwitterSearchExtractor(TwitterExtractor): class TwitterSearchExtractor(TwitterExtractor):