[twitter] add extractor for followed users (#1337)
https://twitter.com/USER/following or https://twitter.com/id:USERID/following
This commit is contained in:
@@ -362,6 +362,23 @@ class TwitterListMembersExtractor(TwitterExtractor):
|
|||||||
yield Message.Queue, url, user
|
yield Message.Queue, url, user
|
||||||
|
|
||||||
|
|
||||||
|
class TwitterFollowingExtractor(TwitterExtractor):
|
||||||
|
"""Extractor for followed users"""
|
||||||
|
subcategory = "following"
|
||||||
|
pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/following(?!\w)"
|
||||||
|
test = (
|
||||||
|
("https://twitter.com/supernaturepics/following"),
|
||||||
|
("https://www.twitter.com/id:2976459548/following"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def items(self):
|
||||||
|
self.login()
|
||||||
|
for user in TwitterAPI(self).user_following(self.user):
|
||||||
|
user["_extractor"] = TwitterTimelineExtractor
|
||||||
|
url = "{}/i/user/{}".format(self.root, user["rest_id"])
|
||||||
|
yield Message.Queue, url, user
|
||||||
|
|
||||||
|
|
||||||
class TwitterSearchExtractor(TwitterExtractor):
|
class TwitterSearchExtractor(TwitterExtractor):
|
||||||
"""Extractor for all images from a search timeline"""
|
"""Extractor for all images from a search timeline"""
|
||||||
subcategory = "search"
|
subcategory = "search"
|
||||||
@@ -577,16 +594,6 @@ class TwitterAPI():
|
|||||||
params["spelling_corrections"] = "1"
|
params["spelling_corrections"] = "1"
|
||||||
return self._pagination(endpoint, params)
|
return self._pagination(endpoint, params)
|
||||||
|
|
||||||
def list_members(self, list_id):
|
|
||||||
endpoint = "/graphql/tA7h9hy4U0Yc9COfIOh3qQ/ListMembers"
|
|
||||||
variables = {
|
|
||||||
"listId": list_id,
|
|
||||||
"count" : 20,
|
|
||||||
"withTweetResult": False,
|
|
||||||
"withUserResult" : False,
|
|
||||||
}
|
|
||||||
return self._pagination_members(endpoint, variables)
|
|
||||||
|
|
||||||
def list_by_rest_id(self, list_id):
|
def list_by_rest_id(self, list_id):
|
||||||
endpoint = "/graphql/18MAHTcDU-TdJSjWWmoH7w/ListByRestId"
|
endpoint = "/graphql/18MAHTcDU-TdJSjWWmoH7w/ListByRestId"
|
||||||
params = {"variables": '{"listId":"' + list_id + '"'
|
params = {"variables": '{"listId":"' + list_id + '"'
|
||||||
@@ -596,6 +603,31 @@ class TwitterAPI():
|
|||||||
except KeyError:
|
except KeyError:
|
||||||
raise exception.NotFoundError("list")
|
raise exception.NotFoundError("list")
|
||||||
|
|
||||||
|
def list_members(self, list_id):
|
||||||
|
endpoint = "/graphql/tA7h9hy4U0Yc9COfIOh3qQ/ListMembers"
|
||||||
|
variables = {
|
||||||
|
"listId": list_id,
|
||||||
|
"count" : 100,
|
||||||
|
"withTweetResult": False,
|
||||||
|
"withUserResult" : False,
|
||||||
|
}
|
||||||
|
return self._pagination_graphql(
|
||||||
|
endpoint, variables, "list", "members_timeline")
|
||||||
|
|
||||||
|
def user_following(self, screen_name):
|
||||||
|
endpoint = "/graphql/Q_QTiPvoXwsA13eoA7okIQ/Following"
|
||||||
|
variables = {
|
||||||
|
"userId": self._user_id_by_screen_name(screen_name),
|
||||||
|
"count" : 100,
|
||||||
|
"withTweetResult": False,
|
||||||
|
"withUserResult" : False,
|
||||||
|
"withTweetQuoteCount" : False,
|
||||||
|
"withHighlightedLabel" : False,
|
||||||
|
"includePromotedContent": False,
|
||||||
|
}
|
||||||
|
return self._pagination_graphql(
|
||||||
|
endpoint, variables, "user", "following_timeline")
|
||||||
|
|
||||||
def user_by_screen_name(self, screen_name):
|
def user_by_screen_name(self, screen_name):
|
||||||
endpoint = "/graphql/hc-pka9A7gyS3xODIafnrQ/UserByScreenName"
|
endpoint = "/graphql/hc-pka9A7gyS3xODIafnrQ/UserByScreenName"
|
||||||
params = {"variables": '{"screen_name":"' + screen_name + '"'
|
params = {"variables": '{"screen_name":"' + screen_name + '"'
|
||||||
@@ -728,15 +760,15 @@ class TwitterAPI():
|
|||||||
return
|
return
|
||||||
params["cursor"] = cursor
|
params["cursor"] = cursor
|
||||||
|
|
||||||
def _pagination_members(self, endpoint, variables):
|
def _pagination_graphql(self, endpoint, variables, key, timeline):
|
||||||
while True:
|
while True:
|
||||||
cursor = entry = stop = None
|
cursor = entry = stop = None
|
||||||
params = {"variables": json.dumps(variables)}
|
params = {"variables": json.dumps(variables)}
|
||||||
data = self._call(endpoint, params)
|
data = self._call(endpoint, params)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
instructions = (data["data"]["list"]["members_timeline"]
|
instructions = \
|
||||||
["timeline"]["instructions"])
|
data["data"][key][timeline]["timeline"]["instructions"]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise exception.AuthorizationError()
|
raise exception.AuthorizationError()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user