[twitter] add 'replies' option (closes #705)
This commit is contained in:
@@ -1169,12 +1169,21 @@ Description Extract tweet text as ``content`` metadata.
|
|||||||
=========== =====
|
=========== =====
|
||||||
|
|
||||||
|
|
||||||
|
extractor.twitter.replies
|
||||||
|
-------------------------
|
||||||
|
=========== =====
|
||||||
|
Type ``bool``
|
||||||
|
Default ``true``
|
||||||
|
Description Extract media from replies to other Tweets.
|
||||||
|
=========== =====
|
||||||
|
|
||||||
|
|
||||||
extractor.twitter.retweets
|
extractor.twitter.retweets
|
||||||
--------------------------
|
--------------------------
|
||||||
=========== =====
|
=========== =====
|
||||||
Type ``bool``
|
Type ``bool``
|
||||||
Default ``true``
|
Default ``true``
|
||||||
Description Extract images from retweets.
|
Description Extract media from Retweets.
|
||||||
=========== =====
|
=========== =====
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -154,6 +154,7 @@
|
|||||||
"twitter":
|
"twitter":
|
||||||
{
|
{
|
||||||
"content": false,
|
"content": false,
|
||||||
|
"replies": true,
|
||||||
"retweets": true,
|
"retweets": true,
|
||||||
"twitpic": false,
|
"twitpic": false,
|
||||||
"videos": true
|
"videos": true
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ class TwitterExtractor(Extractor):
|
|||||||
self._user_dict = None
|
self._user_dict = None
|
||||||
self.logged_in = False
|
self.logged_in = False
|
||||||
self.retweets = self.config("retweets", True)
|
self.retweets = self.config("retweets", True)
|
||||||
|
self.replies = self.config("replies", True)
|
||||||
self.twitpic = self.config("twitpic", False)
|
self.twitpic = self.config("twitpic", False)
|
||||||
self.content = self.config("content", False)
|
self.content = self.config("content", False)
|
||||||
self.videos = self.config("videos", True)
|
self.videos = self.config("videos", True)
|
||||||
@@ -48,7 +49,9 @@ class TwitterExtractor(Extractor):
|
|||||||
|
|
||||||
for tweet in self.tweets():
|
for tweet in self.tweets():
|
||||||
data = self._data_from_tweet(tweet)
|
data = self._data_from_tweet(tweet)
|
||||||
if not data or not self.retweets and data["retweet_id"]:
|
if not data or \
|
||||||
|
not self.retweets and data["retweet_id"] or \
|
||||||
|
not self.replies and data["reply"]:
|
||||||
continue
|
continue
|
||||||
data.update(metadata)
|
data.update(metadata)
|
||||||
|
|
||||||
@@ -370,6 +373,11 @@ class TwitterTweetExtractor(TwitterExtractor):
|
|||||||
"options": (("videos", "ytdl"),),
|
"options": (("videos", "ytdl"),),
|
||||||
"pattern": r"ytdl:https://twitter.com/i/web.+/1103767554424598528",
|
"pattern": r"ytdl:https://twitter.com/i/web.+/1103767554424598528",
|
||||||
}),
|
}),
|
||||||
|
# 'replies' option (#705)
|
||||||
|
("https://twitter.com/tyson_hesse/status/1103767554424598528", {
|
||||||
|
"options": (("replies", False),),
|
||||||
|
"count": 0,
|
||||||
|
}),
|
||||||
# /i/web/ URL
|
# /i/web/ URL
|
||||||
("https://twitter.com/i/web/status/1155074198240292865", {
|
("https://twitter.com/i/web/status/1155074198240292865", {
|
||||||
"pattern": r"https://pbs.twimg.com/media/EAel0vUUYAAZ4Bq.jpg:orig",
|
"pattern": r"https://pbs.twimg.com/media/EAel0vUUYAAZ4Bq.jpg:orig",
|
||||||
|
|||||||
Reference in New Issue
Block a user