From 7c8eab8d52eb2ef434789f30c6465021efef0dd1 Mon Sep 17 00:00:00 2001 From: ClosedPort22 <44864697+ClosedPort22@users.noreply.github.com> Date: Fri, 30 Dec 2022 20:39:36 +0800 Subject: [PATCH 1/2] [twitter] implement 'syndication=extended' to be able to fetch extended user metadata --- docs/configuration.rst | 11 +++++++++-- gallery_dl/extractor/twitter.py | 7 +++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 2e5e7f6a..dfe3f19b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2655,11 +2655,18 @@ Description extractor.twitter.syndication ----------------------------- Type - ``bool`` + ``bool`` or ``string`` Default ``false`` Description - Retrieve age-restricted content using Twitter's syndication API. + Controls how to retrieve age-restricted content when not logged in. + + * ``false``: Skip age-restricted Tweets. + * ``true``: Download using Twitter's syndication API. + * ``"extended"``: Try to fetch Tweet metadata using the normal API + in addition to the syndication API. This requires additional HTTP + requests in some cases (e.g. when `retweets `_ + is enabled). extractor.twitter.logout diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 22aa78e6..270848c8 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -39,6 +39,7 @@ class TwitterExtractor(Extractor): self.videos = self.config("videos", True) self.cards = self.config("cards", False) self.cards_blacklist = self.config("cards-blacklist") + self.syndication = self.config("syndication") self._user = self._user_obj = None self._user_cache = {} self._init_sizes() @@ -297,8 +298,11 @@ class TwitterExtractor(Extractor): except KeyError: pass + # try to fetch extended user data if "legacy" in user: user = user["legacy"] + elif "statuses_count" not in user and self.syndication == "extended": + user = self.api.user_by_screen_name(user["screen_name"])["legacy"] uget = user.get entities = user["entities"] @@ -991,7 +995,6 @@ class TwitterAPI(): } self._nsfw_warning = True - self._syndication = extractor.config("syndication") self._json_dumps = json.JSONEncoder(separators=(",", ":")).encode cookies = extractor.session.cookies @@ -1494,7 +1497,7 @@ class TwitterAPI(): tweet_id = entry["entryId"].rpartition("-")[2] if text.startswith("Age-restricted"): - if self._syndication: + if self.extractor.syndication: return self._syndication_tweet(tweet_id) elif self._nsfw_warning: self._nsfw_warning = False From 6853b14be37a042a7ed169745511d362b44c0454 Mon Sep 17 00:00:00 2001 From: ClosedPort22 <44864697+ClosedPort22@users.noreply.github.com> Date: Mon, 2 Jan 2023 21:03:01 +0800 Subject: [PATCH 2/2] [twitter] apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mike Fährmann --- docs/configuration.rst | 9 +++++---- gallery_dl/extractor/twitter.py | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index dfe3f19b..1749dc61 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2655,7 +2655,8 @@ Description extractor.twitter.syndication ----------------------------- Type - ``bool`` or ``string`` + * ``bool`` + * ``string`` Default ``false`` Description @@ -2664,9 +2665,9 @@ Description * ``false``: Skip age-restricted Tweets. * ``true``: Download using Twitter's syndication API. * ``"extended"``: Try to fetch Tweet metadata using the normal API - in addition to the syndication API. This requires additional HTTP - requests in some cases (e.g. when `retweets `_ - is enabled). + in addition to the syndication API. This requires additional HTTP + requests in some cases (e.g. when `retweets `_ + are enabled). extractor.twitter.logout diff --git a/gallery_dl/extractor/twitter.py b/gallery_dl/extractor/twitter.py index 270848c8..aad251df 100644 --- a/gallery_dl/extractor/twitter.py +++ b/gallery_dl/extractor/twitter.py @@ -298,10 +298,10 @@ class TwitterExtractor(Extractor): except KeyError: pass - # try to fetch extended user data if "legacy" in user: user = user["legacy"] elif "statuses_count" not in user and self.syndication == "extended": + # try to fetch extended user data user = self.api.user_by_screen_name(user["screen_name"])["legacy"] uget = user.get @@ -995,6 +995,7 @@ class TwitterAPI(): } self._nsfw_warning = True + self._syndication = self.extractor.syndication self._json_dumps = json.JSONEncoder(separators=(",", ":")).encode cookies = extractor.session.cookies @@ -1497,7 +1498,7 @@ class TwitterAPI(): tweet_id = entry["entryId"].rpartition("-")[2] if text.startswith("Age-restricted"): - if self.extractor.syndication: + if self._syndication: return self._syndication_tweet(tweet_id) elif self._nsfw_warning: self._nsfw_warning = False