[twitter] remove 'syndication' option (#3889)

docs/configuration.rst

@@ -3262,8 +3262,6 @@ Description
     for each Tweet in said timeline.
 
     Note: This requires at least 1 additional API call per initial Tweet.
-    Age-restricted replies cannot be expanded when using the
-    `syndication <extractor.twitter.syndication_>`__ API.
 
 
 extractor.twitter.include
@@ -3331,30 +3329,6 @@ Description
     ``4096x4096``, ``orig``, ``large``, ``medium``, and ``small``.
 
 
-extractor.twitter.syndication
------------------------------
-Type
-    * ``bool``
-    * ``string``
-Default
-    ``false``
-Description
-    Controls how to retrieve age-restricted content when not logged in.
-
-    * ``false``: Skip age-restricted Tweets.
-    * ``true``: Download using Twitter's syndication API.
-    * ``"extended"``: Try to fetch Tweet metadata using the normal API
-      in addition to the syndication API. This requires additional HTTP
-      requests in some cases (e.g. when `retweets <extractor.twitter.retweets_>`_
-      are enabled).
-
-    Note: This does not apply to search results (including
-    `timeline strategies <extractor.twitter.timeline.strategy_>`__).
-    To retrieve such content from search results, you must log in and
-    disable "Hide sensitive content" in your `search settings
-    <https://twitter.com/settings/search>`__.
-
-
 extractor.twitter.logout
 ------------------------
 Type

gallery_dl/extractor/twitter.py

@@ -45,7 +45,6 @@ class TwitterExtractor(Extractor):
         self.cards = self.config("cards", False)
         self.ads = self.config("ads", False)
         self.cards_blacklist = self.config("cards-blacklist")
-        self.syndication = self.config("syndication")
 
         if not self.config("transform", True):
             self._transform_user = util.identity
@@ -367,9 +366,6 @@ class TwitterExtractor(Extractor):
 
         if "legacy" in user:
             user = user["legacy"]
-        elif "statuses_count" not in user and self.syndication == "extended":
-            # try to fetch extended user data
-            user = self.api.user_by_screen_name(user["screen_name"])["legacy"]
 
         uget = user.get
         if uget("withheld_scope"):
@@ -865,7 +861,6 @@ class TwitterAPI():
 
         self.root = "https://twitter.com/i/api"
         self._nsfw_warning = True
-        self._syndication = self.extractor.syndication
         self._json_dumps = json.JSONEncoder(separators=(",", ":")).encode
 
         cookies = extractor.cookies
@@ -1645,67 +1640,12 @@ class TwitterAPI():
         tweet_id = entry["entryId"].rpartition("-")[2]
 
         if text.startswith("Age-restricted"):
-            if self._syndication:
+            if self._nsfw_warning:
-                return self._syndication_tweet(tweet_id)
-            elif self._nsfw_warning:
                 self._nsfw_warning = False
                 self.extractor.log.warning('"%s"', text)
 
         self.extractor.log.debug("Skipping %s (\"%s\")", tweet_id, text)
 
-    def _syndication_tweet(self, tweet_id):
-        base_url = "https://cdn.syndication.twimg.com/tweet-result?id="
-        tweet = self.extractor.request(base_url + tweet_id).json()
-
-        tweet["user"]["description"] = ""
-        tweet["user"]["entities"] = {"description": {}}
-        tweet["user_id_str"] = tweet["user"]["id_str"]
-
-        if tweet["id_str"] != tweet_id:
-            tweet["retweeted_status_id_str"] = tweet["id_str"]
-            tweet["id_str"] = retweet_id = tweet_id
-        else:
-            retweet_id = None
-
-        # assume 'conversation_id' is the same as 'id' when the tweet
-        # is not a reply
-        if "conversation_id_str" not in tweet and \
-                "in_reply_to_status_id_str" not in tweet:
-            tweet["conversation_id_str"] = tweet["id_str"]
-
-        if int(tweet_id) < 300000000000000:
-            tweet["created_at"] = text.parse_datetime(
-                tweet["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime(
-                "%a %b %d %H:%M:%S +0000 %Y")
-
-        if "video" in tweet:
-            video = tweet["video"]
-            video["variants"] = (max(
-                (v for v in video["variants"] if v["type"] == "video/mp4"),
-                key=lambda v: text.parse_int(
-                    v["src"].split("/")[-2].partition("x")[0])
-            ),)
-            video["variants"][0]["url"] = video["variants"][0]["src"]
-            tweet["extended_entities"] = {"media": [{
-                "video_info"   : video,
-                "original_info": {"width" : 0, "height": 0},
-            }]}
-        elif "photos" in tweet:
-            for p in tweet["photos"]:
-                p["media_url_https"] = p["url"]
-                p["original_info"] = {
-                    "width" : p["width"],
-                    "height": p["height"],
-                }
-            tweet["extended_entities"] = {"media": tweet["photos"]}
-
-        return {
-            "rest_id": tweet["id_str"],
-            "legacy" : tweet,
-            "core"   : {"user_results": {"result": tweet["user"]}},
-            "_retweet_id_str": retweet_id,
-        }
-
 
 @cache(maxage=365*86400, keyarg=1)
 def _login_impl(extr, username, password):