[twitter] add 'notifications' extractor (#7974)

docs/supportedsites.md

@@ -1132,7 +1132,7 @@ Consider all listed sites to potentially be NSFW.
 <tr id="twitter" title="twitter">
     <td>Twitter</td>
     <td>https://x.com/</td>
-    <td>Avatars, Backgrounds, Bookmarks, Communities, Events, Followers, Followed Users, Hashtags, Highlights, Home Feed, individual Images, User Profile Information, Likes, Lists, List Members, Media Timelines, Quotes, Search Results, Timelines, Tweets, User Profiles</td>
+    <td>Avatars, Backgrounds, Bookmarks, Communities, Events, Followers, Followed Users, Hashtags, Highlights, Home Feed, individual Images, User Profile Information, Likes, Lists, List Members, Media Timelines, Notifications, Quotes, Search Results, Timelines, Tweets, User Profiles</td>
     <td><a href="https://github.com/mikf/gallery-dl#cookies">Cookies</a></td>
 </tr>
 <tr id="unsplash" title="unsplash">

gallery_dl/extractor/twitter.py

@@ -521,10 +521,13 @@ class TwitterExtractor(Extractor):
         except KeyError:
             pass
 
-        core = user.get("core") or user
+        if "core" in user:
-        legacy = user.get("legacy") or user
+            core = user["core"]
-        lget = legacy.get
+            legacy = user["legacy"]
+        else:
+            core = legacy = user
 
+        lget = legacy.get
         if lget("withheld_scope"):
             self.log.warning("'%s'", lget("description"))
 
@@ -533,14 +536,9 @@ class TwitterExtractor(Extractor):
             "id"              : text.parse_int(uid),
             "name"            : core.get("screen_name"),
             "nick"            : core.get("name"),
-            "location"        : user["location"].get("location"),
             "date"            : self.parse_datetime(
                 core["created_at"], "%a %b %d %H:%M:%S %z %Y"),
-            "verified"        : user["verification"]["verified"],
-            "protected"       : user["privacy"]["protected"],
             "profile_banner"  : lget("profile_banner_url", ""),
-            "profile_image"   : user["avatar"].get("image_url", "").replace(
-                "_normal.", "."),
             "favourites_count": lget("favourites_count"),
             "followers_count" : lget("followers_count"),
             "friends_count"   : lget("friends_count"),
@@ -549,6 +547,19 @@ class TwitterExtractor(Extractor):
             "statuses_count"  : lget("statuses_count"),
         }
 
+        if "core" in user:
+            udata["location"] = user["location"].get("location")
+            udata["verified"] = user["verification"]["verified"]
+            udata["protected"] = user["privacy"]["protected"]
+            udata["profile_image"] = user["avatar"].get(
+                "image_url", "").replace("_normal.", ".")
+        else:
+            udata["location"] = user["location"]
+            udata["verified"] = user["verified"]
+            udata["protected"] = user["protected"]
+            udata["profile_image"] = user["profile_image_url_https"].replace(
+                "_normal.", ".")
+
         descr = legacy["description"]
         if urls := entities["description"].get("urls"):
             for url in urls:
@@ -667,8 +678,7 @@ class TwitterExtractor(Extractor):
 class TwitterHomeExtractor(TwitterExtractor):
     """Extractor for Twitter home timelines"""
     subcategory = "home"
-    pattern = (BASE_PATTERN +
+    pattern = BASE_PATTERN + r"/home(?:/fo(?:llowing|r[-_ ]?you()))?/?$"
-               r"/(?:home(?:/fo(?:llowing|r[-_ ]?you()))?|i/timeline)/?$")
     example = "https://x.com/home"
 
     def tweets(self):
@@ -677,6 +687,16 @@ class TwitterHomeExtractor(TwitterExtractor):
         return self.api.home_timeline()
 
 
+class TwitterNotificationsExtractor(TwitterExtractor):
+    """Extractor for Twitter notifications timelines"""
+    subcategory = "notifications"
+    pattern = BASE_PATTERN + r"/(?:notifications|i/timeline())"
+    example = "https://x.com/notifications"
+
+    def tweets(self):
+        return self.api.notifications_devicefollow()
+
+
 class TwitterSearchExtractor(TwitterExtractor):
     """Extractor for Twitter search results"""
     subcategory = "search"
@@ -1226,18 +1246,17 @@ class TwitterAPI():
             "include_mute_edge": "1",
             "include_can_dm": "1",
             "include_can_media_tag": "1",
-            "include_ext_has_nft_avatar": "1",
             "include_ext_is_blue_verified": "1",
             "include_ext_verified_type": "1",
+            "include_ext_profile_image_shape": "1",
             "skip_status": "1",
             "cards_platform": "Web-12",
             "include_cards": "1",
             "include_ext_alt_text": "true",
-            "include_ext_limited_action_results": "false",
+            "include_ext_limited_action_results": "true",
             "include_quote_count": "true",
             "include_reply_count": "1",
             "tweet_mode": "extended",
-            "include_ext_collab_control": "true",
             "include_ext_views": "true",
             "include_entities": "true",
             "include_user_entities": "true",
@@ -1247,16 +1266,11 @@ class TwitterAPI():
             "include_ext_trusted_friends_metadata": "true",
             "send_error_codes": "true",
             "simple_quoted_tweet": "true",
-            "q": None,
-            "count": "100",
-            "query_source": None,
             "cursor": None,
-            "pc": None,
+            "count": "20",
-            "spelling_corrections": None,
+            "ext": "mediaStats,highlightedLabel,parodyCommentaryFanLabel,"
-            "include_ext_edit_control": "true",
+                   "voiceInfo,birdwatchPivot,superFollowMetadata,"
-            "ext": "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,"
+                   "unmentionInfo,editControl,article",
-                   "enrichments,superFollowMetadata,unmentionInfo,editControl,"
-                   "collab_control,vibe",
         }
         self.features = {
             "hidden_profile_subscriptions_enabled": True,
@@ -1576,7 +1590,7 @@ class TwitterAPI():
         params["timeline_id"] = "recap"
         params["urt"] = "true"
         params["get_annotations"] = "true"
-        return self._pagination_legacy(endpoint, params)
+        return self._pagination_rest(endpoint, params)
 
     def live_event(self, event_id):
         endpoint = f"/1.1/live_event/1/{event_id}/timeline.json"
@@ -1604,6 +1618,12 @@ class TwitterAPI():
         return self._pagination_users(
             endpoint, variables, ("list", "members_timeline", "timeline"))
 
+    def notifications_devicefollow(self):
+        endpoint = "/2/notifications/device_follow.json"
+        params = self.params.copy()
+        params["count"] = self.extractor.config("limit", 50)
+        return self._pagination_rest(endpoint, params)
+
     def user_followers(self, screen_name):
         endpoint = "/graphql/i6PPdIMm1MO7CpAqjau7sw/Followers"
         variables = {
@@ -1797,7 +1817,7 @@ class TwitterAPI():
             raise exception.AbortExtraction(
                 f"{response.status_code} {response.reason} ({errors})")
 
-    def _pagination_legacy(self, endpoint, params):
+    def _pagination_rest(self, endpoint, params):
         extr = self.extractor
         if cursor := extr._init_cursor():
             params["cursor"] = cursor

test/results/twitter.py

@@ -802,9 +802,14 @@ The Washington Post writes, "Three weeks after the toxic train derailment in Ohi
     "#class"   : twitter.TwitterHomeExtractor,
 },
 
+{
+    "#url"     : "https://x.com/notifications",
+    "#class"   : twitter.TwitterNotificationsExtractor,
+},
+
 {
     "#url"     : "https://x.com/i/timeline",
-    "#class"   : twitter.TwitterHomeExtractor,
+    "#class"   : twitter.TwitterNotificationsExtractor,
 },
 
 )