[twitter] support communities (#4913)

docs/supportedsites.md

@@ -898,7 +898,7 @@ Consider all listed sites to potentially be NSFW.
 <tr>
     <td>Twitter</td>
     <td>https://twitter.com/</td>
-    <td>Avatars, Backgrounds, Bookmarks, Events, Followed Users, Hashtags, individual Images, Likes, Lists, List Members, Media Timelines, Search Results, Timelines, Tweets, User Profiles</td>
+    <td>Avatars, Backgrounds, Bookmarks, Communities, Events, Followed Users, Hashtags, individual Images, Likes, Lists, List Members, Media Timelines, Search Results, Timelines, Tweets, User Profiles</td>
     <td>Supported</td>
 </tr>
 <tr>

gallery_dl/extractor/twitter.py

@@ -693,6 +693,28 @@ class TwitterHashtagExtractor(TwitterExtractor):
         yield Message.Queue, url, data
 
 
+class TwitterCommunityExtractor(TwitterExtractor):
+    """Extractor for a Twitter community"""
+    subcategory = "community"
+    pattern = BASE_PATTERN + r"/i/communities/(\d+)"
+    example = "https://twitter.com/i/communities/12345"
+
+    def tweets(self):
+        if self.textonly:
+            return self.api.community_tweets_timeline(self.user)
+        return self.api.community_media_timeline(self.user)
+
+
+class TwitterCommunitiesExtractor(TwitterExtractor):
+    """Extractor for followed Twitter communities"""
+    subcategory = "communities"
+    pattern = BASE_PATTERN + r"/([^/?#]+)/communities/?$"
+    example = "https://twitter.com/i/communities"
+
+    def tweets(self):
+        return self.api.communities_main_page_timeline(self.user)
+
+
 class TwitterEventExtractor(TwitterExtractor):
     """Extractor for Tweets from a Twitter Event"""
     subcategory = "event"
@@ -1100,6 +1122,43 @@ class TwitterAPI():
             endpoint, variables,
             ("search_by_raw_query", "search_timeline", "timeline"))
 
+    def community_tweets_timeline(self, community_id):
+        endpoint = "/graphql/7B2AdxSuC-Er8qUr3Plm_w/CommunityTweetsTimeline"
+        variables = {
+            "communityId": community_id,
+            "count": 100,
+            "displayLocation": "Community",
+            "rankingMode": "Recency",
+            "withCommunity": True,
+        }
+        return self._pagination_tweets(
+            endpoint, variables,
+            ("communityResults", "result", "ranked_community_timeline",
+             "timeline"))
+
+    def community_media_timeline(self, community_id):
+        endpoint = "/graphql/qAGUldfcIoMv5KyAyVLYog/CommunityMediaTimeline"
+        variables = {
+            "communityId": community_id,
+            "count": 100,
+            "withCommunity": True,
+        }
+        return self._pagination_tweets(
+            endpoint, variables,
+            ("communityResults", "result", "community_media_timeline",
+             "timeline"))
+
+    def communities_main_page_timeline(self, screen_name):
+        endpoint = ("/graphql/GtOhw2mstITBepTRppL6Uw"
+                    "/CommunitiesMainPageTimeline")
+        variables = {
+            "count": 100,
+            "withCommunity": True,
+        }
+        return self._pagination_tweets(
+            endpoint, variables,
+            ("viewer", "communities_timeline", "timeline"))
+
     def live_event_timeline(self, event_id):
         endpoint = "/2/live_event/timeline/{}.json".format(event_id)
         params = self.params.copy()
@@ -1433,7 +1492,8 @@ class TwitterAPI():
 
                 if esw("tweet-"):
                     tweets.append(entry)
-                elif esw("profile-grid-"):
+                elif esw(("profile-grid-",
+                          "communities-grid-")):
                     if "content" in entry:
                         tweets.extend(entry["content"]["items"])
                     else:

scripts/supportedsites.py

@@ -284,6 +284,7 @@ SUBCATEGORY_MAP = {
         "media": "Media Timelines",
         "tweets": "",
         "replies": "",
+        "community": "",
         "list-members": "List Members",
     },
     "vk": {

test/results/twitter.py

@@ -229,6 +229,22 @@ __tests__ = (
     "#count"   : ">=1",
 },
 
+{
+    "#url"     : "https://twitter.com/i/communities",
+    "#category": ("", "twitter", "communities"),
+    "#class"   : twitter.TwitterCommunitiesExtractor,
+    "#range"   : "1-20",
+    "#count"   : 20,
+},
+
+{
+    "#url"     : "https://twitter.com/i/communities/1651515740753735697",
+    "#category": ("", "twitter", "community"),
+    "#class"   : twitter.TwitterCommunityExtractor,
+    "#range"   : "1-20",
+    "#count"   : 20,
+},
+
 {
     "#url"     : "https://twitter.com/supernaturepics/status/604341487988576256",
     "#comment" : "all Tweets from a 'conversation' (#1319)",