[mangadex] add 'following' extractor (#7487)

also fixes the URL pattern for the Updates feed at
https://mangadex.org/titles/feed

docs/supportedsites.md

@@ -562,7 +562,7 @@ Consider all listed sites to potentially be NSFW.
 <tr>
     <td>MangaDex</td>
     <td>https://mangadex.org/</td>
-    <td>Authors, Chapters, Followed Feed, Lists, Manga</td>
+    <td>Authors, Chapters, Updates Feed, Library, MDLists, Manga</td>
     <td>Supported</td>
 </tr>
 <tr>

gallery_dl/extractor/mangadex.py

@@ -41,6 +41,12 @@ class MangadexExtractor(Extractor):
             self._cache[uuid] = data
             yield Message.Queue, self.root + "/chapter/" + uuid, data
 
+    def _items_manga(self):
+        data = {"_extractor": MangadexMangaExtractor}
+        for manga in self.manga():
+            url = "{}/title/{}".format(self.root, manga["id"])
+            yield Message.Queue, url, data
+
     def _transform(self, chapter):
         relationships = defaultdict(list)
         for item in chapter["relationships"]:
@@ -127,7 +133,7 @@ class MangadexChapterExtractor(MangadexExtractor):
 class MangadexMangaExtractor(MangadexExtractor):
     """Extractor for manga from mangadex.org"""
     subcategory = "manga"
-    pattern = BASE_PATTERN + r"/(?:title|manga)/(?!feed$)([0-9a-f-]+)"
+    pattern = BASE_PATTERN + r"/(?:title|manga)/(?!follows|feed$)([0-9a-f-]+)"
     example = ("https://mangadex.org/title"
                "/01234567-89ab-cdef-0123-456789abcdef")
 
@@ -136,17 +142,29 @@ class MangadexMangaExtractor(MangadexExtractor):
 
 
 class MangadexFeedExtractor(MangadexExtractor):
-    """Extractor for chapters from your Followed Feed"""
+    """Extractor for chapters from your Updates Feed"""
     subcategory = "feed"
-    pattern = BASE_PATTERN + r"/title/feed$()"
+    pattern = BASE_PATTERN + r"/titles?/feed$()"
     example = "https://mangadex.org/title/feed"
 
     def chapters(self):
         return self.api.user_follows_manga_feed()
 
 
+class MangadexFollowingExtractor(MangadexExtractor):
+    """Extractor for followed manga from your Library"""
+    subcategory = "following"
+    pattern = BASE_PATTERN + r"/titles?/follows(?:\?([^#]+))?$"
+    example = "https://mangadex.org/title/follows"
+
+    items = MangadexExtractor._items_manga
+
+    def manga(self):
+        return self.api.user_follows_manga()
+
+
 class MangadexListExtractor(MangadexExtractor):
-    """Extractor for mangadex lists"""
+    """Extractor for mangadex MDLists"""
     subcategory = "list"
     pattern = (BASE_PATTERN +
                r"/list/([0-9a-f-]+)(?:/[^/?#]*)?(?:\?tab=(\w+))?")
@@ -158,17 +176,17 @@ class MangadexListExtractor(MangadexExtractor):
         if match.group(2) == "feed":
             self.subcategory = "list-feed"
         else:
-            self.items = self._items_titles
+            self.items = self._items_manga
 
     def chapters(self):
         return self.api.list_feed(self.uuid)
 
-    def _items_titles(self):
+    def manga(self):
-        data = {"_extractor": MangadexMangaExtractor}
+        return [
-        for item in self.api.list(self.uuid)["relationships"]:
+            item
-            if item["type"] == "manga":
+            for item in self.api.list(self.uuid)["relationships"]
-                url = "{}/title/{}".format(self.root, item["id"])
+            if item["type"] == "manga"
-                yield Message.Queue, url, data
+        ]
 
 
 class MangadexAuthorExtractor(MangadexExtractor):
@@ -244,6 +262,10 @@ class MangadexAPI():
         }
         return self._pagination_chapters("/manga/" + uuid + "/feed", params)
 
+    def user_follows_manga(self):
+        params = {"contentRating": None}
+        return self._pagination_manga("/user/follows/manga", params)
+
     def user_follows_manga_feed(self):
         params = {"order[publishAt]": "desc"}
         return self._pagination_chapters("/user/follows/manga/feed", params)
@@ -327,7 +349,7 @@ class MangadexAPI():
                 self.extractor.wait(until=until)
                 continue
 
-            msg = ", ".join('{title}: {detail}'.format_map(error)
+            msg = ", ".join('{title}: "{detail}"'.format_map(error)
                             for error in response.json()["errors"])
             raise exception.StopExtraction(
                 "%s %s (%s)", response.status_code, response.reason, msg)
@@ -353,10 +375,11 @@ class MangadexAPI():
     def _pagination(self, endpoint, params):
         config = self.extractor.config
 
-        ratings = config("ratings")
+        if "contentRating" not in params:
-        if ratings is None:
+            ratings = config("ratings")
-            ratings = ("safe", "suggestive", "erotica", "pornographic")
+            if ratings is None:
-        params["contentRating[]"] = ratings
+                ratings = ("safe", "suggestive", "erotica", "pornographic")
+            params["contentRating[]"] = ratings
         params["offset"] = 0
 
         api_params = config("api-parameters")

scripts/supportedsites.py

@@ -295,7 +295,9 @@ SUBCATEGORY_MAP = {
         "blog-posts": "Blog Posts",
     },
     "mangadex": {
-        "feed" : "Followed Feed",
+        "feed": "Updates Feed",
+        "following" : "Library",
+        "list": "MDLists",
     },
     "nijie": {
         "followed": "Followed Users",

test/results/mangadex.py

@@ -108,9 +108,31 @@ __tests__ = (
 },
 
 {
-    "#url"     : "https://mangadex.org/title/feed",
+    "#url"     : "https://mangadex.org/titles/feed",
-    "#category": ("", "mangadex", "feed"),
     "#class"   : mangadex.MangadexFeedExtractor,
+    "#auth"    : True,
+},
+
+{
+    "#url"     : "https://mangadex.org/title/feed",
+    "#class"   : mangadex.MangadexFeedExtractor,
+    "#auth"    : True,
+},
+
+{
+    "#url"     : "https://mangadex.org/titles/follows",
+    "#class"   : mangadex.MangadexFollowingExtractor,
+    "#auth"    : True,
+    "#urls"    : (
+        "https://mangadex.org/title/cad76ec6-ca22-42f6-96f8-eca164da6545",
+        "https://mangadex.org/title/7546ff2d-2310-47a4-b1f3-1a2561f20ce7",
+    ),
+},
+
+{
+    "#url"     : "https://mangadex.org/title/follows",
+    "#class"   : mangadex.MangadexFollowingExtractor,
+    "#auth"    : True,
 },
 
 {
@@ -134,8 +156,8 @@ __tests__ = (
     "#category": ("", "mangadex", "list-feed"),
     "#class"   : mangadex.MangadexListExtractor,
     "#urls"    : (
-        "https://mangadex.org/chapter/fa8a695d-260f-4dcc-95a3-1f30e66d6571",
         "https://mangadex.org/chapter/c765d6d5-5712-4360-be0b-0c8e0914fc94",
+        "https://mangadex.org/chapter/fa8a695d-260f-4dcc-95a3-1f30e66d6571",
         "https://mangadex.org/chapter/788766b9-41c6-422e-97ba-552f03ba9655",
     ),
 },