[misskey] implement 'include' option (#5347)

docs/configuration.rst

@@ -3509,6 +3509,29 @@ Description
     Your access token, necessary to fetch favorited notes.
 
 
+extractor.[misskey].include
+---------------------------
+Type
+    * ``string``
+    * ``list`` of ``strings``
+Default
+    ``"notes"``
+Example
+    * ``"avatar,background,notes"``
+    * ``["avatar", "background", "notes"]``
+Description
+    A (comma-separated) list of subcategories to include
+    when processing a user profile.
+
+    Possible values are
+    ``"info"``,
+    ``"avatar"``,
+    ``"background"``,
+    ``"notes"``,
+
+    It is possible to use ``"all"`` instead of listing all values separately.
+
+
 extractor.[misskey].renotes
 ---------------------------
 Type

docs/gallery-dl.conf

@@ -913,6 +913,7 @@
         "misskey":
         {
             "access-token": null,
+            "include"     : ["notes"],
             "renotes"     : false,
             "replies"     : true
         },

docs/supportedsites.md

@@ -1424,25 +1424,25 @@ Consider all listed sites to potentially be NSFW.
 <tr>
     <td>Misskey.io</td>
     <td>https://misskey.io/</td>
-    <td>Avatars, Backgrounds, Favorites, Followed Users, User Profile Information, Images from Notes, User Profiles</td>
+    <td>Avatars, Backgrounds, Favorites, Followed Users, User Profile Information, Notes, User Notes, User Profiles</td>
     <td></td>
 </tr>
 <tr>
     <td>Misskey.design</td>
     <td>https://misskey.design/</td>
-    <td>Avatars, Backgrounds, Favorites, Followed Users, User Profile Information, Images from Notes, User Profiles</td>
+    <td>Avatars, Backgrounds, Favorites, Followed Users, User Profile Information, Notes, User Notes, User Profiles</td>
     <td></td>
 </tr>
 <tr>
     <td>Lesbian.energy</td>
     <td>https://lesbian.energy/</td>
-    <td>Avatars, Backgrounds, Favorites, Followed Users, User Profile Information, Images from Notes, User Profiles</td>
+    <td>Avatars, Backgrounds, Favorites, Followed Users, User Profile Information, Notes, User Notes, User Profiles</td>
     <td></td>
 </tr>
 <tr>
     <td>Sushi.ski</td>
     <td>https://sushi.ski/</td>
-    <td>Avatars, Backgrounds, Favorites, Followed Users, User Profile Information, Images from Notes, User Profiles</td>
+    <td>Avatars, Backgrounds, Favorites, Followed Users, User Profile Information, Notes, User Notes, User Profiles</td>
     <td></td>
 </tr>
 

gallery_dl/extractor/misskey.py

@@ -6,7 +6,7 @@
 
 """Extractors for Misskey instances"""
 
-from .common import BaseExtractor, Message
+from .common import BaseExtractor, Message, Dispatch
 from .. import text, exception
 from ..cache import memcache
 
@@ -103,12 +103,28 @@ BASE_PATTERN = MisskeyExtractor.update({
 })
 
 
-class MisskeyUserExtractor(MisskeyExtractor):
+class MisskeyUserExtractor(Dispatch, MisskeyExtractor):
     """Extractor for all images of a Misskey user"""
     subcategory = "user"
     pattern = BASE_PATTERN + r"/@([^/?#]+)/?$"
     example = "https://misskey.io/@USER"
 
+    def items(self):
+        base = "{}/@{}/".format(self.root, self.item)
+        return self._dispatch_extractors((
+            (MisskeyInfoExtractor      , base + "info"),
+            (MisskeyAvatarExtractor    , base + "avatar"),
+            (MisskeyBackgroundExtractor, base + "banner"),
+            (MisskeyNotesExtractor     , base + "notes"),
+        ), ("notes",))
+
+
+class MisskeyNotesExtractor(MisskeyExtractor):
+    """Extractor for a Misskey user's notes"""
+    subcategory = "notes"
+    pattern = BASE_PATTERN + r"/@([^/?#]+)/notes"
+    example = "https://misskey.io/@USER/notes"
+
     def notes(self):
         return self.api.users_notes(self.api.user_id_by_username(self.item))
 

scripts/supportedsites.py

@@ -183,7 +183,6 @@ SUBCATEGORY_MAP = {
     "issue"  : "Comic Issues",
     "manga"  : "Manga",
     "media"  : "Media Files",
-    "note"   : "Images from Notes",
     "popular": "Popular Images",
     "recent" : "Recent Images",
     "search" : "Search Results",
@@ -296,6 +295,10 @@ SUBCATEGORY_MAP = {
         "following" : "Library",
         "list": "MDLists",
     },
+    "misskey": {
+        "note" : "Notes",
+        "notes": "User Notes",
+    },
     "nijie": {
         "followed": "Followed Users",
         "nuita" : "Nuita History",

test/results/misskeydesign.py

@@ -12,7 +12,20 @@ __tests__ = (
     "#url"     : "https://misskey.design/@machina_3D",
     "#category": ("misskey", "misskey.design", "user"),
     "#class"   : misskey.MisskeyUserExtractor,
-    "#pattern" : r"https://file\.misskey\.design/post/[\w-]{36}\.\w+",
+    "#options" : {"include": "all"},
+    "#urls"    : (
+        "https://misskey.design/@machina_3D/info",
+        "https://misskey.design/@machina_3D/avatar",
+        "https://misskey.design/@machina_3D/banner",
+        "https://misskey.design/@machina_3D/notes",
+    ),
+},
+
+{
+    "#url"     : "https://misskey.design/@machina_3D/notes",
+    "#category": ("misskey", "misskey.design", "notes"),
+    "#class"   : misskey.MisskeyNotesExtractor,
+    "#pattern" : r"https://file\.misskey\.design/post/(webpublic-)?[\w-]{36}\.\w+",
     "#range"   : "1-50",
     "#count"   : 50,
 },
@@ -48,9 +61,9 @@ __tests__ = (
 },
 
 {
-    "#url"     : "https://misskey.design/@blooddj@pawoo.net",
+    "#url"     : "https://misskey.design/@blooddj@pawoo.net/notes",
-    "#category": ("misskey", "misskey.design", "user"),
+    "#category": ("misskey", "misskey.design", "notes"),
-    "#class"   : misskey.MisskeyUserExtractor,
+    "#class"   : misskey.MisskeyNotesExtractor,
     "#count"   : "> 30",
 },
 

test/results/misskeyio.py

@@ -12,11 +12,30 @@ __tests__ = (
     "#url"     : "https://misskey.io/@lithla",
     "#category": ("misskey", "misskey.io", "user"),
     "#class"   : misskey.MisskeyUserExtractor,
-    "#pattern" : r"https://(media.misskeyusercontent.com|s\d+\.arkjp\.net)/(misskey|io)/[\w-]+\.\w+",
+    "#options" : {"include": "all"},
+    "#urls"    : (
+        "https://misskey.io/@lithla/info",
+        "https://misskey.io/@lithla/avatar",
+        "https://misskey.io/@lithla/banner",
+        "https://misskey.io/@lithla/notes",
+    ),
+},
+
+{
+    "#url"     : "https://misskey.io/@lithla/notes",
+    "#category": ("misskey", "misskey.io", "notes"),
+    "#class"   : misskey.MisskeyNotesExtractor,
+    "#pattern" : r"https://(media.misskeyusercontent.(jp|com)|s\d+\.arkjp\.net)/(misskey|io)/[\w-]+\.\w+",
     "#range"   : "1-50",
     "#count"   : 50,
 },
 
+{
+    "#url"     : "https://misskey.io/@lithla/info",
+    "#category": ("misskey", "misskey.io", "info"),
+    "#class"   : misskey.MisskeyInfoExtractor,
+},
+
 {
     "#url"     : "https://misskey.io/@lithla/avatar",
     "#category": ("misskey", "misskey.io", "avatar"),
@@ -48,9 +67,9 @@ __tests__ = (
 },
 
 {
-    "#url"     : "https://misskey.io/@blooddj@pawoo.net",
+    "#url"     : "https://misskey.io/@blooddj@pawoo.net/notes",
-    "#category": ("misskey", "misskey.io", "user"),
+    "#category": ("misskey", "misskey.io", "notes"),
-    "#class"   : misskey.MisskeyUserExtractor,
+    "#class"   : misskey.MisskeyNotesExtractor,
     "#range"   : "1-50",
     "#count"   : 50,
 },
@@ -67,8 +86,12 @@ __tests__ = (
     "#url"     : "https://misskey.io/notes/9bhqfo835v",
     "#category": ("misskey", "misskey.io", "note"),
     "#class"   : misskey.MisskeyNoteExtractor,
-    "#pattern" : r"https://(media\.misskeyusercontent\.com|s\d+\.arkjp\.net)/misskey/[\w-]+\.\w+",
+    "#urls"    : (
-    "#count"   : 4,
+        "https://media.misskeyusercontent.jp/misskey/1cbba095-5a19-4107-8e20-3efb0456dda4.png?sensitive=true",
+        "https://media.misskeyusercontent.jp/misskey/6baa558b-94ac-4bd2-a393-a52324a9d2d4.png?sensitive=true",
+        "https://media.misskeyusercontent.jp/misskey/14133ad0-ea40-4fed-b6e7-65d4cbe19b96.png?sensitive=true",
+        "https://media.misskeyusercontent.jp/misskey/e11164a2-9de5-4769-8c73-0ae44124b565.png?sensitive=true",
+    ),
 },
 
 {