[misskey] add 'avatar' and 'background' extractors (#5347)

docs/supportedsites.md

@@ -1424,25 +1424,25 @@ Consider all listed sites to potentially be NSFW.
 <tr>
     <td>Misskey.io</td>
     <td>https://misskey.io/</td>
-    <td>Favorites, Followed Users, Images from Notes, User Profiles</td>
+    <td>Avatars, Backgrounds, Favorites, Followed Users, Images from Notes, User Profiles</td>
     <td></td>
 </tr>
 <tr>
     <td>Misskey.design</td>
     <td>https://misskey.design/</td>
-    <td>Favorites, Followed Users, Images from Notes, User Profiles</td>
+    <td>Avatars, Backgrounds, Favorites, Followed Users, Images from Notes, User Profiles</td>
     <td></td>
 </tr>
 <tr>
     <td>Lesbian.energy</td>
     <td>https://lesbian.energy/</td>
-    <td>Favorites, Followed Users, Images from Notes, User Profiles</td>
+    <td>Avatars, Backgrounds, Favorites, Followed Users, Images from Notes, User Profiles</td>
     <td></td>
 </tr>
 <tr>
     <td>Sushi.ski</td>
     <td>https://sushi.ski/</td>
-    <td>Favorites, Followed Users, Images from Notes, User Profiles</td>
+    <td>Avatars, Backgrounds, Favorites, Followed Users, Images from Notes, User Profiles</td>
     <td></td>
 </tr>
 

gallery_dl/extractor/misskey.py

@@ -8,6 +8,7 @@
 
 from .common import BaseExtractor, Message
 from .. import text, exception
+from ..cache import memcache
 
 
 class MisskeyExtractor(BaseExtractor):
@@ -64,6 +65,23 @@ class MisskeyExtractor(BaseExtractor):
         """Return an iterable containing all relevant Note objects"""
         return ()
 
+    def _make_note(self, type, user, url):
+        # extract real URL from potential proxy
+        path, sep, query = url.partition("?")
+        if sep:
+            url = text.parse_query(query).get("url") or path
+
+        return {
+            "id"   : type,
+            "user" : user,
+            "files": ({
+                "id" : url.rpartition("/")[2].partition(".")[0],  # ID from URL
+                "url": url,
+                "createdAt": "",
+            },),
+            "createdAt": "",
+        }
+
 
 BASE_PATTERN = MisskeyExtractor.update({
     "misskey.io": {
@@ -95,6 +113,30 @@ class MisskeyUserExtractor(MisskeyExtractor):
         return self.api.users_notes(self.api.user_id_by_username(self.item))
 
 
+class MisskeyAvatarExtractor(MisskeyExtractor):
+    """Extractor for a Misskey user's avatar"""
+    subcategory = "avatar"
+    pattern = BASE_PATTERN + r"/@([^/?#]+)/avatar"
+    example = "https://misskey.io/@USER/avatar"
+
+    def notes(self):
+        user = self.api.users_show(self.item)
+        url = user.get("avatarUrl")
+        return (self._make_note("avatar", user, url),) if url else ()
+
+
+class MisskeyBackgroundExtractor(MisskeyExtractor):
+    """Extractor for a Misskey user's banner image"""
+    subcategory = "background"
+    pattern = BASE_PATTERN + r"/@([^/?#]+)/ba(?:nner|ckground)"
+    example = "https://misskey.io/@USER/banner"
+
+    def notes(self):
+        user = self.api.users_show(self.item)
+        url = user.get("bannerUrl")
+        return (self._make_note("background", user, url),) if url else ()
+
+
 class MisskeyFollowingExtractor(MisskeyExtractor):
     """Extractor for followed Misskey users"""
     subcategory = "following"
@@ -144,15 +186,10 @@ class MisskeyAPI():
     def __init__(self, extractor):
         self.root = extractor.root
         self.extractor = extractor
-        self.headers = {"Content-Type": "application/json"}
         self.access_token = extractor.config("access-token")
 
     def user_id_by_username(self, username):
-        endpoint = "/users/show"
-        data = {"username": username}
-        if "@" in username:
-            data["username"], _, data["host"] = username.partition("@")
-        return self._call(endpoint, data)["id"]
+        return self.users_show(username)["id"]
 
     def users_following(self, user_id):
         endpoint = "/users/following"
@@ -164,6 +201,13 @@ class MisskeyAPI():
         data = {"userId": user_id}
         return self._pagination(endpoint, data)
 
+    @memcache(keyarg=1)
+    def users_show(self, username):
+        endpoint = "/users/show"
+        username, _, host = username.partition("@")
+        data = {"username": username, "host": host or None}
+        return self._call(endpoint, data)
+
     def notes_show(self, note_id):
         endpoint = "/notes/show"
         data = {"noteId": note_id}
@@ -178,8 +222,7 @@ class MisskeyAPI():
 
     def _call(self, endpoint, data):
         url = self.root + "/api" + endpoint
-        return self.extractor.request(
-            url, method="POST", headers=self.headers, json=data).json()
+        return self.extractor.request_json(url, method="POST", json=data)
 
     def _pagination(self, endpoint, data):
         data["limit"] = 100

test/results/misskeydesign.py

@@ -17,6 +17,36 @@ __tests__ = (
     "#count"   : 50,
 },
 
+{
+    "#url"     : "https://misskey.design/@machina_3D/avatar",
+    "#category": ("misskey", "misskey.design", "avatar"),
+    "#class"   : misskey.MisskeyAvatarExtractor,
+    "#urls"    : "https://file.misskey.design/post/f2cd76a4-e5e5-4c46-b45d-5cd28e0fdafe.png",
+    "#sha1_content": "5ffa5b43513ed2e77f26ef9d49a91bb0d3e76847",
+
+    "extension": "png",
+    "filename" : "f2cd76a4-e5e5-4c46-b45d-5cd28e0fdafe",
+    "id"       : "avatar",
+    "instance" : "misskey.design",
+    "file"     : {"id": "f2cd76a4-e5e5-4c46-b45d-5cd28e0fdafe"},
+    "user"     : {"id": "9bxksu5vqi"},
+},
+
+{
+    "#url"     : "https://misskey.design/@machina_3D/banner",
+    "#category": ("misskey", "misskey.design", "background"),
+    "#class"   : misskey.MisskeyBackgroundExtractor,
+    "#urls"    : "https://file.misskey.design/post/1ebf70d4-0175-454d-8d74-31829305582f.png",
+    "#sha1_content": "ba2d151e32114a894d342ed7408d970b9213e361",
+
+    "extension": "png",
+    "filename" : "1ebf70d4-0175-454d-8d74-31829305582f",
+    "id"       : "background",
+    "instance" : "misskey.design",
+    "file"     : {"id": "1ebf70d4-0175-454d-8d74-31829305582f"},
+    "user"     : {"id": "9bxksu5vqi"},
+},
+
 {
     "#url"     : "https://misskey.design/@blooddj@pawoo.net",
     "#category": ("misskey", "misskey.design", "user"),

test/results/misskeyio.py

@@ -17,6 +17,36 @@ __tests__ = (
     "#count"   : 50,
 },
 
+{
+    "#url"     : "https://misskey.io/@lithla/avatar",
+    "#category": ("misskey", "misskey.io", "avatar"),
+    "#class"   : misskey.MisskeyAvatarExtractor,
+    "#urls"    : "https://media.misskeyusercontent.jp/io/d84e09f8-99b7-423a-9229-78ba65ab8a82.gif",
+    "#sha1_content": "375af4d302a4aef0bc8fc3f15b2c75ec952ac086",
+
+    "extension": "gif",
+    "filename" : "d84e09f8-99b7-423a-9229-78ba65ab8a82",
+    "id"       : "avatar",
+    "instance" : "misskey.io",
+    "file"     : {"id": "d84e09f8-99b7-423a-9229-78ba65ab8a82"},
+    "user"     : {"id": "9bhpt59w5k"},
+},
+
+{
+    "#url"     : "https://misskey.io/@lithla/banner",
+    "#category": ("misskey", "misskey.io", "background"),
+    "#class"   : misskey.MisskeyBackgroundExtractor,
+    "#urls"    : "https://media.misskeyusercontent.jp/io/ddea6f5f-9cde-42ff-8e0b-dafbfa9cca9b.png",
+    "#sha1_content": "02e1d33a7aa03d8e63baa82ea6d75c7d5de80112",
+
+    "extension": "png",
+    "filename" : "ddea6f5f-9cde-42ff-8e0b-dafbfa9cca9b",
+    "id"       : "background",
+    "instance" : "misskey.io",
+    "file"     : {"id": "ddea6f5f-9cde-42ff-8e0b-dafbfa9cca9b"},
+    "user"     : {"id": "9bhpt59w5k"},
+},
+
 {
     "#url"     : "https://misskey.io/@blooddj@pawoo.net",
     "#category": ("misskey", "misskey.io", "user"),