[pixiv] update (#1304)

- remove login with username & password
- require a refresh token
- add 'oauth:pixiv' functionality

See also:
- https://github.com/upbit/pixivpy/issues/158
- https://gist.github.com/ZipFile/c9ebedb224406f4f11845ab700124362

README.rst

@@ -214,7 +214,7 @@ Username & Password
 
 Some extractors require you to provide valid login credentials in the form of
 a username & password pair. This is necessary for
-``pixiv``, ``nijie``, and ``seiga``
+``nijie`` and ``seiga``
 and optional for
 ``aryion``,
 ``danbooru``,
@@ -237,7 +237,7 @@ You can set the necessary information in your configuration file
 
     {
         "extractor": {
-            "pixiv": {
+            "seiga": {
                 "username": "<username>",
                 "password": "<password>"
             }

docs/configuration.rst

@@ -272,7 +272,6 @@ Description
 
     Specifying a username and password is required for
 
-    * ``pixiv``
     * ``nijie``
     * ``seiga``
 

docs/supportedsites.rst

@@ -103,7 +103,7 @@ PhotoVogue           https://www.vogue.it/en/photovogue/ User Profiles
 Piczel               https://piczel.tv/                  Folders, individual Images, User Profiles
 Pillowfort           https://www.pillowfort.social/      Posts, User Profiles
 Pinterest            https://www.pinterest.com/          |pinterest-C|                                      Supported
-Pixiv                https://www.pixiv.net/              |pixiv-C|                                          Required
+Pixiv                https://www.pixiv.net/              |pixiv-C|                                          `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 Pixnet               https://www.pixnet.net/             Folders, individual Images, Sets, User Profiles
 Plurk                https://www.plurk.com/              Posts, Timelines
 Pornhub              https://www.pornhub.com/            Galleries, User Profiles

gallery_dl/extractor/oauth.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2017-2020 Mike Fährmann
+# Copyright 2017-2021 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -9,10 +9,12 @@
 """Utility classes to setup OAuth and link accounts to gallery-dl"""
 
 from .common import Extractor, Message
-from . import deviantart, flickr, reddit, smugmug, tumblr
+from . import deviantart, flickr, pixiv, reddit, smugmug, tumblr
 from .. import text, oauth, util, config, exception
 from ..cache import cache
 import urllib.parse
+import hashlib
+import base64
 
 REDIRECT_URI_LOCALHOST = "http://localhost:6414/"
 REDIRECT_URI_HTTPS = "https://mikf.github.io/gallery-dl/oauth-redirect.html"
@@ -62,14 +64,14 @@ class OAuthBase(Extractor):
         self.client.send(b"HTTP/1.1 200 OK\r\n\r\n" + msg.encode())
         self.client.close()
 
-    def open(self, url, params):
+    def open(self, url, params, recv=None):
         """Open 'url' in browser amd return response parameters"""
         import webbrowser
         url += "?" + urllib.parse.urlencode(params)
         if not self.config("browser", True) or not webbrowser.open(url):
             print("Please open this URL in your browser:")
             print(url, end="\n\n", flush=True)
-        return self.recv()
+        return (recv or self.recv)()
 
     def _oauth1_authorization_flow(
             self, request_token_url, authorize_url, access_token_url):
@@ -362,6 +364,69 @@ class OAuthMastodon(OAuthBase):
         return data
 
 
+class OAuthPixiv(OAuthBase):
+    subcategory = "pixiv"
+    pattern = "oauth:pixiv$"
+
+    def items(self):
+        yield Message.Version, 1
+
+        code_verifier = util.generate_token(32)
+        digest = hashlib.sha256(code_verifier.encode("ascii")).digest()
+        code_challenge = base64.urlsafe_b64encode(
+            digest).rstrip(b"=").decode("ascii")
+
+        url = "https://app-api.pixiv.net/web/v1/login"
+        params = {
+            "code_challenge": code_challenge,
+            "code_challenge_method": "S256",
+            "client": "pixiv-android",
+        }
+        code = self.open(url, params, self._input)
+
+        url = "https://oauth.secure.pixiv.net/auth/token"
+        headers = {
+            "User-Agent": "PixivAndroidApp/5.0.234 (Android 11; Pixel 5)",
+        }
+        data = {
+            "client_id"     : self.oauth_config(
+                "client-id"    , pixiv.PixivAppAPI.CLIENT_ID),
+            "client_secret" : self.oauth_config(
+                "client-secret", pixiv.PixivAppAPI.CLIENT_SECRET),
+            "code"          : code,
+            "code_verifier" : code_verifier,
+            "grant_type"    : "authorization_code",
+            "include_policy": "true",
+            "redirect_uri"  : "https://app-api.pixiv.net"
+                              "/web/v1/users/auth/pixiv/callback",
+        }
+        data = self.session.post(url, headers=headers, data=data).json()
+
+        if "error" in data:
+            print(data)
+            if data["error"] == "invalid_request":
+                print("'code' expired, try again")
+            return
+
+        token = data["refresh_token"]
+        if self.cache:
+            username = self.oauth_config("username")
+            pixiv._refresh_token_cache.update(username, token)
+            self.log.info("Writing 'refresh-token' to cache")
+
+        print(self._generate_message(("refresh-token",), (token,)))
+
+    def _input(self):
+        print("""
+1) Open your browser's Developer Tools (F12) and switch to the Network tab
+2) Login
+4) Select the last network monitor entry ('callback?state=...')
+4) Copy its 'code' query parameter, paste it below, and press Enter
+""")
+        code = input("code: ")
+        return code.rpartition("=")[2].strip()
+
+
 MASTODON_MSG_TEMPLATE = """
 Your 'access-token' is
 

gallery_dl/extractor/pixiv.py

@@ -510,49 +510,48 @@ class PixivAppAPI():
     def __init__(self, extractor):
         self.extractor = extractor
         self.log = extractor.log
-        self.username, self.password = extractor._get_auth_info()
+        self.username = extractor._get_auth_info()[0]
         self.user = None
 
+        extractor.session.headers.update({
+            "App-OS"        : "ios",
+            "App-OS-Version": "13.1.2",
+            "App-Version"   : "7.7.6",
+            "User-Agent"    : "PixivIOSApp/7.7.6 (iOS 13.1.2; iPhone11,8)",
+            "Referer"       : "https://app-api.pixiv.net/",
+        })
+
         self.client_id = extractor.config(
             "client-id", self.CLIENT_ID)
         self.client_secret = extractor.config(
             "client-secret", self.CLIENT_SECRET)
-        extractor.session.headers.update({
+
-            "App-OS": "ios",
+        token = extractor.config("refresh-token")
-            "App-OS-Version": "10.3.1",
+        if token is None or token == "cache":
-            "App-Version": "6.7.1",
+            token = _refresh_token_cache(self.username)
-            "User-Agent": "PixivIOSApp/6.7.1 (iOS 10.3.1; iPhone8,1)",
+        self.refresh_token = token
-            "Referer": "https://app-api.pixiv.net/",
-        })
 
     def login(self):
         """Login and gain an access token"""
-        self.user, auth = self._login_impl(self.username, self.password)
+        self.user, auth = self._login_impl(self.username)
         self.extractor.session.headers["Authorization"] = auth
 
     @cache(maxage=3600, keyarg=1)
-    def _login_impl(self, username, password):
+    def _login_impl(self, username):
-        if not username or not password:
+        if not self.refresh_token:
             raise exception.AuthenticationError(
-                "Username and password required")
+                "'refresh-token' required.\n"
+                "Run `gallery-dl oauth:pixiv` to get one.")
 
+        self.log.info("Refreshing access token")
         url = "https://oauth.secure.pixiv.net/auth/token"
         data = {
-            "client_id": self.client_id,
+            "client_id"     : self.client_id,
-            "client_secret": self.client_secret,
+            "client_secret" : self.client_secret,
-            "get_secure_url": 1,
+            "grant_type"    : "refresh_token",
+            "refresh_token" : self.refresh_token,
+            "get_secure_url": "1",
         }
-        refresh_token = _refresh_token_cache(username)
-
-        if refresh_token:
-            self.log.info("Refreshing access token")
-            data["grant_type"] = "refresh_token"
-            data["refresh_token"] = refresh_token
-        else:
-            self.log.info("Logging in as %s", username)
-            data["grant_type"] = "password"
-            data["username"] = username
-            data["password"] = password
 
         time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S+00:00")
         headers = {
@@ -565,11 +564,9 @@ class PixivAppAPI():
             url, method="POST", headers=headers, data=data, fatal=False)
         if response.status_code >= 400:
             self.log.debug(response.text)
-            raise exception.AuthenticationError()
+            raise exception.AuthenticationError("Invalid refresh token")
 
         data = response.json()["response"]
-        if not refresh_token:
-            _refresh_token_cache.update(username, data["refresh_token"])
         return data["user"], "Bearer " + data["access_token"]
 
     def illust_detail(self, illust_id):

scripts/supportedsites.py

@@ -189,7 +189,7 @@ AUTH_MAP = {
     "patreon"        : _COOKIES,
     "pawoo"          : _OAUTH,
     "pinterest"      : "Supported",
-    "pixiv"          : "Required",
+    "pixiv"          : _OAUTH,
     "reddit"         : _OAUTH,
     "sankaku"        : "Supported",
     "seiga"          : "Required",