[pinterest] implement oauth:pinterest (#83)

Pinterest access tokens are rate limited at 200 requests per
hour (or maybe per 2 or 3 hours?) so having just one access token
for all users isn't going to work in the long run.

README.rst

@@ -194,7 +194,7 @@ OAuth
 -----
 
 *gallery-dl* supports user authentication via OAuth_ for
-``deviantart``, ``flickr``, ``reddit`` and ``tumblr``.
+``deviantart``, ``flickr``, ``pinterest``, ``reddit`` and ``tumblr``.
 This is entirely optional, but grants *gallery-dl* the ability
 to issue requests on your account's behalf and enables it to access resources
 which would otherwise be unavailable to a public user.

docs/configuration.rst

@@ -788,7 +788,9 @@ extractor.deviantart.client-id & .client-secret
 -----------------------------------------------
 =========== =====
 Type        ``string``
-How To      - login and visit DeviantArt's `Applications & Keys`_ section
+How To      - login and visit DeviantArt's
+              `Applications & Keys <https://www.deviantart.com/developers/apps>`__
+              section
             - click "Register your Application"
             - click "Save" (top right; default settings are fine)
             - copy ``client_id`` and ``client_secret`` of your new "Untitled"
@@ -800,7 +802,8 @@ extractor.flickr.api-key & .api-secret
 --------------------------------------
 =========== =====
 Type        ``string``
-How To      - login and `Create an App`_ in Flickr's `App Garden`_
+How To      - login and `Create an App <https://www.flickr.com/services/apps/create/apply/>`__
+              in Flickr's `App Garden <https://www.flickr.com/services/>`__
             - click "APPLY FOR A NON-COMMERCIAL KEY"
             - fill out the form with a random name and description
               and click "SUBMIT"
@@ -817,11 +820,19 @@ How To
 =========== =====
 
 
-extractor.pinterest.access-token
+extractor.pinterest.client-id & .secret
---------------------------------
+---------------------------------------
 =========== =====
 Type        ``string``
-How To
+How To      - login and visit Pinterest's
+              `Apps <https://developers.pinterest.com/apps/>`__ section
+            - click "Create app"
+            - choose a random name and description and click "Create"
+            - scroll down and set a Site URL (e.g. https://example.org/)
+              and allow https://mikf.github.io/gallery-dl/oauth-redirect.html
+              as Redirect URI
+            - scroll back up again, copy the "App ID" and "App secret" values
+              and put them in your configuration file
 =========== =====
 
 
@@ -829,7 +840,8 @@ extractor.reddit.client-id & .user-agent
 ----------------------------------------
 =========== =====
 Type        ``string``
-How To      - login and visit the apps_ section of your account's preferences
+How To      - login and visit the `apps <https://www.reddit.com/prefs/apps/>`__
+              section of your account's preferences
             - click the "are you a developer? create an app..." button
             - fill out the form, choose "installed app", preferably set
               "http://localhost:6414/" as "redirect uri" and finally click
@@ -838,7 +850,8 @@ How To      - login and visit the apps_ section of your account's preferences
               "installed app") and put it in your configuration file
             - use "``Python:<application name>:v1.0 (by /u/<username>)``" as
               user-agent and replace ``<application name>`` and ``<username>``
-              accordingly (see Reddit's `API access rules`_)
+              accordingly (see Reddit's
+              `API access rules <https://github.com/reddit/reddit/wiki/API>`__)
 =========== =====
 
 
@@ -846,7 +859,8 @@ extractor.tumblr.api-key
 ------------------------
 =========== =====
 Type        ``string``
-How To      - login and visit Tumblr's Applications_ section
+How To      - login and visit Tumblr's
+              `Applications <https://www.tumblr.com/oauth/apps>`__ section
             - click "Register application"
             - fill out the form: use a random name and description, set
               https://example.org/ as "Application Website" and "Default
@@ -886,10 +900,3 @@ How To      - login and visit Tumblr's Applications_ section
 .. _webbrowser.open(): https://docs.python.org/3/library/webbrowser.html
 .. _datetime.max:      https://docs.python.org/3/library/datetime.html#datetime.datetime.max
 .. _Authentication:    https://github.com/mikf/gallery-dl#5authentication
-
-.. _`Applications & Keys`: https://www.deviantart.com/developers/apps
-.. _`Create an App`:       https://www.flickr.com/services/apps/create/apply/
-.. _`App Garden`:          https://www.flickr.com/services/
-.. _apps:                  https://www.reddit.com/prefs/apps/
-.. _`API access rules`:    https://github.com/reddit/reddit/wiki/API
-.. _Applications:          https://www.tumblr.com/oauth/apps

docs/supportedsites.rst

@@ -55,7 +55,7 @@ Niconico Seiga       http://seiga.nicovideo.jp           Images from Users, indi
 nijie                https://nijie.info/                 |Images from Use-3|                                Required
 Nyafuu Archive       https://archive.nyafuu.org/         Threads
 Pawoo                https://pawoo.net                   Images from Users, Images from Statuses
-Pinterest            https://www.pinterest.com           Boards, Pins, pin.it Links
+Pinterest            https://www.pinterest.com           Boards, Pins, pin.it Links                         Optional (OAuth)
 Pixiv                https://www.pixiv.net/              |Images from Use-4|                                Required
 PowerManga           https://powermanga.org/             Chapters, Manga
 Pure Mashiro         http://reader.puremashiro.moe/      Chapters, Manga

gallery_dl/extractor/oauth.py

@@ -9,7 +9,7 @@
 """Utility classes to setup OAuth and link a users account to gallery-dl"""
 
 from .common import Extractor, Message
-from . import deviantart, flickr, reddit, tumblr
+from . import deviantart, flickr, pinterest, reddit, tumblr
 from .. import text, util, config
 import os
 import urllib.parse
@@ -50,14 +50,7 @@ class OAuthBase(Extractor):
 
         data = self.client.recv(1024).decode()
         path = data.split(" ", 2)[1]
-        query = path.partition("?")[2]
+        return text.parse_query(path.partition("?")[2])
-        return {
-            key: urllib.parse.unquote(value)
-            for key, _, value in [
-                part.partition("=")
-                for part in query.split("&")
-            ]
-        }
 
     def send(self, msg):
         """Send 'msg' to the socket opened in 'recv()'"""
@@ -69,8 +62,7 @@ class OAuthBase(Extractor):
         """Open 'url' in browser amd return response parameters"""
         import webbrowser
         url += "?" + urllib.parse.urlencode(params)
-        browser = self.config("browser", True)
+        if not self.config("browser", True) or not webbrowser.open(url):
-        if not browser or not webbrowser.open(url):
             print("Please open this URL in your browser:")
             print(url, end="\n\n", flush=True)
         return self.recv()
@@ -80,7 +72,7 @@ class OAuthBase(Extractor):
         """Perform the OAuth 1.0a authorization flow"""
         del self.session.params["oauth_token"]
 
-        # Get a Request Token
+        # get a request token
         params = {"oauth_callback": self.redirect_uri}
         data = self.session.get(request_token_url, params=params).text
 
@@ -88,25 +80,29 @@ class OAuthBase(Extractor):
         self.session.params["oauth_token"] = token = data["oauth_token"]
         self.session.token_secret = data["oauth_token_secret"]
 
-        # Get the User's Authorization
+        # get the user's authorization
         params = {"oauth_token": token, "perms": "read"}
         data = self.open(authorize_url, params)
 
-        # Exchange the Request Token for an Access Token
+        # exchange the request token for an access token
         data = self.session.get(access_token_url, params=data).text
 
         data = text.parse_query(data)
         self.send(OAUTH1_MSG_TEMPLATE.format(
             category=self.subcategory,
             token=data["oauth_token"],
-            token_secret=data["oauth_token_secret"]))
+            token_secret=data["oauth_token_secret"],
+        ))
 
     def _oauth2_authorization_code_grant(
-            self, client_id, client_secret, auth_url, token_url, scope):
+            self, client_id, client_secret, auth_url, token_url,
+            scope="read", key="refresh_token", auth=True):
         """Perform an OAuth2 authorization code grant"""
 
-        state = "gallery-dl:{}:{}".format(
+        state = "gallery-dl_{}_{}".format(
-            self.subcategory, util.OAuthSession.nonce(8))
+            self.subcategory,
+            util.OAuthSession.nonce(8)
+        )
 
         auth_params = {
             "client_id": client_id,
@@ -117,25 +113,33 @@ class OAuthBase(Extractor):
             "scope": scope,
         }
 
-        # receive 'code'
+        # receive an authorization code
         params = self.open(auth_url, auth_params)
 
-        # check auth response
+        # check authorization response
         if state != params.get("state"):
             self.send("'state' mismatch: expected {}, got {}.".format(
-                state, params.get("state")))
+                state, params.get("state")
+            ))
             return
         if "error" in params:
             self.send(params["error"])
             return
 
-        # exchange 'code' for 'refresh_token'
+        # exchange the authorization code for a token
         data = {
             "grant_type": "authorization_code",
             "code": params["code"],
             "redirect_uri": self.redirect_uri,
         }
-        auth = (client_id, client_secret)
+
+        if auth:
+            auth = (client_id, client_secret)
+        else:
+            auth = None
+            data["client_id"] = client_id
+            data["client_secret"] = client_secret
+
         data = self.session.post(token_url, data=data, auth=auth).json()
 
         # check token response
@@ -143,10 +147,13 @@ class OAuthBase(Extractor):
             self.send(data["error"])
             return
 
-        # display refresh token
+        # display token
+        part = key.partition("_")[0]
         self.send(OAUTH2_MSG_TEMPLATE.format(
             category=self.subcategory,
-            token=data["refresh_token"]
+            key=part,
+            Key=part.capitalize(),
+            token=data[key],
         ))
 
 
@@ -165,7 +172,7 @@ class OAuthDeviantart(OAuthBase):
                 "client-secret", deviantart.DeviantartAPI.CLIENT_SECRET),
             "https://www.deviantart.com/oauth2/authorize",
             "https://www.deviantart.com/oauth2/token",
-            "browse",
+            scope="browse",
         )
 
 
@@ -191,6 +198,27 @@ class OAuthFlickr(OAuthBase):
         )
 
 
+class OAuthPinterest(OAuthBase):
+    subcategory = "pinterest"
+    pattern = ["oauth:pinterest$"]
+    redirect_uri = "https://mikf.github.io/gallery-dl/oauth-redirect.html"
+
+    def items(self):
+        yield Message.Version, 1
+
+        self._oauth2_authorization_code_grant(
+            self.oauth_config(
+                "client-id", pinterest.PinterestAPI.CLIENT_ID),
+            self.oauth_config(
+                "client-secret", pinterest.PinterestAPI.CLIENT_SECRET),
+            "https://api.pinterest.com/oauth/",
+            "https://api.pinterest.com/v1/oauth/token",
+            scope="read_public",
+            key="access_token",
+            auth=False,
+        )
+
+
 class OAuthReddit(OAuthBase):
     subcategory = "reddit"
     pattern = ["oauth:reddit$"]
@@ -204,7 +232,7 @@ class OAuthReddit(OAuthBase):
             "",
             "https://www.reddit.com/api/v1/authorize",
             "https://www.reddit.com/api/v1/access_token",
-            "read",
+            scope="read",
         )
 
 
@@ -253,18 +281,18 @@ Example:
 
 
 OAUTH2_MSG_TEMPLATE = """
-Your Refresh Token is
+Your {Key} Token is
 
 {token}
 
 Put this value into your configuration file as
-'extractor.{category}.refesh-token'.
+'extractor.{category}.{key}-token'.
 
 Example:
 {{
     "extractor": {{
         "{category}": {{
-            "refresh-token": "{token}"
+            "{key}-token": "{token}"
         }}
     }}
 }}

gallery_dl/extractor/pinterest.py

@@ -134,6 +134,9 @@ class PinterestPinitExtractor(PinterestExtractor):
 
 class PinterestAPI():
     """Minimal interface for the pinterest API"""
+    CLIENT_ID = "4959725425749142746"
+    CLIENT_SECRET = ("2ea77dc64ca02974a728e46c5a9d2adf"
+                     "cdd42f4d4ffb40ad064072165ad4b10d")
 
     def __init__(self, extractor, access_token="AfyIXxi1MJ6et0NlIl_vBchHbex-"
                                                "FSWylPyr2GJE2uu3W8A97QAAAAA"):

scripts/build_supportedsites.py

@@ -88,6 +88,7 @@ AUTH_MAP = {
     "flickr"     : "Optional (OAuth)",
     "idolcomplex": "Optional",
     "nijie"      : "Required",
+    "pinterest"  : "Optional (OAuth)",
     "pixiv"      : "Required",
     "reddit"     : "Optional (OAuth)",
     "sankaku"    : "Optional",

test/test_results.py

@@ -40,6 +40,8 @@ class TestExtractorResults(unittest.TestCase):
         config.set(("extractor", "deviantart", "client-id"), "7777")
         config.set(("extractor", "deviantart", "client-secret"),
                    "ff14994c744d9208e5caeec7aab4a026")
+        config.set(("extractor", "pinterest", "access-token"),
+                   "Ab1gUJFF5TFoWXRbX0p7_ue7jOHeFSX8iOrCIOZE24bOp0A6TQAAAAA")
         config.set(("extractor", "tumblr", "api-key"),
                    "0cXoHfIqVzMQcc3HESZSNsVlulGxEXGDTTZCDrRrjaa0jmuTc6")