[tumblr] add support for OAuth authentication (#65)

README.rst

@@ -191,7 +191,7 @@ OAuth
 -----
 
 *gallery-dl* supports user authentication via OAuth_ for
-``deviantart``, ``flickr`` and ``reddit``.
+``deviantart``, ``flickr``, ``reddit`` and ``tumblr``.
 This is entirely optional, but grants *gallery-dl* the ability
 to issue requests on your account's behalf and enables it to access resources
 which would otherwise be unavailable to a public user.

docs/supportedsites.rst

@@ -71,7 +71,7 @@ Sense-Scans          http://sensescans.com/              Chapters, Manga
 SlideShare           https://www.slideshare.net/         Presentations
 Spectrum Nexus       |http://www.thes-0|                 Chapters, Manga
 The /b/ Archive      https://thebarchive.com/            Threads
-Tumblr               https://www.tumblr.com/             Images from Users, Posts, Tag-Searches
+Tumblr               https://www.tumblr.com/             Images from Users, Posts, Tag-Searches             Optional (OAuth)
 Twitter              https://twitter.com/                Tweets
 Warosu               https://warosu.org/                 Threads
 World Three          http://www.slide.world-three.org/   Chapters, Manga

gallery_dl/extractor/oauth.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2017 Mike Fährmann
+# Copyright 2017-2018 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -9,8 +9,8 @@
 """Utility classes to setup OAuth and link a users account to gallery-dl"""
 
 from .common import Extractor, Message
-from . import deviantart, flickr, reddit
+from . import deviantart, flickr, reddit, tumblr
-from .. import util
+from .. import text, util
 import os
 import urllib.parse
 
@@ -71,6 +71,32 @@ class OAuthBase(Extractor):
             print(url, end="\n\n", flush=True)
         return self.recv()
 
+    def _oauth1_authorization_flow(
+            self, request_token_url, authorize_url, access_token_url):
+        """Perform the OAuth 1.0a authorization flow"""
+        del self.session.params["oauth_token"]
+
+        # Get a Request Token
+        params = {"oauth_callback": self.redirect_uri}
+        data = self.session.get(request_token_url, params=params).text
+
+        data = text.parse_query(data)
+        self.session.params["oauth_token"] = token = data["oauth_token"]
+        self.session.token_secret = data["oauth_token_secret"]
+
+        # Get the User's Authorization
+        params = {"oauth_token": token, "perms": "read"}
+        data = self.open(authorize_url, params)
+
+        # Exchange the Request Token for an Access Token
+        data = self.session.get(access_token_url, params=data).text
+
+        data = text.parse_query(data)
+        self.send(OAUTH1_MSG_TEMPLATE.format(
+            category=self.subcategory,
+            token=data["oauth_token"],
+            token_secret=data["oauth_token_secret"]))
+
     def _oauth2_authorization_code_grant(
             self, client_id, client_secret, auth_url, token_url, scope):
         """Perform an OAuth2 authorization code grant"""
@@ -120,23 +146,6 @@ class OAuthBase(Extractor):
         ))
 
 
-class OAuthReddit(OAuthBase):
-    subcategory = "reddit"
-    pattern = ["oauth:reddit$"]
-
-    def items(self):
-        yield Message.Version, 1
-
-        self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT
-        self._oauth2_authorization_code_grant(
-            reddit.RedditAPI.CLIENT_ID,
-            "",
-            "https://www.reddit.com/api/v1/authorize",
-            "https://www.reddit.com/api/v1/access_token",
-            "read",
-        )
-
-
 class OAuthDeviantart(OAuthBase):
     subcategory = "deviantart"
     pattern = ["oauth:deviantart$"]
@@ -162,35 +171,79 @@ class OAuthFlickr(OAuthBase):
         OAuthBase.__init__(self, match)
         self.session = util.OAuthSession(
             self.session,
-            flickr.FlickrAPI.API_KEY, flickr.FlickrAPI.API_SECRET
+            flickr.FlickrAPI.API_KEY,
+            flickr.FlickrAPI.API_SECRET,
         )
-        del self.session.params["oauth_token"]
 
     def items(self):
         yield Message.Version, 1
 
-        # Get a Request Token
+        self._oauth1_authorization_flow(
-        url = "https://www.flickr.com/services/oauth/request_token"
+            "https://www.flickr.com/services/oauth/request_token",
-        params = {"oauth_callback": self.redirect_uri}
+            "https://www.flickr.com/services/oauth/authorize",
-        data = self.session.get(url, params=params).text
+            "https://www.flickr.com/services/oauth/access_token",
+        )
 
-        data = urllib.parse.parse_qs(data)
-        self.session.params["oauth_token"] = token = data["oauth_token"][0]
-        self.session.token_secret = data["oauth_token_secret"][0]
 
-        # Get the User's Authorization
+class OAuthReddit(OAuthBase):
-        url = "https://www.flickr.com/services/oauth/authorize"
+    subcategory = "reddit"
-        params = {"oauth_token": token, "perms": "read"}
+    pattern = ["oauth:reddit$"]
-        data = self.open(url, params)
 
-        # Exchange the Request Token for an Access Token
+    def items(self):
-        url = "https://www.flickr.com/services/oauth/access_token"
+        yield Message.Version, 1
-        data = self.session.get(url, params=data).text
 
-        data = urllib.parse.parse_qs(data)
+        self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT
-        self.send(FLICKR_MSG_TEMPLATE.format(
+        self._oauth2_authorization_code_grant(
-            token=data["oauth_token"][0],
+            reddit.RedditAPI.CLIENT_ID,
-            token_secret=data["oauth_token_secret"][0]))
+            "",
+            "https://www.reddit.com/api/v1/authorize",
+            "https://www.reddit.com/api/v1/access_token",
+            "read",
+        )
+
+
+class OAuthTumblr(OAuthBase):
+    subcategory = "tumblr"
+    pattern = ["oauth:tumblr$"]
+
+    def __init__(self, match):
+        OAuthBase.__init__(self, match)
+        self.session = util.OAuthSession(
+            self.session,
+            tumblr.TumblrAPI.API_KEY,
+            tumblr.TumblrAPI.API_SECRET,
+        )
+
+    def items(self):
+        yield Message.Version, 1
+
+        self._oauth1_authorization_flow(
+            "https://www.tumblr.com/oauth/request_token",
+            "https://www.tumblr.com/oauth/authorize",
+            "https://www.tumblr.com/oauth/access_token",
+        )
+
+
+OAUTH1_MSG_TEMPLATE = """
+Your Access Token and Access Token Secret are
+
+{token}
+{token_secret}
+
+Put these values into your configuration file as
+'extractor.{category}.access-token' and
+'extractor.{category}.access-token-secret'.
+
+Example:
+{{
+    "extractor": {{
+        "{category}": {{
+            "access-token": "{token}",
+            "access-token-secret": "{token_secret}"
+        }}
+    }}
+}}
+"""
 
 
 OAUTH2_MSG_TEMPLATE = """
@@ -210,23 +263,3 @@ Example:
     }}
 }}
 """
-
-FLICKR_MSG_TEMPLATE = """
-Your Access Token and Access Token Secret are
-
-{token}
-{token_secret}
-
-Put these values into your configuration file as
-'extractor.flickr.access-token' and 'extractor.flickr.access-token-secret'.
-
-Example:
-{{
-    "extractor": {{
-        "flickr": {{
-            "access-token": "{token}",
-            "access-token-secret": "{token_secret}"
-        }}
-    }}
-}}
-"""

gallery_dl/extractor/tumblr.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2016-2017 Mike Fährmann
+# Copyright 2016-2018 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -9,7 +9,7 @@
 """Extract images from https://www.tumblr.com/"""
 
 from .common import Extractor, Message
-from .. import text, exception
+from .. import text, util, exception
 from ..cache import memcache
 import re
 
@@ -202,9 +202,20 @@ class TumblrTagExtractor(TumblrExtractor):
 class TumblrAPI():
     """Minimal interface for the Tumblr API v2"""
     API_KEY = "O3hU2tMi5e4Qs5t3vezEi6L0qRORJ5y9oUpSGsrWu8iA3UCc3B"
+    API_SECRET = "sFdsK3PDdP2QpYMRAoq0oDnw0sFS24XigXmdfnaeNZpJpqAn03"
 
     def __init__(self, extractor):
-        self.api_key = extractor.config("api-key", TumblrAPI.API_KEY)
+        self.api_key = extractor.config("api-key", self.API_KEY)
+        api_secret = extractor.config("api-secret", self.API_SECRET)
+        token = extractor.config("access-token")
+        token_secret = extractor.config("access-token-secret")
+        if token and token_secret:
+            self.session = util.OAuthSession(
+                extractor.session,
+                self.api_key, api_secret, token, token_secret)
+            self.api_key = None
+        else:
+            self.session = extractor.session
         self.params = {"offset": 0, "limit": 50, "reblog_info": "true"}
         self.extractor = extractor
 
@@ -219,12 +230,12 @@ class TumblrAPI():
         return self._pagination(blog, "posts", params)
 
     def _call(self, blog, endpoint, params):
-        params["api_key"] = self.api_key
+        if self.api_key:
+            params["api_key"] = self.api_key
         url = "https://api.tumblr.com/v2/blog/{}.tumblr.com/{}".format(
             blog, endpoint)
 
-        response = self.extractor.request(
+        response = self.session.get(url, params=params).json()
-            url, params=params, fatal=False).json()
         if response["meta"]["status"] == 404:
             raise exception.NotFoundError("user or post")
         elif response["meta"]["status"] != 200:

gallery_dl/util.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2017 Mike Fährmann
+# Copyright 2017-2018 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -486,21 +486,26 @@ class OAuthSession():
         params.update(self.params)
         params["oauth_nonce"] = self.nonce(16)
         params["oauth_timestamp"] = int(time.time())
-        params["oauth_signature"] = self.signature(url, params)
+        return self.session.get(url + self.sign(url, params))
-        return self.session.get(url, params=params)
 
-    def signature(self, url, params):
+    def sign(self, url, params):
-        """Generate 'oauth_signature' value"""
+        """Generate 'oauth_signature' value and return query string"""
-        query = urllib.parse.urlencode(sorted(params.items()))
+        query = urllib.parse.urlencode(
+            sorted(params.items()), quote_via=self.quote)
         message = self.concat("GET", url, query).encode()
         key = self.concat(self.consumer_secret, self.token_secret).encode()
         signature = hmac.new(key, message, hashlib.sha1).digest()
-        return base64.b64encode(signature).decode()
+        return "?{}&oauth_signature={}".format(
+            query, self.quote(base64.b64encode(signature).decode()))
 
     @staticmethod
     def concat(*args):
-        return "&".join(urllib.parse.quote(item, "") for item in args)
+        return "&".join(OAuthSession.quote(item) for item in args)
 
     @staticmethod
     def nonce(N, alphabet=string.ascii_letters):
         return "".join(random.choice(alphabet) for _ in range(N))
+
+    @staticmethod
+    def quote(value, _=None, encoding=None, errors=None):
+        return urllib.parse.quote(value, "~", encoding, errors)

scripts/build_supportedsites.py

@@ -88,6 +88,7 @@ AUTH_MAP = {
     "reddit"     : "Optional (OAuth)",
     "sankaku"    : "Optional",
     "seiga"      : "Required",
+    "tumblr"     : "Optional (OAuth)",
 }
 
 IGNORE_LIST = (