[deviantart] add support for OAuth2 authentication

Some user galleries [*] require you to be either logged in or
authenticated via OAuth2 to access their deviations.

[*] e.g. https://polinaegorussia.deviantart.com/gallery/

--------------

known issue:
A deviantart 'refresh_token' can only be used once and gets updated
whenever it is used to request a new 'access_token', so storing its
initial value in a config file and reusing it again and again is not
possible.

CHANGELOG.md

@@ -4,6 +4,7 @@
 - Added support for:
   - `slideshare` - https://www.slideshare.net/ ([#54](https://github.com/mikf/gallery-dl/issues/54))
 - Added pool- and post-extractors for `sankaku`
+- Added OAuth user authentication for `deviantart`
 - Improved `luscious` to support `members.luscious.net` URLs ([#55](https://github.com/mikf/gallery-dl/issues/55))
 - Fixed extraction issues for `nhentai` and `khinsider`
 

README.rst

@@ -190,8 +190,9 @@ or you can provide them directly via the
 OAuth
 -----
 
-*gallery-dl* supports user authentication via OAuth_ for ``flickr`` and
+*gallery-dl* supports user authentication via OAuth_ for
-``reddit``. This is entirely optional, but grants *gallery-dl* the ability
+``deviantart``, ``flickr`` and ``reddit``.
+This is entirely optional, but grants *gallery-dl* the ability
 to issue requests on your account's behalf and enables it to access resources
 which would otherwise be unavailable to a public user.
 

docs/configuration.rst

@@ -271,7 +271,7 @@ Description Controls the behavior when downloading a file whose filename
 
 
 extractor.*.sleep
-----------------
+-----------------
 =========== =====
 Type        ``float``
 Default     ``0``
@@ -370,6 +370,19 @@ Description Request full-sized original images if available.
 =========== =====
 
 
+extractor.deviantart.refresh-token
+----------------------------------
+=========== =====
+Type        ``string``
+Default     ``null``
+Description The ``refresh_token`` value you get from linking your
+            DeviantArt account to *gallery-dl*.
+
+            Using a ``refresh_token`` allows you to access private or otherwise
+            not publicly available deviations.
+=========== =====
+
+
 extractor.exhentai.original
 ---------------------------
 =========== =====
@@ -576,10 +589,10 @@ extractor.reddit.refresh-token
 =========== =====
 Type        ``string``
 Default     ``null``
-Description The ``refresh_token`` value you get from linking your Reddit account
+Description The ``refresh_token`` value you get from linking your
-            to *gallery-dl*.
+            Reddit account to *gallery-dl*.
 
-            Using the ``refresh_token`` allows you to access private or otherwise
+            Using a ``refresh_token`` allows you to access private or otherwise
             not publicly available subreddits, given that your account is
             authorized to do so,
             but requests to the reddit API are going to be rate limited

docs/gallery-dl.conf

@@ -93,6 +93,7 @@
         },
         "deviantart":
         {
+            "refresh-token": null,
             "flat": true,
             "mature": true,
             "original": true

docs/supportedsites.rst

@@ -13,7 +13,7 @@ Archived.Moe         https://archived.moe/               Threads
 Batoto               https://bato.to/                    Chapters, Manga                                    Optional
 Danbooru             https://danbooru.donmai.us/         Pools, Popular Images, Posts, Tag-Searches
 Desuarchive          https://desuarchive.org/            Threads
-DeviantArt           https://www.deviantart.com/         |Collections, De-1|
+DeviantArt           https://www.deviantart.com/         |Collections, De-1|                                Optional (OAuth)
 Doki Reader          https://kobato.hologfx.com/         Chapters, Manga
 Dynasty Reader       https://dynasty-scans.com/          Chapters
 e621                 https://e621.net/                   Pools, Popular Images, Posts, Tag-Searches

gallery_dl/extractor/deviantart.py

@@ -341,18 +341,24 @@ class DeviantartJournalExtractor(DeviantartExtractor):
 
 class DeviantartAPI():
     """Minimal interface for the deviantart API"""
-    def __init__(self, extractor, client_id="5388",
+    CLIENT_ID = "5388"
-                 client_secret="76b08c69cfb27f26d6161f9ab6d061a1"):
+    CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1"
+
+    def __init__(self, extractor):
         self.session = extractor.session
-        self.headers = {}
         self.log = extractor.log
-        self.client_id = extractor.config("client-id", client_id)
+        self.headers = {}
-        self.client_secret = extractor.config("client-secret", client_secret)
         self.delay = 0
+
         self.mature = extractor.config("mature", "true")
         if not isinstance(self.mature, str):
             self.mature = "true" if self.mature else "false"
 
+        self.refresh_token = extractor.config("refresh-token")
+        self.client_id = extractor.config("client-id", self.CLIENT_ID)
+        self.client_secret = extractor.config(
+            "client-secret", self.CLIENT_SECRET)
+
     def browse_user_journals(self, username, offset=0):
         """Yield all journal entries of a specific user"""
         endpoint = "browse/user/journals"
@@ -422,21 +428,22 @@ class DeviantartAPI():
 
     def authenticate(self):
         """Authenticate the application by requesting an access token"""
-        access_token = self._authenticate_impl(
+        access_token = self._authenticate_impl(self.refresh_token)
-            self.client_id, self.client_secret
-        )
         self.headers["Authorization"] = access_token
 
     @cache(maxage=3590, keyarg=1)
-    def _authenticate_impl(self, client_id, client_secret):
+    def _authenticate_impl(self, refresh_token):
         """Actual authenticate implementation"""
         url = "https://www.deviantart.com/oauth2/token"
-        data = {
+        if refresh_token:
-            "grant_type": "client_credentials",
+            self.log.info("Refreshing access token")
-            "client_id": client_id,
+            data = {"grant_type": "refresh_token",
-            "client_secret": client_secret,
+                    "refresh_token": refresh_token}
-        }
+        else:
-        response = self.session.post(url, data=data)
+            self.log.info("Requesting public access token")
+            data = {"grant_type": "client_credentials"}
+        auth = (self.client_id, self.client_secret)
+        response = self.session.post(url, data=data, auth=auth)
         if response.status_code != 200:
             raise exception.AuthenticationError()
         return "Bearer " + response.json()["access_token"]
@@ -470,6 +477,7 @@ class DeviantartAPI():
         try:
             return response.json()
         except ValueError:
+            self.log.error("Failed to parse API response")
             return {}
 
     def _pagination(self, endpoint, params=None):

gallery_dl/extractor/oauth.py

@@ -9,7 +9,7 @@
 """Utility classes to setup OAuth and link a users account to gallery-dl"""
 
 from .common import Extractor, Message
-from . import reddit, flickr
+from . import deviantart, flickr, reddit
 from .. import util
 import os
 import urllib.parse
@@ -20,7 +20,7 @@ class OAuthBase(Extractor):
     category = "oauth"
     redirect_uri = "http://localhost:6414/"
 
-    def __init__(self):
+    def __init__(self, match):
         Extractor.__init__(self)
         self.client = None
 
@@ -71,56 +71,87 @@ class OAuthBase(Extractor):
             print(url, end="\n\n", flush=True)
         return self.recv()
 
+    def _oauth2_authorization_code_grant(
+            self, client_id, client_secret, auth_url, token_url, scope):
+        """Perform an OAuth2 authorization code grant"""
 
-class OAuthReddit(OAuthBase):
+        state = "gallery-dl:{}:{}".format(
-    subcategory = "reddit"
-    pattern = ["oauth:reddit$"]
-
-    def __init__(self, match):
-        OAuthBase.__init__(self)
-        self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT
-        self.client_id = reddit.RedditAPI.CLIENT_ID
-        self.state = "gallery-dl:{}:{}".format(
             self.subcategory, util.OAuthSession.nonce(8))
 
-    def items(self):
+        auth_params = {
-        yield Message.Version, 1
+            "client_id": client_id,
-
-        url = "https://www.reddit.com/api/v1/authorize"
-        params = {
-            "client_id": self.client_id,
             "response_type": "code",
-            "state": self.state,
+            "state": state,
             "redirect_uri": self.redirect_uri,
             "duration": "permanent",
-            "scope": "read",
+            "scope": scope,
         }
 
         # receive 'code'
-        params = self.open(url, params)
+        params = self.open(auth_url, auth_params)
 
-        if self.state != params.get("state"):
+        # check auth response
+        if state != params.get("state"):
             self.send("'state' mismatch: expected {}, got {}.".format(
-                      self.state, params.get("state")))
+                state, params.get("state")))
             return
         if "error" in params:
             self.send(params["error"])
             return
 
         # exchange 'code' for 'refresh_token'
-        url = "https://www.reddit.com/api/v1/access_token"
-        auth = (self.client_id, "")
         data = {
             "grant_type": "authorization_code",
             "code": params["code"],
             "redirect_uri": self.redirect_uri,
         }
-        data = self.session.post(url, auth=auth, data=data).json()
+        auth = (client_id, client_secret)
+        data = self.session.post(token_url, data=data, auth=auth).json()
 
+        # check token response
         if "error" in data:
             self.send(data["error"])
-        else:
+            return
-            self.send(REDDIT_MSG_TEMPLATE.format(token=data["refresh_token"]))
+
+        # display refresh token
+        self.send(OAUTH2_MSG_TEMPLATE.format(
+            category=self.subcategory,
+            token=data["refresh_token"]
+        ))
+
+
+class OAuthReddit(OAuthBase):
+    subcategory = "reddit"
+    pattern = ["oauth:reddit$"]
+
+    def items(self):
+        yield Message.Version, 1
+
+        self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT
+        self._oauth2_authorization_code_grant(
+            reddit.RedditAPI.CLIENT_ID,
+            "",
+            "https://www.reddit.com/api/v1/authorize",
+            "https://www.reddit.com/api/v1/access_token",
+            "read",
+        )
+
+
+class OAuthDeviantart(OAuthBase):
+    subcategory = "deviantart"
+    pattern = ["oauth:deviantart$"]
+    redirect_uri = "https://mikf.github.io/gallery-dl/oauth-redirect.html"
+
+    def items(self):
+        yield Message.Version, 1
+
+        self._oauth2_authorization_code_grant(
+            deviantart.DeviantartAPI.CLIENT_ID,
+            deviantart.DeviantartAPI.CLIENT_SECRET,
+            "https://www.deviantart.com/oauth2/authorize",
+            "https://www.deviantart.com/oauth2/token",
+            "browse",
+        )
 
 
 class OAuthFlickr(OAuthBase):
@@ -128,7 +159,7 @@ class OAuthFlickr(OAuthBase):
     pattern = ["oauth:flickr$"]
 
     def __init__(self, match):
-        OAuthBase.__init__(self)
+        OAuthBase.__init__(self, match)
         self.session = util.OAuthSession(
             self.session,
             flickr.FlickrAPI.API_KEY, flickr.FlickrAPI.API_SECRET
@@ -162,17 +193,18 @@ class OAuthFlickr(OAuthBase):
             token_secret=data["oauth_token_secret"][0]))
 
 
-REDDIT_MSG_TEMPLATE = """
+OAUTH2_MSG_TEMPLATE = """
 Your Refresh Token is
 
 {token}
 
-Put this value into your configuration file as 'extractor.reddit.refesh-token'.
+Put this value into your configuration file as
+'extractor.{category}.refesh-token'.
 
 Example:
 {{
     "extractor": {{
-        "reddit": {{
+        "{category}": {{
             "refresh-token": "{token}"
         }}
     }}

scripts/build_supportedsites.py

@@ -76,14 +76,15 @@ SUBCATEGORY_MAP = {
 }
 
 AUTH_MAP = {
-    "batoto"  : "Optional",
+    "batoto"    : "Optional",
-    "exhentai": "Optional",
+    "deviantart": "Optional (OAuth)",
-    "flickr"  : "Optional (OAuth)",
+    "exhentai"  : "Optional",
-    "nijie"   : "Required",
+    "flickr"    : "Optional (OAuth)",
-    "pixiv"   : "Required",
+    "nijie"     : "Required",
-    "reddit"  : "Optional (OAuth)",
+    "pixiv"     : "Required",
-    "sankaku" : "Optional",
+    "reddit"    : "Optional (OAuth)",
-    "seiga"   : "Required",
+    "sankaku"   : "Optional",
+    "seiga"     : "Required",
 }
 
 IGNORE_LIST = (

test/test_extractors.py

@@ -82,8 +82,7 @@ skip = [
     "exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie",
     "archivedmoe", "archiveofsins", "thebarchive",
     # temporary issues
-    "mangareader",
+    "hbrowse",
-    "mangapanda",
 ]
 # enable selective testing for direct calls
 if __name__ == '__main__' and len(sys.argv) > 1: