From fc7d165c97c65536799ac1a5dcc45b3593caf52d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Mon, 18 Dec 2017 00:12:08 +0100 Subject: [PATCH] [deviantart] add support for OAuth2 authentication Some user galleries [*] require you to be either logged in or authenticated via OAuth2 to access their deviations. [*] e.g. https://polinaegorussia.deviantart.com/gallery/ -------------- known issue: A deviantart 'refresh_token' can only be used once and gets updated whenever it is used to request a new 'access_token', so storing its initial value in a config file and reusing it again and again is not possible. --- CHANGELOG.md | 1 + README.rst | 5 +- docs/configuration.rst | 21 +++++-- docs/gallery-dl.conf | 1 + docs/supportedsites.rst | 2 +- gallery_dl/extractor/deviantart.py | 38 +++++++----- gallery_dl/extractor/oauth.py | 94 ++++++++++++++++++++---------- scripts/build_supportedsites.py | 17 +++--- test/test_extractors.py | 3 +- 9 files changed, 119 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b097cf53..c3ace438 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Added support for: - `slideshare` - https://www.slideshare.net/ ([#54](https://github.com/mikf/gallery-dl/issues/54)) - Added pool- and post-extractors for `sankaku` +- Added OAuth user authentication for `deviantart` - Improved `luscious` to support `members.luscious.net` URLs ([#55](https://github.com/mikf/gallery-dl/issues/55)) - Fixed extraction issues for `nhentai` and `khinsider` diff --git a/README.rst b/README.rst index 5ceb4919..28645b2b 100644 --- a/README.rst +++ b/README.rst @@ -190,8 +190,9 @@ or you can provide them directly via the OAuth ----- -*gallery-dl* supports user authentication via OAuth_ for ``flickr`` and -``reddit``. This is entirely optional, but grants *gallery-dl* the ability +*gallery-dl* supports user authentication via OAuth_ for +``deviantart``, ``flickr`` and ``reddit``. +This is entirely optional, but grants *gallery-dl* the ability to issue requests on your account's behalf and enables it to access resources which would otherwise be unavailable to a public user. diff --git a/docs/configuration.rst b/docs/configuration.rst index f728f63a..b8a76cf5 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -271,7 +271,7 @@ Description Controls the behavior when downloading a file whose filename extractor.*.sleep ----------------- +----------------- =========== ===== Type ``float`` Default ``0`` @@ -370,6 +370,19 @@ Description Request full-sized original images if available. =========== ===== +extractor.deviantart.refresh-token +---------------------------------- +=========== ===== +Type ``string`` +Default ``null`` +Description The ``refresh_token`` value you get from linking your + DeviantArt account to *gallery-dl*. + + Using a ``refresh_token`` allows you to access private or otherwise + not publicly available deviations. +=========== ===== + + extractor.exhentai.original --------------------------- =========== ===== @@ -576,10 +589,10 @@ extractor.reddit.refresh-token =========== ===== Type ``string`` Default ``null`` -Description The ``refresh_token`` value you get from linking your Reddit account - to *gallery-dl*. +Description The ``refresh_token`` value you get from linking your + Reddit account to *gallery-dl*. - Using the ``refresh_token`` allows you to access private or otherwise + Using a ``refresh_token`` allows you to access private or otherwise not publicly available subreddits, given that your account is authorized to do so, but requests to the reddit API are going to be rate limited diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index ff27a412..2ea7e303 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -93,6 +93,7 @@ }, "deviantart": { + "refresh-token": null, "flat": true, "mature": true, "original": true diff --git a/docs/supportedsites.rst b/docs/supportedsites.rst index 8b5cae01..c014795d 100644 --- a/docs/supportedsites.rst +++ b/docs/supportedsites.rst @@ -13,7 +13,7 @@ Archived.Moe https://archived.moe/ Threads Batoto https://bato.to/ Chapters, Manga Optional Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag-Searches Desuarchive https://desuarchive.org/ Threads -DeviantArt https://www.deviantart.com/ |Collections, De-1| +DeviantArt https://www.deviantart.com/ |Collections, De-1| Optional (OAuth) Doki Reader https://kobato.hologfx.com/ Chapters, Manga Dynasty Reader https://dynasty-scans.com/ Chapters e621 https://e621.net/ Pools, Popular Images, Posts, Tag-Searches diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index c7cc1fd9..bb83f2d7 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -341,18 +341,24 @@ class DeviantartJournalExtractor(DeviantartExtractor): class DeviantartAPI(): """Minimal interface for the deviantart API""" - def __init__(self, extractor, client_id="5388", - client_secret="76b08c69cfb27f26d6161f9ab6d061a1"): + CLIENT_ID = "5388" + CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1" + + def __init__(self, extractor): self.session = extractor.session - self.headers = {} self.log = extractor.log - self.client_id = extractor.config("client-id", client_id) - self.client_secret = extractor.config("client-secret", client_secret) + self.headers = {} self.delay = 0 + self.mature = extractor.config("mature", "true") if not isinstance(self.mature, str): self.mature = "true" if self.mature else "false" + self.refresh_token = extractor.config("refresh-token") + self.client_id = extractor.config("client-id", self.CLIENT_ID) + self.client_secret = extractor.config( + "client-secret", self.CLIENT_SECRET) + def browse_user_journals(self, username, offset=0): """Yield all journal entries of a specific user""" endpoint = "browse/user/journals" @@ -422,21 +428,22 @@ class DeviantartAPI(): def authenticate(self): """Authenticate the application by requesting an access token""" - access_token = self._authenticate_impl( - self.client_id, self.client_secret - ) + access_token = self._authenticate_impl(self.refresh_token) self.headers["Authorization"] = access_token @cache(maxage=3590, keyarg=1) - def _authenticate_impl(self, client_id, client_secret): + def _authenticate_impl(self, refresh_token): """Actual authenticate implementation""" url = "https://www.deviantart.com/oauth2/token" - data = { - "grant_type": "client_credentials", - "client_id": client_id, - "client_secret": client_secret, - } - response = self.session.post(url, data=data) + if refresh_token: + self.log.info("Refreshing access token") + data = {"grant_type": "refresh_token", + "refresh_token": refresh_token} + else: + self.log.info("Requesting public access token") + data = {"grant_type": "client_credentials"} + auth = (self.client_id, self.client_secret) + response = self.session.post(url, data=data, auth=auth) if response.status_code != 200: raise exception.AuthenticationError() return "Bearer " + response.json()["access_token"] @@ -470,6 +477,7 @@ class DeviantartAPI(): try: return response.json() except ValueError: + self.log.error("Failed to parse API response") return {} def _pagination(self, endpoint, params=None): diff --git a/gallery_dl/extractor/oauth.py b/gallery_dl/extractor/oauth.py index dc8fbea9..7e816a86 100644 --- a/gallery_dl/extractor/oauth.py +++ b/gallery_dl/extractor/oauth.py @@ -9,7 +9,7 @@ """Utility classes to setup OAuth and link a users account to gallery-dl""" from .common import Extractor, Message -from . import reddit, flickr +from . import deviantart, flickr, reddit from .. import util import os import urllib.parse @@ -20,7 +20,7 @@ class OAuthBase(Extractor): category = "oauth" redirect_uri = "http://localhost:6414/" - def __init__(self): + def __init__(self, match): Extractor.__init__(self) self.client = None @@ -71,56 +71,87 @@ class OAuthBase(Extractor): print(url, end="\n\n", flush=True) return self.recv() + def _oauth2_authorization_code_grant( + self, client_id, client_secret, auth_url, token_url, scope): + """Perform an OAuth2 authorization code grant""" -class OAuthReddit(OAuthBase): - subcategory = "reddit" - pattern = ["oauth:reddit$"] - - def __init__(self, match): - OAuthBase.__init__(self) - self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT - self.client_id = reddit.RedditAPI.CLIENT_ID - self.state = "gallery-dl:{}:{}".format( + state = "gallery-dl:{}:{}".format( self.subcategory, util.OAuthSession.nonce(8)) - def items(self): - yield Message.Version, 1 - - url = "https://www.reddit.com/api/v1/authorize" - params = { - "client_id": self.client_id, + auth_params = { + "client_id": client_id, "response_type": "code", - "state": self.state, + "state": state, "redirect_uri": self.redirect_uri, "duration": "permanent", - "scope": "read", + "scope": scope, } # receive 'code' - params = self.open(url, params) + params = self.open(auth_url, auth_params) - if self.state != params.get("state"): + # check auth response + if state != params.get("state"): self.send("'state' mismatch: expected {}, got {}.".format( - self.state, params.get("state"))) + state, params.get("state"))) return if "error" in params: self.send(params["error"]) return # exchange 'code' for 'refresh_token' - url = "https://www.reddit.com/api/v1/access_token" - auth = (self.client_id, "") data = { "grant_type": "authorization_code", "code": params["code"], "redirect_uri": self.redirect_uri, } - data = self.session.post(url, auth=auth, data=data).json() + auth = (client_id, client_secret) + data = self.session.post(token_url, data=data, auth=auth).json() + # check token response if "error" in data: self.send(data["error"]) - else: - self.send(REDDIT_MSG_TEMPLATE.format(token=data["refresh_token"])) + return + + # display refresh token + self.send(OAUTH2_MSG_TEMPLATE.format( + category=self.subcategory, + token=data["refresh_token"] + )) + + +class OAuthReddit(OAuthBase): + subcategory = "reddit" + pattern = ["oauth:reddit$"] + + def items(self): + yield Message.Version, 1 + + self.session.headers["User-Agent"] = reddit.RedditAPI.USER_AGENT + self._oauth2_authorization_code_grant( + reddit.RedditAPI.CLIENT_ID, + "", + "https://www.reddit.com/api/v1/authorize", + "https://www.reddit.com/api/v1/access_token", + "read", + ) + + +class OAuthDeviantart(OAuthBase): + subcategory = "deviantart" + pattern = ["oauth:deviantart$"] + redirect_uri = "https://mikf.github.io/gallery-dl/oauth-redirect.html" + + def items(self): + yield Message.Version, 1 + + self._oauth2_authorization_code_grant( + deviantart.DeviantartAPI.CLIENT_ID, + deviantart.DeviantartAPI.CLIENT_SECRET, + "https://www.deviantart.com/oauth2/authorize", + "https://www.deviantart.com/oauth2/token", + "browse", + ) class OAuthFlickr(OAuthBase): @@ -128,7 +159,7 @@ class OAuthFlickr(OAuthBase): pattern = ["oauth:flickr$"] def __init__(self, match): - OAuthBase.__init__(self) + OAuthBase.__init__(self, match) self.session = util.OAuthSession( self.session, flickr.FlickrAPI.API_KEY, flickr.FlickrAPI.API_SECRET @@ -162,17 +193,18 @@ class OAuthFlickr(OAuthBase): token_secret=data["oauth_token_secret"][0])) -REDDIT_MSG_TEMPLATE = """ +OAUTH2_MSG_TEMPLATE = """ Your Refresh Token is {token} -Put this value into your configuration file as 'extractor.reddit.refesh-token'. +Put this value into your configuration file as +'extractor.{category}.refesh-token'. Example: {{ "extractor": {{ - "reddit": {{ + "{category}": {{ "refresh-token": "{token}" }} }} diff --git a/scripts/build_supportedsites.py b/scripts/build_supportedsites.py index e2b34824..8d0d03a3 100755 --- a/scripts/build_supportedsites.py +++ b/scripts/build_supportedsites.py @@ -76,14 +76,15 @@ SUBCATEGORY_MAP = { } AUTH_MAP = { - "batoto" : "Optional", - "exhentai": "Optional", - "flickr" : "Optional (OAuth)", - "nijie" : "Required", - "pixiv" : "Required", - "reddit" : "Optional (OAuth)", - "sankaku" : "Optional", - "seiga" : "Required", + "batoto" : "Optional", + "deviantart": "Optional (OAuth)", + "exhentai" : "Optional", + "flickr" : "Optional (OAuth)", + "nijie" : "Required", + "pixiv" : "Required", + "reddit" : "Optional (OAuth)", + "sankaku" : "Optional", + "seiga" : "Required", } IGNORE_LIST = ( diff --git a/test/test_extractors.py b/test/test_extractors.py index 11c00ea8..ea5ec2ad 100644 --- a/test/test_extractors.py +++ b/test/test_extractors.py @@ -82,8 +82,7 @@ skip = [ "exhentai", "kissmanga", "mangafox", "dynastyscans", "nijie", "archivedmoe", "archiveofsins", "thebarchive", # temporary issues - "mangareader", - "mangapanda", + "hbrowse", ] # enable selective testing for direct calls if __name__ == '__main__' and len(sys.argv) > 1: