From 1d145a6186466385467b2a87e57c31aa4c146af3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 31 Jan 2021 01:38:23 +0100 Subject: [PATCH] [mastodon] use cache for OAuth tokens (#616) --- gallery_dl/extractor/mastodon.py | 29 +++++++++---- gallery_dl/extractor/oauth.py | 72 +++++++++----------------------- 2 files changed, 39 insertions(+), 62 deletions(-) diff --git a/gallery_dl/extractor/mastodon.py b/gallery_dl/extractor/mastodon.py index dfc153c5..daa3d658 100644 --- a/gallery_dl/extractor/mastodon.py +++ b/gallery_dl/extractor/mastodon.py @@ -10,6 +10,7 @@ from .common import BaseExtractor, Message from .. import text, exception +from ..cache import cache class MastodonExtractor(BaseExtractor): @@ -24,7 +25,6 @@ class MastodonExtractor(BaseExtractor): BaseExtractor.__init__(self, match) self.instance = self.root.partition("://")[2] self.item = match.group(match.lastindex) - self.api = MastodonAPI(self) def items(self): for status in self.statuses(): @@ -96,14 +96,15 @@ class MastodonUserExtractor(MastodonExtractor): ) def statuses(self): + api = MastodonAPI(self) username = self.item handle = "@{}@{}".format(username, self.instance) - for account in self.api.account_search(handle, 1): + for account in api.account_search(handle, 1): if account["username"] == username: break else: raise exception.NotFoundError("account") - return self.api.account_statuses(account["id"]) + return api.account_statuses(account["id"]) class MastodonStatusExtractor(MastodonExtractor): @@ -123,7 +124,7 @@ class MastodonStatusExtractor(MastodonExtractor): ) def statuses(self): - return (self.api.status(self.item),) + return (MastodonAPI(self).status(self.item),) class MastodonAPI(): @@ -133,16 +134,21 @@ class MastodonAPI(): https://github.com/tootsuite/documentation/blob/master/Using-the-API/API.md """ - def __init__(self, extractor, access_token=None): + def __init__(self, extractor): self.root = extractor.root self.extractor = extractor + access_token = extractor.config("access-token") + if access_token is None or access_token == "cache": + access_token = _access_token_cache(extractor.instance) if not access_token: - access_token = extractor.config("access-token") - if not access_token: - if extractor.category not in INSTANCES: - raise exception.StopExtraction("missing access token") + try: access_token = INSTANCES[extractor.category]["access-token"] + except (KeyError, TypeError): + raise exception.StopExtraction( + "Missing access token.\n" + "Run 'gallery-dl oauth:mastodon:%s' to obtain one.", + extractor.instance) self.headers = {"Authorization": "Bearer " + access_token} @@ -196,3 +202,8 @@ class MastodonAPI(): if not url: return url = url["url"] + + +@cache(maxage=100*365*24*3600, keyarg=0) +def _access_token_cache(instance): + return None diff --git a/gallery_dl/extractor/oauth.py b/gallery_dl/extractor/oauth.py index ab1c7931..5b56e843 100644 --- a/gallery_dl/extractor/oauth.py +++ b/gallery_dl/extractor/oauth.py @@ -104,9 +104,9 @@ class OAuthBase(Extractor): )) def _oauth2_authorization_code_grant( - self, client_id, client_secret, auth_url, token_url, + self, client_id, client_secret, auth_url, token_url, *, scope="read", key="refresh_token", auth=True, - message_template=None, cache=None): + cache=None, instance=None): """Perform an OAuth2 authorization code grant""" state = "gallery-dl_{}_{}".format( @@ -115,12 +115,12 @@ class OAuthBase(Extractor): ) auth_params = { - "client_id": client_id, + "client_id" : client_id, "response_type": "code", - "state": state, - "redirect_uri": self.redirect_uri, - "duration": "permanent", - "scope": scope, + "state" : state, + "redirect_uri" : self.redirect_uri, + "duration" : "permanent", + "scope" : scope, } # receive an authorization code @@ -138,8 +138,8 @@ class OAuthBase(Extractor): # exchange the authorization code for a token data = { - "grant_type": "authorization_code", - "code": params["code"], + "grant_type" : "authorization_code", + "code" : params["code"], "redirect_uri": self.redirect_uri, } @@ -157,27 +157,18 @@ class OAuthBase(Extractor): self.send(data["error"]) return + token = data[key] + token_name = key.replace("_", "-") + # write to cache if self.cache and cache: - cache.update("#" + str(client_id), data[key]) - self.log.info("Writing 'refresh-token' to cache") + cache.update(instance or ("#" + str(client_id)), token) + self.log.info("Writing '%s' to cache", token_name) # display token - if message_template: - msg = message_template.format( - category=self.subcategory, - key=key.partition("_")[0], - token=data[key], - instance=getattr(self, "instance", ""), - client_id=client_id, - client_secret=client_secret, - ) - else: - msg = self._generate_message( - ("refresh-token",), - (data[key],), - ) - self.send(msg) + self.send(self._generate_message( + (token_name,), (token,), + )) def _generate_message(self, names, values): _vh, _va, _is, _it = ( @@ -335,8 +326,9 @@ class OAuthMastodon(OAuthBase): application["client-secret"], "https://{}/oauth/authorize".format(self.instance), "https://{}/oauth/token".format(self.instance), + instance=self.instance, key="access_token", - message_template=MASTODON_MSG_TEMPLATE, + cache=mastodon._access_token_cache, ) @cache(maxage=10*365*24*3600, keyarg=1) @@ -362,29 +354,3 @@ class OAuthMastodon(OAuthBase): self.log.info("client-secret:\n%s", data["client-secret"]) return data - - -MASTODON_MSG_TEMPLATE = """ -Your 'access-token' is - -{token} - -Put this value into your configuration file as -'extractor.mastodon.{instance}.{key}-token'. - -You can also add your 'client-id' and 'client-secret' values -if you want to register another account in the future. - -Example: -{{ - "extractor": {{ - "mastodon": {{ - "{instance}": {{ - "{key}-token": "{token}", - "client-id": "{client_id}", - "client-secret": "{client_secret}" - }} - }} - }} -}} -"""