From 7471933d5febe699b3894d0255b9fa4a8f59b8e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sat, 22 Dec 2018 14:40:35 +0100 Subject: [PATCH] use extractor.request for all other API calls - deviantart - pawoo - pixiv - reddit --- gallery_dl/extractor/deviantart.py | 24 ++++++++++++++---------- gallery_dl/extractor/pawoo.py | 20 +++++++++----------- gallery_dl/extractor/pixiv.py | 15 ++++++++------- gallery_dl/extractor/reddit.py | 11 ++++++----- 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index 0a417ada..f3171402 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -458,7 +458,7 @@ class DeviantartAPI(): CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1" def __init__(self, extractor): - self.session = extractor.session + self.extractor = extractor self.log = extractor.log self.headers = {} @@ -553,8 +553,7 @@ class DeviantartAPI(): def authenticate(self, refresh_token): """Authenticate the application by requesting an access token""" - access_token = self._authenticate_impl(refresh_token) - self.headers["Authorization"] = access_token + self.headers["Authorization"] = self._authenticate_impl(refresh_token) @cache(maxage=3590, keyarg=1) def _authenticate_impl(self, refresh_token): @@ -569,7 +568,8 @@ class DeviantartAPI(): data = {"grant_type": "client_credentials"} auth = (self.client_id, self.client_secret) - response = self.session.post(url, data=data, auth=auth) + response = self.extractor.request( + url, method="POST", data=data, auth=auth) data = response.json() if response.status_code != 200: @@ -587,8 +587,12 @@ class DeviantartAPI(): time.sleep(2 ** self.delay) self.authenticate(None if public else self.refresh_token) - response = self.session.get( - url, headers=self.headers, params=params) + response = self.extractor.request( + url, + params=params, + headers=self.headers, + expect=range(400, 500), + ) data = response.json() status = response.status_code @@ -604,12 +608,12 @@ class DeviantartAPI(): self.log.debug(response.text) msg = "API responded with {} {}".format( status, response.reason) - if 400 <= status < 500 and status != 429: - self.log.error(msg) - return data - else: + if status == 429: self.delay += 1 self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay) + else: + self.log.error(msg) + return data def _pagination(self, endpoint, params): public = True diff --git a/gallery_dl/extractor/pawoo.py b/gallery_dl/extractor/pawoo.py index 6a7c4508..23f8af9a 100644 --- a/gallery_dl/extractor/pawoo.py +++ b/gallery_dl/extractor/pawoo.py @@ -105,17 +105,16 @@ class MastodonAPI(): def __init__(self, extractor, root="https://pawoo.net", access_token=("286462927198d0cf3e24683e91c8259a" "ac4367233064e0570ca18df2ac65b226")): - access_token = extractor.config("access-token", access_token) - self.session = extractor.session - self.session.headers["Authorization"] = "Bearer " + access_token self.root = root + self.extractor = extractor + extractor.session.headers["Authorization"] = "Bearer {}".format( + extractor.config("access-token", access_token)) def account_search(self, query, limit=40): """Search for content""" - response = self.session.get( - self.root + "/api/v1/accounts/search", - params={"q": query, "limit": limit}, - ) + url = "{}/api/v1/accounts/search".format(self.root) + params = {"q": query, "limit": limit} + response = self.extractor.request(url, params=params) return self._parse(response) def account_statuses(self, account_id): @@ -123,15 +122,14 @@ class MastodonAPI(): url = "{}/api/v1/accounts/{}/statuses?only_media=1".format( self.root, account_id) while url: - response = self.session.get(url) + response = self.extractor.request(url) yield from self._parse(response) url = response.links.get("next", {}).get("url") def status(self, status_id): """Fetch a Status""" - response = self.session.get( - self.root + "/api/v1/statuses/" + status_id - ) + url = "{}/api/v1/statuses/{}".format(self.root, status_id) + response = self.extractor.request(url, expect=(404,)) return self._parse(response) @staticmethod diff --git a/gallery_dl/extractor/pixiv.py b/gallery_dl/extractor/pixiv.py index 9a5aa8e3..c7d0eacd 100644 --- a/gallery_dl/extractor/pixiv.py +++ b/gallery_dl/extractor/pixiv.py @@ -406,7 +406,7 @@ class PixivAppAPI(): CLIENT_SECRET = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj" def __init__(self, extractor): - self.session = extractor.session + self.extractor = extractor self.log = extractor.log self.username, self.password = extractor._get_auth_info() self.user = None @@ -416,7 +416,7 @@ class PixivAppAPI(): self.client_secret = extractor.config( "client-secret", self.CLIENT_SECRET) - self.session.headers.update({ + extractor.session.headers.update({ "App-OS": "ios", "App-OS-Version": "10.3.1", "App-Version": "6.7.1", @@ -426,9 +426,8 @@ class PixivAppAPI(): def login(self): """Login and gain an access token""" - self.user, auth = self._login_impl( - self.username, self.password) - self.session.headers["Authorization"] = auth + self.user, auth = self._login_impl(self.username, self.password) + self.extractor.session.headers["Authorization"] = auth @cache(maxage=3590, keyarg=1) def _login_impl(self, username, password): @@ -450,7 +449,8 @@ class PixivAppAPI(): data["username"] = username data["password"] = password - response = self.session.post(url, data=data) + response = self.extractor.request( + url, method="POST", data=data, expect=(400,)) if response.status_code >= 400: raise exception.AuthenticationError() @@ -496,7 +496,8 @@ class PixivAppAPI(): url = "https://app-api.pixiv.net/" + endpoint self.login() - response = self.session.get(url, params=params) + response = self.extractor.request( + url, params=params, expect=range(400, 500)) if 200 <= response.status_code < 400: return response.json() diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index 0fd0c23a..5ab7c065 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -160,7 +160,6 @@ class RedditAPI(): self.morecomments = extractor.config("morecomments", False) self.refresh_token = extractor.config("refresh-token") self.log = extractor.log - self.session = extractor.session client_id = extractor.config("client-id", self.CLIENT_ID) user_agent = extractor.config("user-agent", self.USER_AGENT) @@ -172,7 +171,7 @@ class RedditAPI(): "override either both or none of them.") else: self.client_id = client_id - self.session.headers["User-Agent"] = user_agent + extractor.session.headers["User-Agent"] = user_agent def submission(self, submission_id): """Fetch the (submission, comments)=-tuple for a submission id""" @@ -209,7 +208,7 @@ class RedditAPI(): def authenticate(self): """Authenticate the application by requesting an access token""" access_token = self._authenticate_impl(self.refresh_token) - self.session.headers["Authorization"] = access_token + self.extractor.session.headers["Authorization"] = access_token @cache(maxage=3590, keyarg=1) def _authenticate_impl(self, refresh_token=None): @@ -224,7 +223,8 @@ class RedditAPI(): data = {"grant_type": ("https://oauth.reddit.com/" "grants/installed_client"), "device_id": "DO_NOT_TRACK_THIS_DEVICE"} - response = self.session.post(url, data=data, auth=(self.client_id, "")) + response = self.extractor.request( + url, method="POST", data=data, auth=(self.client_id, "")) if response.status_code != 200: raise exception.AuthenticationError('"{} ({})"'.format( response.json().get("message"), response.status_code)) @@ -234,7 +234,8 @@ class RedditAPI(): url = "https://oauth.reddit.com" + endpoint params["raw_json"] = 1 self.authenticate() - response = self.session.get(url, params=params) + response = self.extractor.request( + url, params=params, expect=range(400, 500)) remaining = response.headers.get("x-ratelimit-remaining") if remaining and float(remaining) < 2: wait = int(response.headers["x-ratelimit-reset"])