github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use extractor.request for all other API calls

Browse Source 
- deviantart
- pawoo
- pixiv
- reddit
This commit is contained in:
Mike Fährmann
2018-12-22 14:40:35 +01:00
parent 995844c915
commit 7471933d5f
4 changed files with 37 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
gallery_dl/extractor/deviantart.py
View File

@@ -458,7 +458,7 @@ class DeviantartAPI():
CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1"
def __init__(self, extractor):
self.session = extractor.session
self.extractor = extractor
self.log = extractor.log
self.headers = {}
@@ -553,8 +553,7 @@ class DeviantartAPI():
def authenticate(self, refresh_token):
"""Authenticate the application by requesting an access token"""
access_token = self._authenticate_impl(refresh_token)
self.headers["Authorization"] = access_token
self.headers["Authorization"] = self._authenticate_impl(refresh_token)
@cache(maxage=3590, keyarg=1)
def _authenticate_impl(self, refresh_token):
@@ -569,7 +568,8 @@ class DeviantartAPI():
data = {"grant_type": "client_credentials"}
auth = (self.client_id, self.client_secret)
response = self.session.post(url, data=data, auth=auth)
response = self.extractor.request(
url, method="POST", data=data, auth=auth)
data = response.json()
if response.status_code != 200:
@@ -587,8 +587,12 @@ class DeviantartAPI():
time.sleep(2 ** self.delay)
self.authenticate(None if public else self.refresh_token)
response = self.session.get(
url, headers=self.headers, params=params)
response = self.extractor.request(
url,
params=params,
headers=self.headers,
expect=range(400, 500),
)
data = response.json()
status = response.status_code
@@ -604,12 +608,12 @@ class DeviantartAPI():
self.log.debug(response.text)
msg = "API responded with {} {}".format(
status, response.reason)
if 400 <= status < 500 and status != 429:
self.log.error(msg)
return data
else:
if status == 429:
self.delay += 1
self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay)
else:
self.log.error(msg)
return data
def _pagination(self, endpoint, params):
public = True

20
gallery_dl/extractor/pawoo.py
View File

@@ -105,17 +105,16 @@ class MastodonAPI():
def __init__(self, extractor, root="https://pawoo.net",
access_token=("286462927198d0cf3e24683e91c8259a"
"ac4367233064e0570ca18df2ac65b226")):
access_token = extractor.config("access-token", access_token)
self.session = extractor.session
self.session.headers["Authorization"] = "Bearer " + access_token
self.root = root
self.extractor = extractor
extractor.session.headers["Authorization"] = "Bearer {}".format(
extractor.config("access-token", access_token))
def account_search(self, query, limit=40):
"""Search for content"""
response = self.session.get(
self.root + "/api/v1/accounts/search",
params={"q": query, "limit": limit},
)
url = "{}/api/v1/accounts/search".format(self.root)
params = {"q": query, "limit": limit}
response = self.extractor.request(url, params=params)
return self._parse(response)
def account_statuses(self, account_id):
@@ -123,15 +122,14 @@ class MastodonAPI():
url = "{}/api/v1/accounts/{}/statuses?only_media=1".format(
self.root, account_id)
while url:
response = self.session.get(url)
response = self.extractor.request(url)
yield from self._parse(response)
url = response.links.get("next", {}).get("url")
def status(self, status_id):
"""Fetch a Status"""
response = self.session.get(
self.root + "/api/v1/statuses/" + status_id
)
url = "{}/api/v1/statuses/{}".format(self.root, status_id)
response = self.extractor.request(url, expect=(404,))
return self._parse(response)
@staticmethod

15
gallery_dl/extractor/pixiv.py
View File

@@ -406,7 +406,7 @@ class PixivAppAPI():
CLIENT_SECRET = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj"
def __init__(self, extractor):
self.session = extractor.session
self.extractor = extractor
self.log = extractor.log
self.username, self.password = extractor._get_auth_info()
self.user = None
@@ -416,7 +416,7 @@ class PixivAppAPI():
self.client_secret = extractor.config(
"client-secret", self.CLIENT_SECRET)
self.session.headers.update({
extractor.session.headers.update({
"App-OS": "ios",
"App-OS-Version": "10.3.1",
"App-Version": "6.7.1",
@@ -426,9 +426,8 @@ class PixivAppAPI():
def login(self):
"""Login and gain an access token"""
self.user, auth = self._login_impl(
self.username, self.password)
self.session.headers["Authorization"] = auth
self.user, auth = self._login_impl(self.username, self.password)
self.extractor.session.headers["Authorization"] = auth
@cache(maxage=3590, keyarg=1)
def _login_impl(self, username, password):
@@ -450,7 +449,8 @@ class PixivAppAPI():
data["username"] = username
data["password"] = password
response = self.session.post(url, data=data)
response = self.extractor.request(
url, method="POST", data=data, expect=(400,))
if response.status_code >= 400:
raise exception.AuthenticationError()
@@ -496,7 +496,8 @@ class PixivAppAPI():
url = "https://app-api.pixiv.net/" + endpoint
self.login()
response = self.session.get(url, params=params)
response = self.extractor.request(
url, params=params, expect=range(400, 500))
if 200 <= response.status_code < 400:
return response.json()

11
gallery_dl/extractor/reddit.py
View File

@@ -160,7 +160,6 @@ class RedditAPI():
self.morecomments = extractor.config("morecomments", False)
self.refresh_token = extractor.config("refresh-token")
self.log = extractor.log
self.session = extractor.session
client_id = extractor.config("client-id", self.CLIENT_ID)
user_agent = extractor.config("user-agent", self.USER_AGENT)
@@ -172,7 +171,7 @@ class RedditAPI():
"override either both or none of them.")
else:
self.client_id = client_id
self.session.headers["User-Agent"] = user_agent
extractor.session.headers["User-Agent"] = user_agent
def submission(self, submission_id):
"""Fetch the (submission, comments)=-tuple for a submission id"""
@@ -209,7 +208,7 @@ class RedditAPI():
def authenticate(self):
"""Authenticate the application by requesting an access token"""
access_token = self._authenticate_impl(self.refresh_token)
self.session.headers["Authorization"] = access_token
self.extractor.session.headers["Authorization"] = access_token
@cache(maxage=3590, keyarg=1)
def _authenticate_impl(self, refresh_token=None):
@@ -224,7 +223,8 @@ class RedditAPI():
data = {"grant_type": ("https://oauth.reddit.com/"
"grants/installed_client"),
"device_id": "DO_NOT_TRACK_THIS_DEVICE"}
response = self.session.post(url, data=data, auth=(self.client_id, ""))
response = self.extractor.request(
url, method="POST", data=data, auth=(self.client_id, ""))
if response.status_code != 200:
raise exception.AuthenticationError('"{} ({})"'.format(
response.json().get("message"), response.status_code))
@@ -234,7 +234,8 @@ class RedditAPI():
url = "https://oauth.reddit.com" + endpoint
params["raw_json"] = 1
self.authenticate()
response = self.session.get(url, params=params)
response = self.extractor.request(
url, params=params, expect=range(400, 500))
remaining = response.headers.get("x-ratelimit-remaining")
if remaining and float(remaining) < 2:
wait = int(response.headers["x-ratelimit-reset"])