use extractor.request for all other API calls

- deviantart
- pawoo
- pixiv
- reddit

gallery_dl/extractor/deviantart.py

@@ -458,7 +458,7 @@ class DeviantartAPI():
     CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1"
 
     def __init__(self, extractor):
-        self.session = extractor.session
+        self.extractor = extractor
         self.log = extractor.log
         self.headers = {}
 
@@ -553,8 +553,7 @@ class DeviantartAPI():
 
     def authenticate(self, refresh_token):
         """Authenticate the application by requesting an access token"""
-        access_token = self._authenticate_impl(refresh_token)
+        self.headers["Authorization"] = self._authenticate_impl(refresh_token)
-        self.headers["Authorization"] = access_token
 
     @cache(maxage=3590, keyarg=1)
     def _authenticate_impl(self, refresh_token):
@@ -569,7 +568,8 @@ class DeviantartAPI():
             data = {"grant_type": "client_credentials"}
 
         auth = (self.client_id, self.client_secret)
-        response = self.session.post(url, data=data, auth=auth)
+        response = self.extractor.request(
+            url, method="POST", data=data, auth=auth)
         data = response.json()
 
         if response.status_code != 200:
@@ -587,8 +587,12 @@ class DeviantartAPI():
                 time.sleep(2 ** self.delay)
 
             self.authenticate(None if public else self.refresh_token)
-            response = self.session.get(
+            response = self.extractor.request(
-                url, headers=self.headers, params=params)
+                url,
+                params=params,
+                headers=self.headers,
+                expect=range(400, 500),
+            )
             data = response.json()
             status = response.status_code
 
@@ -604,12 +608,12 @@ class DeviantartAPI():
             self.log.debug(response.text)
             msg = "API responded with {} {}".format(
                 status, response.reason)
-            if 400 <= status < 500 and status != 429:
+            if status == 429:
-                self.log.error(msg)
-                return data
-            else:
                 self.delay += 1
                 self.log.warning("%s. Using %ds delay.", msg, 2 ** self.delay)
+            else:
+                self.log.error(msg)
+                return data
 
     def _pagination(self, endpoint, params):
         public = True

gallery_dl/extractor/pawoo.py

@@ -105,17 +105,16 @@ class MastodonAPI():
     def __init__(self, extractor, root="https://pawoo.net",
                  access_token=("286462927198d0cf3e24683e91c8259a"
                                "ac4367233064e0570ca18df2ac65b226")):
-        access_token = extractor.config("access-token", access_token)
-        self.session = extractor.session
-        self.session.headers["Authorization"] = "Bearer " + access_token
         self.root = root
+        self.extractor = extractor
+        extractor.session.headers["Authorization"] = "Bearer {}".format(
+            extractor.config("access-token", access_token))
 
     def account_search(self, query, limit=40):
         """Search for content"""
-        response = self.session.get(
+        url = "{}/api/v1/accounts/search".format(self.root)
-            self.root + "/api/v1/accounts/search",
+        params = {"q": query, "limit": limit}
-            params={"q": query, "limit": limit},
+        response = self.extractor.request(url, params=params)
-        )
         return self._parse(response)
 
     def account_statuses(self, account_id):
@@ -123,15 +122,14 @@ class MastodonAPI():
         url = "{}/api/v1/accounts/{}/statuses?only_media=1".format(
             self.root, account_id)
         while url:
-            response = self.session.get(url)
+            response = self.extractor.request(url)
             yield from self._parse(response)
             url = response.links.get("next", {}).get("url")
 
     def status(self, status_id):
         """Fetch a Status"""
-        response = self.session.get(
+        url = "{}/api/v1/statuses/{}".format(self.root, status_id)
-            self.root + "/api/v1/statuses/" + status_id
+        response = self.extractor.request(url, expect=(404,))
-        )
         return self._parse(response)
 
     @staticmethod

gallery_dl/extractor/pixiv.py

@@ -406,7 +406,7 @@ class PixivAppAPI():
     CLIENT_SECRET = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj"
 
     def __init__(self, extractor):
-        self.session = extractor.session
+        self.extractor = extractor
         self.log = extractor.log
         self.username, self.password = extractor._get_auth_info()
         self.user = None
@@ -416,7 +416,7 @@ class PixivAppAPI():
         self.client_secret = extractor.config(
             "client-secret", self.CLIENT_SECRET)
 
-        self.session.headers.update({
+        extractor.session.headers.update({
             "App-OS": "ios",
             "App-OS-Version": "10.3.1",
             "App-Version": "6.7.1",
@@ -426,9 +426,8 @@ class PixivAppAPI():
 
     def login(self):
         """Login and gain an access token"""
-        self.user, auth = self._login_impl(
+        self.user, auth = self._login_impl(self.username, self.password)
-            self.username, self.password)
+        self.extractor.session.headers["Authorization"] = auth
-        self.session.headers["Authorization"] = auth
 
     @cache(maxage=3590, keyarg=1)
     def _login_impl(self, username, password):
@@ -450,7 +449,8 @@ class PixivAppAPI():
             data["username"] = username
             data["password"] = password
 
-        response = self.session.post(url, data=data)
+        response = self.extractor.request(
+            url, method="POST", data=data, expect=(400,))
         if response.status_code >= 400:
             raise exception.AuthenticationError()
 
@@ -496,7 +496,8 @@ class PixivAppAPI():
         url = "https://app-api.pixiv.net/" + endpoint
 
         self.login()
-        response = self.session.get(url, params=params)
+        response = self.extractor.request(
+            url, params=params, expect=range(400, 500))
 
         if 200 <= response.status_code < 400:
             return response.json()

gallery_dl/extractor/reddit.py

@@ -160,7 +160,6 @@ class RedditAPI():
         self.morecomments = extractor.config("morecomments", False)
         self.refresh_token = extractor.config("refresh-token")
         self.log = extractor.log
-        self.session = extractor.session
 
         client_id = extractor.config("client-id", self.CLIENT_ID)
         user_agent = extractor.config("user-agent", self.USER_AGENT)
@@ -172,7 +171,7 @@ class RedditAPI():
                 "override either both or none of them.")
         else:
             self.client_id = client_id
-            self.session.headers["User-Agent"] = user_agent
+            extractor.session.headers["User-Agent"] = user_agent
 
     def submission(self, submission_id):
         """Fetch the (submission, comments)=-tuple for a submission id"""
@@ -209,7 +208,7 @@ class RedditAPI():
     def authenticate(self):
         """Authenticate the application by requesting an access token"""
         access_token = self._authenticate_impl(self.refresh_token)
-        self.session.headers["Authorization"] = access_token
+        self.extractor.session.headers["Authorization"] = access_token
 
     @cache(maxage=3590, keyarg=1)
     def _authenticate_impl(self, refresh_token=None):
@@ -224,7 +223,8 @@ class RedditAPI():
             data = {"grant_type": ("https://oauth.reddit.com/"
                                    "grants/installed_client"),
                     "device_id": "DO_NOT_TRACK_THIS_DEVICE"}
-        response = self.session.post(url, data=data, auth=(self.client_id, ""))
+        response = self.extractor.request(
+            url, method="POST", data=data, auth=(self.client_id, ""))
         if response.status_code != 200:
             raise exception.AuthenticationError('"{} ({})"'.format(
                 response.json().get("message"), response.status_code))
@@ -234,7 +234,8 @@ class RedditAPI():
         url = "https://oauth.reddit.com" + endpoint
         params["raw_json"] = 1
         self.authenticate()
-        response = self.session.get(url, params=params)
+        response = self.extractor.request(
+            url, params=params, expect=range(400, 500))
         remaining = response.headers.get("x-ratelimit-remaining")
         if remaining and float(remaining) < 2:
             wait = int(response.headers["x-ratelimit-reset"])