[deviantart] support watchers-only/paid deviations (#995)

gallery_dl/extractor/deviantart.py

@@ -54,6 +54,7 @@ class DeviantartExtractor(Extractor):
             self._update_content = self._update_content_image
             self.original = True
 
+        self._premium_cache = {}
         self.commit_journal = {
             "html": self._commit_journal_html,
             "text": self._commit_journal_text,
@@ -65,6 +66,8 @@ class DeviantartExtractor(Extractor):
 
     def items(self):
         self.api = DeviantartOAuthAPI(self)
+        if not self.api.refresh_token_key:
+            self._fetch_premium = self._fetch_premium_notoken
 
         if self.user:
             profile = self.api.user_profile(self.user)
@@ -82,6 +85,10 @@ class DeviantartExtractor(Extractor):
                 yield Message.Queue, url, data
                 continue
 
+            if "premium_folder_data" in deviation:
+                if not self._fetch_premium(deviation):
+                    continue
+
             self.prepare(deviation)
             yield Message.Directory, deviation
 
@@ -260,7 +267,9 @@ class DeviantartExtractor(Extractor):
         return [(url + folder["name"], folder) for folder in folders]
 
     def _update_content_default(self, deviation, content):
-        content.update(self.api.deviation_download(deviation["deviationid"]))
+        public = "premium_folder_data" not in deviation
+        data = self.api.deviation_download(deviation["deviationid"], public)
+        content.update(data)
 
     def _update_content_image(self, deviation, content):
         data = self.api.deviation_download(deviation["deviationid"])
@@ -289,6 +298,41 @@ class DeviantartExtractor(Extractor):
                 return response
             self.wait(seconds=180)
 
+    def _fetch_premium(self, deviation):
+        cache = self._premium_cache
+
+        if deviation["deviationid"] not in cache:
+
+            # check accessibility
+            dev = self.api.deviation(deviation["deviationid"], False)
+            has_access = dev["premium_folder_data"]["has_access"]
+
+            if has_access:
+                self.log.info("Fetching premium folder data")
+            else:
+                self.log.warning("Unable to access premium content (type: %s)",
+                                 dev["premium_folder_data"]["type"])
+            # fill cache
+            for dev in self.api.gallery(
+                deviation["author"]["username"],
+                deviation["premium_folder_data"]["gallery_id"],
+                public=False,
+            ):
+                cache[dev["deviationid"]] = dev if has_access else None
+
+        data = cache[deviation["deviationid"]]
+        if data:
+            deviation.update(data)
+            return True
+        return False
+
+    def _fetch_premium_notoken(self, deviation):
+        if not self._premium_cache:
+            self.log.warning(
+                "Unable to access premium content (no refresh-token)")
+            self._premium_cache = True
+        return False
+
 
 class DeviantartUserExtractor(DeviantartExtractor):
     """Extractor for an artist's user profile"""
@@ -895,27 +939,27 @@ class DeviantartOAuthAPI():
                   "mature_content": self.mature}
         return self._pagination_folders(endpoint, params)
 
-    def deviation(self, deviation_id):
+    def deviation(self, deviation_id, public=True):
         """Query and return info about a single Deviation"""
         endpoint = "deviation/" + deviation_id
-        deviation = self._call(endpoint)
+        deviation = self._call(endpoint, public=public)
         if self.metadata:
             self._metadata((deviation,))
         if self.folders:
             self._folders((deviation,))
         return deviation
 
-    def deviation_content(self, deviation_id):
+    def deviation_content(self, deviation_id, public=False):
         """Get extended content of a single Deviation"""
         endpoint = "deviation/content"
         params = {"deviationid": deviation_id}
-        return self._call(endpoint, params, public=False)
+        return self._call(endpoint, params, public=public)
 
-    def deviation_download(self, deviation_id):
+    def deviation_download(self, deviation_id, public=True):
         """Get the original file download (if allowed)"""
         endpoint = "deviation/download/" + deviation_id
         params = {"mature_content": self.mature}
-        return self._call(endpoint, params)
+        return self._call(endpoint, params, public=public)
 
     def deviation_metadata(self, deviations):
         """ Fetch deviation metadata for a set of deviations"""
@@ -928,12 +972,12 @@ class DeviantartOAuthAPI():
         params = {"mature_content": self.mature}
         return self._call(endpoint, params)["metadata"]
 
-    def gallery(self, username, folder_id="", offset=0, extend=True):
+    def gallery(self, username, folder_id, offset=0, extend=True, public=True):
         """Yield all Deviation-objects contained in a gallery folder"""
         endpoint = "gallery/" + folder_id
         params = {"username": username, "offset": offset, "limit": 24,
                   "mature_content": self.mature, "mode": "newest"}
-        return self._pagination(endpoint, params, extend)
+        return self._pagination(endpoint, params, extend, public)
 
     def gallery_all(self, username, offset=0):
         """Yield all Deviation-objects of a specific user"""
@@ -1020,8 +1064,8 @@ class DeviantartOAuthAPI():
                 self.log.error(msg)
                 return data
 
-    def _pagination(self, endpoint, params, extend=True):
+    def _pagination(self, endpoint, params, extend=True, public=True):
-        public = warn = True
+        warn = True
         while True:
             data = self._call(endpoint, params, public=public)
             if "results" not in data: