[deviantart] add 'search' extractor

(#538, #1264, #2954, #2970, #3577)

Requires login to fetch any results, since the API endpoint raises an
error for not logged in requests.

TODO: parse HTML search results

docs/supportedsites.md

@@ -148,7 +148,7 @@ Consider all sites to be NSFW unless otherwise known.
 <tr>
     <td>DeviantArt</td>
     <td>https://www.deviantart.com/</td>
-    <td>Collections, Deviations, Favorites, Folders, Galleries, Journals, Popular Images, Scraps, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
+    <td>Collections, Deviations, Favorites, Folders, Galleries, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
     <td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
 </tr>
 <tr>

gallery_dl/extractor/deviantart.py

@@ -6,7 +6,7 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-"""Extract images from https://www.deviantart.com/"""
+"""Extractors for https://www.deviantart.com/"""
 
 from .common import Extractor, Message
 from .. import text, util, exception
@@ -29,21 +29,23 @@ BASE_PATTERN = (
 class DeviantartExtractor(Extractor):
     """Base class for deviantart extractors"""
     category = "deviantart"
+    root = "https://www.deviantart.com"
     directory_fmt = ("{category}", "{username}")
     filename_fmt = "{category}_{index}_{title}.{extension}"
     cookiedomain = None
-    root = "https://www.deviantart.com"
+    cookienames = ("auth", "auth_secure", "userinfo")
+    _warning = True
     _last_request = 0
 
     def __init__(self, match):
         Extractor.__init__(self, match)
-        self.offset = 0
         self.flat = self.config("flat", True)
         self.extra = self.config("extra", False)
         self.original = self.config("original", True)
         self.comments = self.config("comments", False)
         self.user = match.group(1) or match.group(2)
         self.group = False
+        self.offset = 0
         self.api = None
 
         unwatch = self.config("auto-unwatch")
@@ -69,6 +71,17 @@ class DeviantartExtractor(Extractor):
         self.offset += num
         return num
 
+    def login(self):
+        if not self._check_cookies(self.cookienames):
+            username, password = self._get_auth_info()
+            if username:
+                self._update_cookies(_login_impl(self, username, password))
+            elif self._warning:
+                self.log.warning(
+                    "No session cookies or login credentials available. "
+                    "Unable to fetch mature-rated content.")
+            DeviantartExtractor._warning = False
+
     def items(self):
         self.api = DeviantartOAuthAPI(self)
 
@@ -413,6 +426,16 @@ class DeviantartExtractor(Extractor):
             self.log.info("Unwatching %s", username)
             self.api.user_friends_unwatch(username)
 
+    def _eclipse_to_oauth(self, eclipse_api, deviations):
+        for obj in deviations:
+            deviation = obj["deviation"] if "deviation" in obj else obj
+            deviation_uuid = eclipse_api.deviation_extended_fetch(
+                deviation["deviationId"],
+                deviation["author"]["username"],
+                "journal" if deviation["isJournal"] else "art",
+            )["deviation"]["extended"]["deviationUuid"]
+            yield self.api.deviation(deviation_uuid)
+
 
 class DeviantartUserExtractor(DeviantartExtractor):
     """Extractor for an artist's user profile"""
@@ -870,8 +893,7 @@ class DeviantartPopularExtractor(DeviantartExtractor):
                      "{popular[range]}", "{popular[search]}")
     archive_fmt = "P_{popular[range]}_{popular[search]}_{index}.{extension}"
     pattern = (r"(?:https?://)?www\.deviantart\.com/(?:"
-               r"search(?:/deviations)?"
+               r"(?:deviations/?)?\?order=(popular-[^/?#]+)"
-               r"|(?:deviations/?)?\?order=(popular-[^/?#]+)"
                r"|((?:[\w-]+/)*)(popular-[^/?#]+)"
                r")/?(?:\?([^#]*))?")
     test = (
@@ -885,8 +907,6 @@ class DeviantartPopularExtractor(DeviantartExtractor):
             "range": "1-30",
             "count": 30,
         }),
-        ("https://www.deviantart.com/search?q=tree"),
-        ("https://www.deviantart.com/search/deviations?order=popular-1-week"),
         ("https://www.deviantart.com/artisan/popular-all-time/?q=tree"),
     )
 
@@ -1112,33 +1132,42 @@ class DeviantartScrapsExtractor(DeviantartExtractor):
         ("https://shimoda7.deviantart.com/gallery/?catpath=scraps"),
     )
     cookiedomain = ".deviantart.com"
-    cookienames = ("auth", "auth_secure", "userinfo")
-    _warning = True
 
     def deviations(self):
         self.login()
 
         eclipse_api = DeviantartEclipseAPI(self)
-        for obj in eclipse_api.gallery_scraps(self.user, self.offset):
+        return self._eclipse_to_oauth(
-            deviation = obj["deviation"]
+            eclipse_api, eclipse_api.gallery_scraps(self.user, self.offset))
-            deviation_uuid = eclipse_api.deviation_extended_fetch(
-                deviation["deviationId"],
-                deviation["author"]["username"],
-                "journal" if deviation["isJournal"] else "art",
-            )["deviation"]["extended"]["deviationUuid"]
 
-            yield self.api.deviation(deviation_uuid)
 
-    def login(self):
+class DeviantartSearchExtractor(DeviantartExtractor):
-        """Login and obtain session cookies"""
+    """Extractor for deviantart search results"""
-        if not self._check_cookies(self.cookienames):
+    subcategory = "search"
-            username, password = self._get_auth_info()
+    directory_fmt = ("{category}", "Search", "{search_tags}")
-            if username:
+    archive_fmt = "Q_{search_tags}_{index}.{extension}"
-                self._update_cookies(_login_impl(self, username, password))
+    pattern = (r"(?:https?://)?www\.deviantart\.com"
-            elif self._warning:
+               r"/search(?:/deviations)?/?\?([^#]+)")
-                self.log.warning(
+    test = (
-                    "No session cookies set: Unable to fetch mature scraps.")
+        ("https://www.deviantart.com/search?q=tree"),
-            DeviantartScrapsExtractor._warning = False
+        ("https://www.deviantart.com/search/deviations?order=popular-1-week"),
+    )
+    cookiedomain = ".deviantart.com"
+
+    def deviations(self):
+        self.login()
+
+        query = text.parse_query(self.user)
+        self.search = query.get("q", "")
+        self.user = ""
+
+        eclipse_api = DeviantartEclipseAPI(self)
+        return self._eclipse_to_oauth(
+            eclipse_api, eclipse_api.search_deviations(query))
+
+    def prepare(self, deviation):
+        DeviantartExtractor.prepare(self, deviation)
+        deviation["search_tags"] = self.search
 
 
 class DeviantartFollowingExtractor(DeviantartExtractor):
@@ -1618,6 +1647,10 @@ class DeviantartEclipseAPI():
         }
         return self._pagination(endpoint, params)
 
+    def search_deviations(self, params):
+        endpoint = "/da-browse/api/networkbar/search/deviations"
+        return self._pagination(endpoint, params, key="deviations")
+
     def user_watching(self, user, offset=None):
         endpoint = "/da-user-profile/api/module/watching"
         params = {
@@ -1644,11 +1677,11 @@ class DeviantartEclipseAPI():
         except Exception:
             return {"error": response.text}
 
-    def _pagination(self, endpoint, params):
+    def _pagination(self, endpoint, params, key="results"):
         while True:
             data = self._call(endpoint, params)
 
-            results = data.get("results")
+            results = data.get(key)
             if results is None:
                 return
             yield from results
@@ -1656,11 +1689,16 @@ class DeviantartEclipseAPI():
             if not data.get("hasMore"):
                 return
 
-            next_offset = data.get("nextOffset")
+            if "nextCursor" in data:
-            if next_offset:
+                params["offset"] = None
-                params["offset"] = next_offset
+                params["cursor"] = data["nextCursor"]
+            elif "nextOffset" in data:
+                params["offset"] = data["nextOffset"]
+                params["cursor"] = None
+            elif params.get("offset") is None:
+                return
             else:
-                params["offset"] += params["limit"]
+                params["offset"] = int(params["offset"]) + len(results)
 
     def _module_id_watching(self, user):
         url = "{}/{}/about".format(self.extractor.root, user)

test/test_results.py

@@ -323,7 +323,7 @@ def setup_test_config():
     config.set(("extractor", "mangoxo")   , "password", "5zbQF10_5u25259Ma")
 
     for category in ("danbooru", "atfbooru", "aibooru", "e621", "e926",
-                     "instagram", "twitter", "subscribestar",
+                     "instagram", "twitter", "subscribestar", "deviantart",
                      "inkbunny", "tapas", "pillowfort", "mangadex"):
         config.set(("extractor", category), "username", None)