[deviantart] try to work around journal/status API changes (#5916)

The new /user/profile/posts endpoint is documented to
"return all journals & status updates for a given user in a single feed"
but it does not do this.

There are usually several or even all status updates missing.

docs/supportedsites.md

@@ -178,7 +178,7 @@ Consider all listed sites to potentially be NSFW.
 <tr>
     <td>DeviantArt</td>
     <td>https://www.deviantart.com/</td>
-    <td>Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
+    <td>Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
     <td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
 </tr>
 <tr>

gallery_dl/extractor/deviantart.py

@@ -846,55 +846,6 @@ class DeviantartStatusExtractor(DeviantartExtractor):
             )
 
 
-class DeviantartPopularExtractor(DeviantartExtractor):
-    """Extractor for popular deviations"""
-    subcategory = "popular"
-    directory_fmt = ("{category}", "Popular",
-                     "{popular[range]}", "{popular[search]}")
-    archive_fmt = "P_{popular[range]}_{popular[search]}_{index}.{extension}"
-    pattern = (r"(?:https?://)?www\.deviantart\.com/(?:"
-               r"(?:deviations/?)?\?order=(popular-[^/?#]+)"
-               r"|((?:[\w-]+/)*)(popular-[^/?#]+)"
-               r")/?(?:\?([^#]*))?")
-    example = "https://www.deviantart.com/popular-24-hours/"
-
-    def __init__(self, match):
-        DeviantartExtractor.__init__(self, match)
-        self.user = ""
-
-        trange1, path, trange2, query = match.groups()
-        query = text.parse_query(query)
-        self.search_term = query.get("q")
-
-        trange = trange1 or trange2 or query.get("order", "")
-        if trange.startswith("popular-"):
-            trange = trange[8:]
-        self.time_range = {
-            "newest"      : "now",
-            "most-recent" : "now",
-            "this-week"   : "1week",
-            "this-month"  : "1month",
-            "this-century": "alltime",
-            "all-time"    : "alltime",
-        }.get(trange, "alltime")
-
-        self.popular = {
-            "search": self.search_term or "",
-            "range" : trange or "all-time",
-            "path"  : path.strip("/") if path else "",
-        }
-
-    def deviations(self):
-        if self.time_range == "now":
-            return self.api.browse_newest(self.search_term, self.offset)
-        return self.api.browse_popular(
-            self.search_term, self.time_range, self.offset)
-
-    def prepare(self, deviation):
-        DeviantartExtractor.prepare(self, deviation)
-        deviation["popular"] = self.popular
-
-
 class DeviantartTagExtractor(DeviantartExtractor):
     """Extractor for deviations from tag searches"""
     subcategory = "tag"
@@ -1095,7 +1046,7 @@ class DeviantartFollowingExtractor(DeviantartExtractor):
 class DeviantartOAuthAPI():
     """Interface for the DeviantArt OAuth API
 
-    Ref: https://www.deviantart.com/developers/http/v1/20160316
+    https://www.deviantart.com/developers/http/v1/20160316
     """
     CLIENT_ID = "5388"
     CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1"
@@ -1188,29 +1139,6 @@ class DeviantartOAuthAPI():
                   "mature_content": self.mature}
         return self._pagination(endpoint, params, public=False, unpack=True)
 
-    def browse_newest(self, query=None, offset=0):
-        """Browse newest deviations"""
-        endpoint = "/browse/newest"
-        params = {
-            "q"             : query,
-            "limit"         : 120,
-            "offset"        : offset,
-            "mature_content": self.mature,
-        }
-        return self._pagination(endpoint, params)
-
-    def browse_popular(self, query=None, timerange=None, offset=0):
-        """Yield popular deviations"""
-        endpoint = "/browse/popular"
-        params = {
-            "q"             : query,
-            "limit"         : 120,
-            "timerange"     : timerange,
-            "offset"        : offset,
-            "mature_content": self.mature,
-        }
-        return self._pagination(endpoint, params)
-
     def browse_tags(self, tag, offset=0):
         """ Browse a tag """
         endpoint = "/browse/tags"
@@ -1223,11 +1151,12 @@ class DeviantartOAuthAPI():
         return self._pagination(endpoint, params)
 
     def browse_user_journals(self, username, offset=0):
-        """Yield all journal entries of a specific user"""
+        journals = filter(
-        endpoint = "/browse/user/journals"
+            lambda post: "/journal/" in post["url"],
-        params = {"username": username, "offset": offset, "limit": 50,
+            self.user_profile_posts(username))
-                  "mature_content": self.mature, "featured": "false"}
+        if offset:
-        return self._pagination(endpoint, params)
+            journals = util.advance(journals, offset)
+        return journals
 
     def collections(self, username, folder_id, offset=0):
         """Yield all Deviation-objects contained in a collection folder"""
@@ -1339,18 +1268,6 @@ class DeviantartOAuthAPI():
                   "mature_content": self.mature}
         return self._pagination_list(endpoint, params)
 
-    @memcache(keyarg=1)
-    def user_profile(self, username):
-        """Get user profile information"""
-        endpoint = "/user/profile/" + username
-        return self._call(endpoint, fatal=False)
-
-    def user_statuses(self, username, offset=0):
-        """Yield status updates of a specific user"""
-        endpoint = "/user/statuses/"
-        params = {"username": username, "offset": offset, "limit": 50}
-        return self._pagination(endpoint, params)
-
     def user_friends(self, username, offset=0):
         """Get the users list of friends"""
         endpoint = "/user/friends/" + username
@@ -1382,6 +1299,27 @@ class DeviantartOAuthAPI():
             endpoint, method="POST", public=False, fatal=False,
         ).get("success")
 
+    @memcache(keyarg=1)
+    def user_profile(self, username):
+        """Get user profile information"""
+        endpoint = "/user/profile/" + username
+        return self._call(endpoint, fatal=False)
+
+    def user_profile_posts(self, username):
+        endpoint = "/user/profile/posts"
+        params = {"username": username, "limit": 50,
+                  "mature_content": self.mature}
+        return self._pagination(endpoint, params)
+
+    def user_statuses(self, username, offset=0):
+        """Yield status updates of a specific user"""
+        statuses = filter(
+            lambda post: "/status-update/" in post["url"],
+            self.user_profile_posts(username))
+        if offset:
+            statuses = util.advance(statuses, offset)
+        return statuses
+
     def authenticate(self, refresh_token_key):
         """Authenticate the application by requesting an access token"""
         self.headers["Authorization"] = \
@@ -1470,7 +1408,7 @@ class DeviantartOAuthAPI():
                     self.log.error(msg)
                 return data
 
-    def _switch_tokens(self, results, params):
+    def _should_switch_tokens(self, results, params):
         if len(results) < params["limit"]:
             return True
 
@@ -1502,7 +1440,7 @@ class DeviantartOAuthAPI():
                 results = [item["journal"] for item in results
                            if "journal" in item]
             if extend:
-                if public and self._switch_tokens(results, params):
+                if public and self._should_switch_tokens(results, params):
                     if self.refresh_token_key:
                         self.log.debug("Switching to private access token")
                         public = False
@@ -1546,6 +1484,11 @@ class DeviantartOAuthAPI():
                     return
                 params["offset"] = int(params["offset"]) + len(results)
 
+    def _pagination_list(self, endpoint, params, key="results"):
+        result = []
+        result.extend(self._pagination(endpoint, params, False, key=key))
+        return result
+
     @staticmethod
     def _shared_content(results):
         """Return an iterable of shared deviations in 'results'"""
@@ -1554,11 +1497,6 @@ class DeviantartOAuthAPI():
                 if "deviation" in item:
                     yield item["deviation"]
 
-    def _pagination_list(self, endpoint, params, key="results"):
-        result = []
-        result.extend(self._pagination(endpoint, params, False, key=key))
-        return result
-
     def _metadata(self, deviations):
         """Add extended metadata to each deviation object"""
         if len(deviations) <= self.limit:

test/results/deviantart.py

@@ -48,7 +48,7 @@ __tests__ = (
 
     "allows_comments" : bool,
     "author"          : {
-        "type"    : "regular",
+        "type"    : "premium",
         "usericon": str,
         "userid"  : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B",
         "username": "shimoda7",
@@ -210,7 +210,7 @@ __tests__ = (
     "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f",
 
     "author"         : {
-        "type"    : "regular",
+        "type"    : "premium",
         "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4",
         "userid"  : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B",
         "username": "shimoda7",
@@ -502,7 +502,7 @@ __tests__ = (
     "#url"     : "https://www.deviantart.com/angrywhitewanker/posts/journals/",
     "#category": ("", "deviantart", "journal"),
     "#class"   : deviantart.DeviantartJournalExtractor,
-    "#sha1_url": "38db2a0d3a587a7e0f9dba7ff7d274610ebefe44",
+    "#sha1_url": "48aeed5631763d96f5391d2177ea72d9fdbee4e5",
 },
 
 {
@@ -616,30 +616,6 @@ __tests__ = (
     "#sha1_url": "10a336bdee7b9692919461443a7dde44d495818c",
 },
 
-{
-    "#url"     : "https://www.deviantart.com/?order=popular-all-time",
-    "#category": ("", "deviantart", "popular"),
-    "#class"   : deviantart.DeviantartPopularExtractor,
-    "#options" : {"original": False},
-    "#range"   : "1-30",
-    "#count"   : 30,
-},
-
-{
-    "#url"     : "https://www.deviantart.com/popular-24-hours/?q=tree+house",
-    "#category": ("", "deviantart", "popular"),
-    "#class"   : deviantart.DeviantartPopularExtractor,
-    "#options" : {"original": False},
-    "#range"   : "1-30",
-    "#count"   : 30,
-},
-
-{
-    "#url"     : "https://www.deviantart.com/artisan/popular-all-time/?q=tree",
-    "#category": ("", "deviantart", "popular"),
-    "#class"   : deviantart.DeviantartPopularExtractor,
-},
-
 {
     "#url"     : "https://www.deviantart.com/tag/nature",
     "#category": ("", "deviantart", "tag"),
@@ -811,7 +787,7 @@ __tests__ = (
     "#category": ("", "deviantart", "deviation"),
     "#class"   : deviantart.DeviantartDeviationExtractor,
     "#pattern" : """text:<!DOCTYPE html>\n""",
-    "#sha1_url": "d34b2c9f873423e665a1b8ced20fcb75951694a3",
+    "#sha1_url": "37302947642d1e53392ef8ee9b3f473a3c578e7c",
 },
 
 {
@@ -820,7 +796,7 @@ __tests__ = (
     "#category": ("", "deviantart", "deviation"),
     "#class"   : deviantart.DeviantartDeviationExtractor,
     "#pattern" : """text:<!DOCTYPE html>\n""",
-    "#sha1_url": "e2e0044bd255304412179b6118536dbd9bb3bb0e",
+    "#sha1_url": "8ca1dc8df53d3707c778d08a604f9ad9ddba7469",
 },
 
 {
@@ -829,6 +805,7 @@ __tests__ = (
     "#category": ("", "deviantart", "deviation"),
     "#class"   : deviantart.DeviantartDeviationExtractor,
     "#count"   : 0,
+    "#exception": exception.NotFoundError,
 },
 
 {
@@ -866,7 +843,7 @@ __tests__ = (
     "#url"     : "https://www.deviantart.com/view/706871727",
     "#category": ("", "deviantart", "deviation"),
     "#class"   : deviantart.DeviantartDeviationExtractor,
-    "#sha1_content": "87dff6056fc9a2bf77f75317a1e00e18451b3c80",
+    "#sha1_content": "4d013515e72dec1e3977c82fd71ce4b15b8bd856",
 },
 
 {