[rule34vault] update

- implement 'tags' categorization
- don't use 'totalCount' for pagination end
- update tests

gallery_dl/extractor/rule34vault.py

@@ -8,6 +8,7 @@
 
 from .booru import BooruExtractor
 from .. import text
+import collections
 
 BASE_PATTERN = r"(?:https?://)?rule34vault\.com"
 
@@ -19,11 +20,19 @@ class Rule34vaultExtractor(BooruExtractor):
     filename_fmt = "{category}_{id}.{extension}"
     per_page = 100
 
+    TAG_TYPES = {
+        1: "general",
+        2: "copyright",
+        4: "character",
+        8: "artist",
+    }
+
     def _file_url(self, post):
         post_id = post["id"]
         extension = "jpg" if post["type"] == 0 else "mp4"
-        return "{}/posts/{}/{}/{}.{}".format(
+        post["file_url"] = url = "{}/posts/{}/{}/{}.{}".format(
             self.root_cdn, post_id // 1000, post_id, post_id, extension)
+        return url
 
     def _prepare(self, post):
         post.pop("files", None)
@@ -36,6 +45,13 @@ class Rule34vaultExtractor(BooruExtractor):
         if "tags" not in post:
             post.update(self._fetch_post(post["id"]))
 
+        tags = collections.defaultdict(list)
+        for tag in post["tags"]:
+            tags[tag["type"]].append(tag["value"])
+        types = self.TAG_TYPES
+        for type, values in tags.items():
+            post["tags_" + types[type]] = values
+
     def _fetch_post(self, post_id):
         url = "{}/api/v2/post/{}".format(self.root, post_id)
         return self.request(url).json()
@@ -45,19 +61,19 @@ class Rule34vaultExtractor(BooruExtractor):
 
         if params is None:
             params = {}
-        params["CountTotal"] = True
+        params["CountTotal"] = False
         params["Skip"] = self.page_start * self.per_page
         params["take"] = self.per_page
+        threshold = self.per_page
 
         while True:
             data = self.request(url, method="POST", json=params).json()
 
             yield from data["items"]
 
-            if params["Skip"] + params["take"] > data["totalCount"]:
+            if len(data["items"]) < threshold:
                 return
-            if "cursor" in data:
-                params["cursor"] = data["cursor"]
+            params["cursor"] = data.get("cursor")
             params["Skip"] += params["take"]
 
 
@@ -65,7 +81,7 @@ class Rule34vaultPostExtractor(Rule34vaultExtractor):
     subcategory = "post"
     archive_fmt = "{id}"
     pattern = BASE_PATTERN + r"/post/(\d+)"
-    example = "https://rule34vault.com/post/399437"
+    example = "https://rule34vault.com/post/12345"
 
     def posts(self):
         return (self._fetch_post(self.groups[0]),)
@@ -76,7 +92,7 @@ class Rule34vaultPlaylistExtractor(Rule34vaultExtractor):
     directory_fmt = ("{category}", "{playlist_id}")
     archive_fmt = "p_{playlist_id}_{id}"
     pattern = BASE_PATTERN + r"/playlists/view/(\d+)"
-    example = "https://rule34vault.com/playlists/view/2"
+    example = "https://rule34vault.com/playlists/view/12345"
 
     def metadata(self):
         return {"playlist_id": self.groups[0]}
@@ -90,7 +106,7 @@ class Rule34vaultTagExtractor(Rule34vaultExtractor):
     subcategory = "tag"
     directory_fmt = ("{category}", "{search_tags}")
     archive_fmt = "t_{search_tags}_{id}"
-    pattern = BASE_PATTERN + r"/([^/?#]+)$"
+    pattern = BASE_PATTERN + r"/(?!p(?:ost|laylists)/)([^/?#]+)"
     example = "https://rule34vault.com/TAG"
 
     def metadata(self):