[deviantart:stash] extract more metadata (#7397)

- stash_id
- stash_name
- stash_description
- stash_folder
- stash_parent

gallery_dl/extractor/deviantart.py

@@ -867,6 +867,9 @@ x2="45.4107524%" y2="71.4898596%" id="app-root-3">\
             )["deviation"]["extended"]["deviationUuid"]
             yield self.api.deviation(deviation_uuid)
 
+    def _unescape_json(self, json):
+        return json.replace('\\"', '"').replace("\\\\", "\\")
+
 
 class DeviantartUserExtractor(DeviantartExtractor):
     """Extractor for an artist's user profile"""
@@ -1046,7 +1049,7 @@ class DeviantartStashExtractor(DeviantartExtractor):
         DeviantartExtractor.__init__(self, match)
         self.user = None
 
-    def deviations(self, stash_id=None):
+    def deviations(self, stash_id=None, stash_data=None):
         if stash_id is None:
             legacy_url, stash_id = self.groups
         else:
@@ -1068,14 +1071,33 @@ class DeviantartStashExtractor(DeviantartExtractor):
                 deviation["_page"] = page
                 deviation["index"] = text.parse_int(text.extr(
                     page, '\\"deviationId\\":', ','))
+
+                deviation["stash_id"] = stash_id
+                if stash_data:
+                    folder = stash_data["folder"]
+                    deviation["stash_name"] = folder["name"]
+                    deviation["stash_folder"] = folder["folderId"]
+                    deviation["stash_parent"] = folder["parentId"] or 0
+                    deviation["stash_description"] = \
+                        folder["richDescription"]["excerpt"]
+                else:
+                    deviation["stash_name"] = ""
+                    deviation["stash_description"] = ""
+                    deviation["stash_folder"] = 0
+                    deviation["stash_parent"] = 0
+
                 yield deviation
                 return
 
+        stash_data = text.extr(page, ',\\"stash\\":', ',\\"@@')
+        if stash_data:
+            stash_data = util.json_loads(self._unescape_json(stash_data))
+
         for sid in text.extract_iter(
                 page, 'href="https://www.deviantart.com/stash/', '"'):
             if sid == stash_id or sid.endswith("#comments"):
                 continue
-            yield from self.deviations(sid)
+            yield from self.deviations(sid, stash_data)
 
 
 class DeviantartFavoriteExtractor(DeviantartExtractor):
@@ -1289,12 +1311,8 @@ class DeviantartDeviationExtractor(DeviantartExtractor):
         self.archive_fmt = ("g_{_username}_{index}{index_file:?_//}."
                             "{extension}")
 
-        additional_media = util.json_loads(
-            # unescape quotes: \" -> "
-            # unescape backslashes: \\ -> \
-            # add closing braces
-            additional_media.replace('\\"', '"').replace("\\\\", "\\") + "}]")
-
+        additional_media = util.json_loads(self._unescape_json(
+            additional_media) + "}]")
         deviation["count"] = 1 + len(additional_media)
         yield deviation
 

test/results/deviantart.py

@@ -412,6 +412,11 @@ __tests__ = (
     "index"         : 778297656,
     "index_base36"  : "cvdmbc",
     "published_time": 1545835767,
+    "stash_description": "",
+    "stash_id"      : "022c83odnaxc",
+    "stash_name"    : "",
+    "stash_folder"  : 0,
+    "stash_parent"  : 0,
     "title"         : "01",
     "url"           : "https://www.deviantart.com/stash/022c83odnaxc",
 },
@@ -441,7 +446,18 @@ __tests__ = (
     "#category": ("", "deviantart", "stash"),
     "#class"   : deviantart.DeviantartStashExtractor,
     "#options" : {"original": False},
-    "#count"   : 4,
+    "#pattern" : (
+        r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmcb-b0178127-de9d-48e2-b95c-8627778b1c20.png\?token=ey.+",
+        r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmc6-a2402b2f-b469-42d2-99ca-a3464b3c5889.png\?token=ey.+",
+        r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbz-adf92e46-0481-4c65-9e3b-e142a86a2d44.png\?token=ey.+",
+        r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbx-08eb6ce5-c892-4f16-8676-668cd2276697.png\?token=ey.+",
+    ),
+
+    "stash_description": """Sta.sh Description\ntest-テスト-"&>""",
+    "stash_id"    : {"021rsajamktz", "099tbcst5u5", "02394fx5fagg", "01okox2yh0o2"},
+    "stash_name"  : """Sta.sh Title test-テスト-"&>""",
+    "stash_folder": 7362377764221985,
+    "stash_parent": 0,
 },
 
 {
@@ -460,6 +476,12 @@ __tests__ = (
     "#class"   : deviantart.DeviantartStashExtractor,
     "#options" : {"original": False},
     "#count"   : 4,
+
+    "stash_description": "",
+    "stash_id"    : {"018dfylek5o1", "0vmi73y92tn", "02g2v51kb8y2", "06nnon5vucx"},
+    "stash_name"  : {"Sta.sh Uploads 147", "1"},
+    "stash_folder": {7382365850253347, 2415594944160654},
+    "stash_parent": {0, 7382365850253347},
 },
 
 {