merge #7220: [deviantart] add subfolder support (#4988 #7185)

docs/configuration.rst

@@ -2286,6 +2286,16 @@ Description
     | Leave ``SIZE`` empty to download the regular, small avatar format.
 
 
+extractor.deviantart.folder.subfolders
+--------------------------------------
+Type
+    ``bool``
+Default
+    ``true``
+Description
+    Also extract subfolder content.
+
+
 extractor.discord.embeds
 ------------------------
 Type

docs/gallery-dl.conf

@@ -235,6 +235,9 @@
 
             "avatar": {
                 "formats": null
+            },
+            "folder": {
+                "subfolders": true
             }
         },
         "exhentai":

gallery_dl/extractor/deviantart.py

@@ -687,10 +687,18 @@ x2="45.4107524%" y2="71.4898596%" id="app-root-3">\
             for folder in folders:
                 if match(folder["name"]):
                     return folder
+                elif folder["has_subfolders"]:
+                    for subfolder in folder["subfolders"]:
+                        if match(subfolder["name"]):
+                            return subfolder
         else:
             for folder in folders:
                 if folder["folderid"] == uuid:
                     return folder
+                elif folder["has_subfolders"]:
+                    for subfolder in folder["subfolders"]:
+                        if subfolder["folderid"] == uuid:
+                            return subfolder
         raise exception.NotFoundError("folder")
 
     def _folder_urls(self, folders, category, extractor):
@@ -988,13 +996,36 @@ class DeviantartFolderExtractor(DeviantartExtractor):
     def deviations(self):
         folders = self.api.gallery_folders(self.user)
         folder = self._find_folder(folders, self.folder_name, self.folder_id)
+
+        # Leaving this here for backwards compatibility
         self.folder = {
             "title": folder["name"],
             "uuid" : folder["folderid"],
             "index": self.folder_id,
             "owner": self.user,
+            "parent_uuid": folder["parent"],
         }
-        return self.api.gallery(self.user, folder["folderid"], self.offset)
+
+        if folder.get("subfolder"):
+            self.folder["parent_folder"] = folder["parent_folder"]
+            self.archive_fmt = "F_{folder[parent_uuid]}_{index}.{extension}"
+
+            if self.flat:
+                self.directory_fmt = ("{category}", "{username}",
+                                      "{folder[parent_folder]}")
+            else:
+                self.directory_fmt = ("{category}", "{username}",
+                                      "{folder[parent_folder]}",
+                                      "{folder[title]}")
+
+        if folder.get("has_subfolders") and self.config("subfolders", True):
+            for subfolder in folder["subfolders"]:
+                subfolder["parent_folder"] = folder["name"]
+                subfolder["subfolder"] = True
+            yield from self._folder_urls(
+                folder["subfolders"], "gallery", DeviantartFolderExtractor)
+
+        yield from self.api.gallery(self.user, folder["folderid"], self.offset)
 
     def prepare(self, deviation):
         DeviantartExtractor.prepare(self, deviation)
@@ -1376,7 +1407,7 @@ class DeviantartOAuthAPI():
     def __init__(self, extractor):
         self.extractor = extractor
         self.log = extractor.log
-        self.headers = {"dA-minor-version": "20200519"}
+        self.headers = {"dA-minor-version": "20210526"}
         self._warn_429 = True
 
         self.delay = extractor.config("wait-min", 0)

test/results/deviantart.py

@@ -350,6 +350,33 @@ __tests__ = (
     "#count"   : 1,
 },
 
+{
+    "#url"     : "https://www.deviantart.com/avapithecus/gallery/71028779/drake-hero",
+    "#comment" : "main folder + subfolders",
+    "#category": ("", "deviantart", "folder"),
+    "#class"   : deviantart.DeviantartFolderExtractor,
+    "#options" : {"subfolders": True, "original": False, "image-range": "1"},
+    "#pattern" : (
+        r"https://www.deviantart.com/Avapithecus/gallery/6FCC57FA-F21D-14CC-5E0F-BB76479B6555/Folk Hero",
+        r"https://www.deviantart.com/Avapithecus/gallery/8D5E41B0-4BF5-649B-6620-B1D89C6D6BCE/Denizens of Suwarrow",
+        r"https://www.deviantart.com/Avapithecus/gallery/7FE4D499-E883-23D2-1659-1B64CA67358D/Beyond Suwarrow",
+        r"https://www.deviantart.com/Avapithecus/gallery/38AAB41C-F0F1-4DE9-6FB9-D3493CD77D01/The Drake Number",
+        r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/c5e7b050-4923-4473-b8c0-ca0bc1c1b1fe/dgqc5py-3371d62e-465f-4b17-bd23-5005517fc68d.jpg/v1/fill/.+",
+    ),
+},
+
+{
+    "#url"     : "https://www.deviantart.com/avapithecus/gallery/87003033/the-drake-number",
+    "#comment" : "subfolder",
+    "#category": ("", "deviantart", "folder"),
+    "#class"   : deviantart.DeviantartFolderExtractor,
+    "#options" : {"original": False},
+    "#pattern" : (
+        r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/c5e7b050-4923-4473-b8c0-ca0bc1c1b1fe/dfu7xyj-44d1a551-dbdc-4614-baee-82612fb044a6.jpg\?token=ey.+",
+        r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/c5e7b050-4923-4473-b8c0-ca0bc1c1b1fe/deeoxic-932e966c-6d3b-473c-8053-ed7bad05813a.jpg/v1/fill/.+",
+    ),
+},
+
 {
     "#url"     : "https://shimoda7.deviantart.com/gallery/722019/Miscellaneous",
     "#category": ("", "deviantart", "folder"),