diff --git a/docs/configuration.rst b/docs/configuration.rst index ee254a9a..c4a31cc1 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2286,6 +2286,16 @@ Description | Leave ``SIZE`` empty to download the regular, small avatar format. +extractor.deviantart.folder.subfolders +-------------------------------------- +Type + ``bool`` +Default + ``true`` +Description + Also extract subfolder content. + + extractor.discord.embeds ------------------------ Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 16497bd0..3df119e4 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -235,6 +235,9 @@ "avatar": { "formats": null + }, + "folder": { + "subfolders": true } }, "exhentai": diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index bd81a220..200840dd 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -687,10 +687,18 @@ x2="45.4107524%" y2="71.4898596%" id="app-root-3">\ for folder in folders: if match(folder["name"]): return folder + elif folder["has_subfolders"]: + for subfolder in folder["subfolders"]: + if match(subfolder["name"]): + return subfolder else: for folder in folders: if folder["folderid"] == uuid: return folder + elif folder["has_subfolders"]: + for subfolder in folder["subfolders"]: + if subfolder["folderid"] == uuid: + return subfolder raise exception.NotFoundError("folder") def _folder_urls(self, folders, category, extractor): @@ -988,13 +996,36 @@ class DeviantartFolderExtractor(DeviantartExtractor): def deviations(self): folders = self.api.gallery_folders(self.user) folder = self._find_folder(folders, self.folder_name, self.folder_id) + + # Leaving this here for backwards compatibility self.folder = { "title": folder["name"], "uuid" : folder["folderid"], "index": self.folder_id, "owner": self.user, + "parent_uuid": folder["parent"], } - return self.api.gallery(self.user, folder["folderid"], self.offset) + + if folder.get("subfolder"): + self.folder["parent_folder"] = folder["parent_folder"] + self.archive_fmt = "F_{folder[parent_uuid]}_{index}.{extension}" + + if self.flat: + self.directory_fmt = ("{category}", "{username}", + "{folder[parent_folder]}") + else: + self.directory_fmt = ("{category}", "{username}", + "{folder[parent_folder]}", + "{folder[title]}") + + if folder.get("has_subfolders") and self.config("subfolders", True): + for subfolder in folder["subfolders"]: + subfolder["parent_folder"] = folder["name"] + subfolder["subfolder"] = True + yield from self._folder_urls( + folder["subfolders"], "gallery", DeviantartFolderExtractor) + + yield from self.api.gallery(self.user, folder["folderid"], self.offset) def prepare(self, deviation): DeviantartExtractor.prepare(self, deviation) @@ -1376,7 +1407,7 @@ class DeviantartOAuthAPI(): def __init__(self, extractor): self.extractor = extractor self.log = extractor.log - self.headers = {"dA-minor-version": "20200519"} + self.headers = {"dA-minor-version": "20210526"} self._warn_429 = True self.delay = extractor.config("wait-min", 0) diff --git a/test/results/deviantart.py b/test/results/deviantart.py index 88b5f7af..89f2248e 100644 --- a/test/results/deviantart.py +++ b/test/results/deviantart.py @@ -350,6 +350,33 @@ __tests__ = ( "#count" : 1, }, +{ + "#url" : "https://www.deviantart.com/avapithecus/gallery/71028779/drake-hero", + "#comment" : "main folder + subfolders", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"subfolders": True, "original": False, "image-range": "1"}, + "#pattern" : ( + r"https://www.deviantart.com/Avapithecus/gallery/6FCC57FA-F21D-14CC-5E0F-BB76479B6555/Folk Hero", + r"https://www.deviantart.com/Avapithecus/gallery/8D5E41B0-4BF5-649B-6620-B1D89C6D6BCE/Denizens of Suwarrow", + r"https://www.deviantart.com/Avapithecus/gallery/7FE4D499-E883-23D2-1659-1B64CA67358D/Beyond Suwarrow", + r"https://www.deviantart.com/Avapithecus/gallery/38AAB41C-F0F1-4DE9-6FB9-D3493CD77D01/The Drake Number", + r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/c5e7b050-4923-4473-b8c0-ca0bc1c1b1fe/dgqc5py-3371d62e-465f-4b17-bd23-5005517fc68d.jpg/v1/fill/.+", + ), +}, + +{ + "#url" : "https://www.deviantart.com/avapithecus/gallery/87003033/the-drake-number", + "#comment" : "subfolder", + "#category": ("", "deviantart", "folder"), + "#class" : deviantart.DeviantartFolderExtractor, + "#options" : {"original": False}, + "#pattern" : ( + r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/c5e7b050-4923-4473-b8c0-ca0bc1c1b1fe/dfu7xyj-44d1a551-dbdc-4614-baee-82612fb044a6.jpg\?token=ey.+", + r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/c5e7b050-4923-4473-b8c0-ca0bc1c1b1fe/deeoxic-932e966c-6d3b-473c-8053-ed7bad05813a.jpg/v1/fill/.+", + ), +}, + { "#url" : "https://shimoda7.deviantart.com/gallery/722019/Miscellaneous", "#category": ("", "deviantart", "folder"),