diff --git a/docs/configuration.rst b/docs/configuration.rst index 849fc1e1..db9bc04d 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -3862,6 +3862,16 @@ Description Your account's `API key `__ +extractor.pixeldrain.recursive +-------------------------- +Type + ``bool`` +Default + ``false`` +Description + Recursively download files from subfolders. + + extractor.pixiv.include ----------------------- Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index f6e80345..eac33908 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -488,7 +488,8 @@ }, "pixeldrain": { - "api-key": null + "api-key" : null, + "recursive": false }, "pixiv": { diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 91eabef6..cc4b74ed 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -706,7 +706,7 @@ Consider all listed sites to potentially be NSFW. pixeldrain https://pixeldrain.com/ - Albums, Files + Albums, Files, Filesystems diff --git a/gallery_dl/extractor/pixeldrain.py b/gallery_dl/extractor/pixeldrain.py index 83f35773..7a4d1a57 100644 --- a/gallery_dl/extractor/pixeldrain.py +++ b/gallery_dl/extractor/pixeldrain.py @@ -96,3 +96,73 @@ class PixeldrainAlbumExtractor(PixeldrainExtractor): file["date"] = self.parse_datetime(file["date_upload"]) text.nameext_from_url(file["name"], file) yield Message.Url, url, file + + +class PixeldrainFolderExtractor(PixeldrainExtractor): + """Extractor for pixeldrain filesystem files and directories""" + subcategory = "folder" + filename_fmt = "{filename[:230]}.{extension}" + archive_fmt = "{path}_{num}" + pattern = BASE_PATTERN + r"/(?:d|api/filesystem)/([^?]+)" + example = "https://pixeldrain.com/d/abcdefgh" + + def metadata(self, data): + return { + "type" : data["type"], + "path" : data["path"], + "name" : data["name"], + "mime_type" : data["file_type"], + "size" : data["file_size"], + "hash_sha256": data["sha256_sum"], + "date" : self.parse_datetime(data["created"]), + } + + def items(self): + recursive = self.config("recursive") + + url = "{}/api/filesystem/{}".format(self.root, self.groups[0]) + stat = self.request(url + "?stat").json() + + paths = stat["path"] + path = paths[stat["base_index"]] + if path["type"] == "dir": + children = [ + child + for child in stat["children"] + if child["name"] != ".search_index.gz" + ] + else: + children = (path,) + + folder = self.metadata(path) + folder["id"] = paths[0]["id"] + + yield Message.Directory, folder + + num = 0 + for child in children: + if child["type"] == "file": + num += 1 + url = "{}/api/filesystem{}?attach".format( + self.root, child["path"]) + share_url = "{}/d{}".format(self.root, child["path"]) + data = self.metadata(child) + data.update({ + "id" : folder["id"], + "num" : num, + "url" : url, + "share_url": share_url, + }) + data["filename"], _, data["extension"] = \ + child["name"].rpartition(".") + yield Message.Url, url, data + + elif child["type"] == "dir": + if recursive: + url = "{}/d{}".format(self.root, child["path"]) + child["_extractor"] = PixeldrainFolderExtractor + yield Message.Queue, url, child + + else: + self.log.debug("'%s' is of unknown type (%s)", + child.get("name"), child["type"]) diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index a7ee6592..1eda10c9 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -309,6 +309,9 @@ SUBCATEGORY_MAP = { "created": "Created Pins", "allpins": "All Pins", }, + "pixeldrain": { + "folder": "Filesystems", + }, "pixiv": { "me" : "pixiv.me Links", "novel-bookmark": "Novel Bookmarks", diff --git a/test/results/pixeldrain.py b/test/results/pixeldrain.py index ed944885..80a5bea9 100644 --- a/test/results/pixeldrain.py +++ b/test/results/pixeldrain.py @@ -97,4 +97,98 @@ __tests__ = ( "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", }, +{ + "#url" : "https://pixeldrain.com/d/8xz8hcYJ", + "#category": ("", "pixeldrain", "folder"), + "#class" : pixeldrain.PixeldrainFolderExtractor, + "#urls" : "https://pixeldrain.com/api/filesystem/8xz8hcYJ?attach", + "#sha1_content": "edfea851cad717f5643cb94ac04b32335611acf2", + + "date" : "dt:2025-05-19 15:27:54", + "extension" : "mp4", + "filename" : "test", + "id" : "8xz8hcYJ", + "mime_type" : "video/mp4", + "name" : "test.mp4", + "path" : "/8xz8hcYJ", + "hash_sha256": "c6293d8359cb84723bbf8cf355da6cf1ef9c3e8b3d465110e91db485e53ada54", + "share_url" : "https://pixeldrain.com/d/8xz8hcYJ", + "size" : 3026, + "type" : "file", +}, + +{ + "#url" : "https://pixeldrain.com/api/filesystem/8xz8hcYJ", + "#category": ("", "pixeldrain", "folder"), + "#class" : pixeldrain.PixeldrainFolderExtractor, + "#urls" : "https://pixeldrain.com/api/filesystem/8xz8hcYJ?attach", + "#sha1_content": "edfea851cad717f5643cb94ac04b32335611acf2", + + "date" : "dt:2025-05-19 15:27:54", + "extension" : "mp4", + "filename" : "test", + "id" : "8xz8hcYJ", + "mime_type" : "video/mp4", + "name" : "test.mp4", + "path" : "/8xz8hcYJ", + "hash_sha256": "c6293d8359cb84723bbf8cf355da6cf1ef9c3e8b3d465110e91db485e53ada54", + "share_url" : "https://pixeldrain.com/d/8xz8hcYJ", + "size" : 3026, + "type" : "file", +}, + +{ + "#url" : "https://pixeldrain.com/d/DkdR6QRh", + "#comment" : "dir with file", + "#category": ("", "pixeldrain", "folder"), + "#class" : pixeldrain.PixeldrainFolderExtractor, + "#urls" : ("https://pixeldrain.com/api/filesystem/DkdR6QRh/test.mp4?attach"), + + "id": "DkdR6QRh", +}, + +{ + "#url" : "https://pixeldrain.com/d/STAcYjEh", + "#comment" : "dir with subdir", + "#category": ("", "pixeldrain", "folder"), + "#class" : pixeldrain.PixeldrainFolderExtractor, + + "id": "STAcYjEh", +}, + +{ + "#url" : "https://pixeldrain.com/d/qTnZkhCJ", + "#comment" : "dir with subdir and files", + "#category": ("", "pixeldrain", "folder"), + "#class" : pixeldrain.PixeldrainFolderExtractor, + "#urls" : ( + "https://pixeldrain.com/api/filesystem/qTnZkhCJ/test1.mp4?attach", + "https://pixeldrain.com/api/filesystem/qTnZkhCJ/test2.mp4?attach", + ), + + "id": "qTnZkhCJ", +}, + +{ + "#url" : "https://pixeldrain.com/d/qTnZkhCJ/subdir/test3.mp4", + "#comment" : "file in subdir", + "#category": ("", "pixeldrain", "folder"), + "#class" : pixeldrain.PixeldrainFolderExtractor, + "#urls" : ( + "https://pixeldrain.com/api/filesystem/qTnZkhCJ/subdir/test3.mp4?attach", + ), + + "date" : "dt:2025-05-20 19:02:08", + "extension" : "mp4", + "filename" : "test3", + "hash_sha256": "c6293d8359cb84723bbf8cf355da6cf1ef9c3e8b3d465110e91db485e53ada54", + "id" : "qTnZkhCJ", + "mime_type" : "video/mp4", + "name" : "test3.mp4", + "path" : "/qTnZkhCJ/subdir/test3.mp4", + "share_url" : "https://pixeldrain.com/d/qTnZkhCJ/subdir/test3.mp4", + "size" : 3026, + "type" : "file", +}, + )