diff --git a/docs/configuration.rst b/docs/configuration.rst index 2da8741c..ca5a1ded 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2231,6 +2231,16 @@ Description (see `getpass() `__). +extractor.cyberfile.recursive +----------------------------- +Type + ``bool`` +Default + ``true`` +Description + Recursively download files from subfolders. + + extractor.[Danbooru].external ----------------------------- Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index bbf04186..429c1a93 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -246,7 +246,8 @@ }, "cyberfile": { - "password": "" + "password" : "", + "recursive": true }, "dankefuerslesen": { diff --git a/gallery_dl/extractor/cyberfile.py b/gallery_dl/extractor/cyberfile.py index 635f296b..ad8ca53a 100644 --- a/gallery_dl/extractor/cyberfile.py +++ b/gallery_dl/extractor/cyberfile.py @@ -56,7 +56,9 @@ class CyberfileFolderExtractor(CyberfileExtractor): url = f"{self.root}/folder/{folder_hash}" folder_num = text.extr(self.request(url).text, "ages('folder', '", "'") - extract_urls = text.re(r'dtfullurl="([^"]+)').findall + extract_folders = text.re(r'sharing-url="([^"]+)').findall + extract_files = text.re(r'dtfullurl="([^"]+)').findall + recursive = self.config("recursive", True) perpage = 600 data = { @@ -67,20 +69,27 @@ class CyberfileFolderExtractor(CyberfileExtractor): "filterOrderBy": "", } resp = self.request_api("/account/ajax/load_files", data) + html = resp["html"] folder = { - "_extractor" : CyberfileFileExtractor, "folder_hash": folder_hash, "folder_num" : text.parse_int(folder_num), "folder" : resp["page_title"], } while True: - urls = extract_urls(resp["html"]) - for url in urls: - yield Message.Queue, url, folder + folders = extract_folders(html) + if recursive and folders: + folder["_extractor"] = CyberfileFolderExtractor + for url in folders: + yield Message.Queue, url, folder - if len(urls) < perpage: + if files := extract_files(html): + folder["_extractor"] = CyberfileFileExtractor + for url in files: + yield Message.Queue, url, folder + + if len(folders) + len(files) < perpage: return data["pageStart"] += 1 resp = self.request_api("/account/ajax/load_files", data) @@ -109,12 +118,12 @@ class CyberfileSharedExtractor(CyberfileExtractor): pos = html.find("") + 26 data = {"_extractor": CyberfileFolderExtractor} - for folder in text.extract_iter(html, 'sharing-url="', '"', pos): - yield Message.Queue, folder, data + for url in text.extract_iter(html, 'sharing-url="', '"', pos): + yield Message.Queue, url, data data = {"_extractor": CyberfileFileExtractor} - for file in text.extract_iter(html, 'dtfullurl="', '"', pos): - yield Message.Queue, file, data + for url in text.extract_iter(html, 'dtfullurl="', '"', pos): + yield Message.Queue, url, data class CyberfileFileExtractor(CyberfileExtractor): diff --git a/test/results/cyberfile.py b/test/results/cyberfile.py index 08b9186e..d48a4eb4 100644 --- a/test/results/cyberfile.py +++ b/test/results/cyberfile.py @@ -104,6 +104,19 @@ __tests__ = ( "#exception": exception.AuthorizationError, }, +{ + "#url" : "https://cyberfile.me/folder/8b17bbfdf25fca19aa51176bd246c97c/Helena_Price_Onlyfans", + "#class" : cyberfile.CyberfileFolderExtractor, + "#results" : ( + "https://cyberfile.me/folder/c2cfdcfcf1a6e6e57de7bc948804b0fc/PICS", + "https://cyberfile.me/folder/bdc7c36e7d4dfdc3fb908a6d3fe1cae5/VIDEO", + ), + + "folder" : "Helena Price Onlyfans", + "folder_hash": "8b17bbfdf25fca19aa51176bd246c97c", + "folder_num" : 18322, +}, + { "#url" : "https://cyberfile.me/shared/tao35avvfc", "#class" : cyberfile.CyberfileSharedExtractor,