From 8d676151b7c314dada1d98d4404a2b32d1b18203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Sun, 17 Oct 2021 04:14:58 +0200 Subject: [PATCH] [patreon] implement 'files' option (#1935) --- docs/configuration.rst | 13 +++++++++++++ gallery_dl/extractor/patreon.py | 29 ++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index cb6d9dc9..ec1db053 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1577,6 +1577,19 @@ Description port than the default. +extractor.patreon.files +----------------------- +Type + ``list`` of ``strings`` +Default + ``["images", "attachments", "postfile", "content"]`` +Description + Determines the type and order of files to be downloaded. + + Available types are + ``postfile``, ``images``, ``attachments``, and ``content``. + + extractor.photobucket.subalbums ------------------------------- Type diff --git a/gallery_dl/extractor/patreon.py b/gallery_dl/extractor/patreon.py index 50a86db7..c7df0890 100644 --- a/gallery_dl/extractor/patreon.py +++ b/gallery_dl/extractor/patreon.py @@ -32,22 +32,19 @@ class PatreonExtractor(Extractor): if "session_id" not in self.session.cookies: self.log.warning("no 'session_id' cookie set") PatreonExtractor._warning = False + generators = self._build_file_generators(self.config("files")) for post in self.posts(): if not post.get("current_user_can_view", True): self.log.warning("Not allowed to view post %s", post["id"]) continue + yield Message.Directory, post + post["num"] = 0 hashes = set() - - yield Message.Directory, post - for kind, url, name in itertools.chain( - self._images(post), - self._attachments(post), - self._postfile(post), - self._content(post), - ): + for kind, url, name in itertools.chain.from_iterable( + g(post) for g in generators): fhash = self._filehash(url) if fhash not in hashes or not fhash: hashes.add(fhash) @@ -154,7 +151,7 @@ class PatreonExtractor(Extractor): included[file["type"]][file["id"]] for file in files["data"] ] - return [] + return () @memcache(keyarg=1) def _user(self, url): @@ -211,6 +208,20 @@ class PatreonExtractor(Extractor): "&json-api-version=1.0" ) + def _build_file_generators(self, filetypes): + if filetypes is None: + return (self._images, self._attachments, + self._postfile, self._content) + genmap = { + "images" : self._images, + "attachments": self._attachments, + "postfile" : self._postfile, + "content" : self._content, + } + if isinstance(filetypes, str): + filetypes = filetypes.split(",") + return [genmap[ft] for ft in filetypes] + class PatreonCreatorExtractor(PatreonExtractor): """Extractor for a creator's works"""