diff --git a/docs/configuration.rst b/docs/configuration.rst index fe47be63..c288cb27 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1128,6 +1128,16 @@ Description * ``"html"``: Raw HTML content +extractor.furaffinity.external +------------------------------ +Type + ``bool`` +Default + ``false`` +Description + Follow external URLs linked in descriptions. + + extractor.furaffinity.include ----------------------------- Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 26832068..61db71f0 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -98,6 +98,7 @@ "furaffinity": { "descriptions": "text", + "external": false, "include": "gallery" }, "gfycat": diff --git a/gallery_dl/extractor/furaffinity.py b/gallery_dl/extractor/furaffinity.py index 86e1678b..29c76ff2 100644 --- a/gallery_dl/extractor/furaffinity.py +++ b/gallery_dl/extractor/furaffinity.py @@ -29,9 +29,10 @@ class FuraffinityExtractor(Extractor): self.offset = 0 if self.config("descriptions") == "html": - self._process_description = lambda x: x.strip() + self._process_description = str.strip def items(self): + external = self.config("external", False) metadata = self.metadata() for post_id in util.advance(self.posts(), self.offset): post = self._parse_post(post_id) @@ -41,6 +42,11 @@ class FuraffinityExtractor(Extractor): yield Message.Directory, post yield Message.Url, post["url"], post + if external: + for url in text.extract_iter( + post["_description"], 'href="http', '"'): + yield Message.Queue, "http" + url, post + def posts(self): return self._pagination() @@ -80,8 +86,7 @@ class FuraffinityExtractor(Extractor): data["tags"] = text.split_html(tags) data["title"] = text.unescape(extr("
", "
")) data["artist"] = extr("", "<") - data["description"] = self._process_description(extr( - 'class="section-body">', '')) + data["_description"] = extr('class="section-body">', '') data["views"] = pi(rh(extr('class="views">', ''))) data["favorites"] = pi(rh(extr('class="favorites">', ''))) data["comments"] = pi(rh(extr('class="comments">', ''))) @@ -108,12 +113,12 @@ class FuraffinityExtractor(Extractor): data["tags"] = text.split_html(extr( 'id="keywords">', ''))[::2] data["rating"] = extr('