[foolfuuka] add 'search' extractors (#1174)

docs/supportedsites.rst

@@ -8,21 +8,21 @@ Site                 URL                                 Capabilities
 35PHOTO              https://35photo.pro/                |35photo-C|
 3dbooru              http://behoimi.org/                 Pools, Popular Images, Posts, Tag Searches
 4chan                https://www.4chan.org/              Boards, Threads
-4plebs               https://archive.4plebs.org/         Boards, Threads
+4plebs               https://archive.4plebs.org/         Boards, Search Results, Threads
 500px                https://500px.com/                  Galleries, individual Images, User Profiles
 8kun                 https://8kun.top/                   Boards, Threads
 8muses               https://comics.8muses.com/          Albums
 Adobe Portfolio      https://www.myportfolio.com/        Galleries
 Adult Empire         https://www.adultempire.com/        Galleries
-arch.b4k.co          https://arch.b4k.co/                Boards, Threads
+arch.b4k.co          https://arch.b4k.co/                Boards, Search Results, Threads
-Archive of Sins      https://archiveofsins.com/          Boards, Threads
+Archive of Sins      https://archiveofsins.com/          Boards, Search Results, Threads
-Archived.Moe         https://archived.moe/               Boards, Threads
+Archived.Moe         https://archived.moe/               Boards, Search Results, Threads
 ArtStation           https://www.artstation.com/         |artstation-C|
 baraag               https://baraag.net/                 Images from Statuses, User Profiles                `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 Behance              https://www.behance.net/            Collections, Galleries, User Profiles
 Blogger              https://www.blogger.com/            Blogs, Posts, Search Results
 Danbooru             https://danbooru.donmai.us/         Pools, Popular Images, Posts, Tag Searches         Supported
-Desuarchive          https://desuarchive.org/            Boards, Threads
+Desuarchive          https://desuarchive.org/            Boards, Search Results, Threads
 DeviantArt           https://www.deviantart.com/         |deviantart-C|                                     `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 Doki Reader          https://kobato.hologfx.com/reader/  Chapters, Manga
 Dynasty Reader       https://dynasty-scans.com/          Chapters, individual Images, Search Results
@@ -32,7 +32,7 @@ Eka's Portal         https://aryion.com/                 Galleries, Posts
 ExHentai             https://exhentai.org/               Favorites, Galleries, Search Results               Supported
 Fallen Angels Scans  https://www.fascans.com/            Chapters, Manga
 Fashion Nova         https://www.fashionnova.com/        Collections, Products
-Fireden              https://boards.fireden.net/         Boards, Threads
+Fireden              https://boards.fireden.net/         Boards, Search Results, Threads
 Flickr               https://www.flickr.com/             |flickr-C|                                         `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 Fur Affinity         https://www.furaffinity.net/        |furaffinity-C|                                    `Cookies <https://github.com/mikf/gallery-dl#cookies>`__
 Fuskator             https://fuskator.com/               Galleries, Search Results
@@ -89,7 +89,7 @@ Niconico Seiga       https://seiga.nicovideo.jp/         individual Images, User
 nijie                https://nijie.info/                 |nijie-C|                                          Required
 Nozomi.la            https://nozomi.la/                  Posts, Search Results, Tag Searches
 NSFWalbum.com        https://nsfwalbum.com/              Albums
-Nyafuu Archive       https://archive.nyafuu.org/         Boards, Threads
+Nyafuu Archive       https://archive.nyafuu.org/         Boards, Search Results, Threads
 Patreon              https://www.patreon.com/            Creators, Posts, User Profiles                     `Cookies <https://github.com/mikf/gallery-dl#cookies>`__
 Pawoo                https://pawoo.net/                  Images from Statuses, User Profiles                `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 Photobucket          https://photobucket.com/            Albums, individual Images
@@ -104,7 +104,7 @@ PowerManga           https://read.powermanga.org/        Chapters, Manga
 Pururin              https://pururin.io/                 Galleries
 Read Comic Online    https://readcomiconline.to/         Comic Issues, Comics
 Realbooru            https://realbooru.com/              Pools, Posts, Tag Searches
-RebeccaBlackTech     https://rbt.asia/                   Boards, Threads
+RebeccaBlackTech     https://rbt.asia/                   Boards, Search Results, Threads
 Reddit               https://www.reddit.com/             |reddit-C|                                         `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 RedGIFs              https://redgifs.com/                individual Images, Search Results, User Profiles
 rule #34             https://rule34.paheal.net/          Posts, Tag Searches
@@ -122,7 +122,7 @@ SlideShare           https://www.slideshare.net/         Presentations
 SmugMug              https://www.smugmug.com/            |smugmug-C|                                        `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 Speaker Deck         https://speakerdeck.com/            Presentations
 SubscribeStar        https://www.subscribestar.com/      Posts, User Profiles                               Supported
-The /b/ Archive      https://thebarchive.com/            Boards, Threads
+The /b/ Archive      https://thebarchive.com/            Boards, Search Results, Threads
 Tsumino              https://www.tsumino.com/            Galleries, Search Results                          Supported
 Tumblr               https://www.tumblr.com/             Likes, Posts, Tag Searches, User Profiles          `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
 Twitter              https://twitter.com/                |twitter-C|                                        Supported

gallery_dl/extractor/foolfuuka.py

@@ -11,12 +11,12 @@
 from .common import Extractor, Message, generate_extractors
 from .. import text
 import itertools
-import operator
 
 
 class FoolfuukaExtractor(Extractor):
     """Base extractor for FoolFuuka based boards/archives"""
     basecategory = "foolfuuka"
+    archive_fmt = "{board[shortname]}_{num}_{timestamp}"
     external = "default"
 
     def __init__(self, match):
@@ -26,16 +26,11 @@ class FoolfuukaExtractor(Extractor):
             self.remote = self._remote_direct
 
     def items(self):
-        op = True
+        yield Message.Directory, self.metadata()
-        yield Message.Version, 1
         for post in self.posts():
-            if op:
-                yield Message.Directory, post
-                op = False
-            if not post["media"]:
-                continue
-
             media = post["media"]
+            if not media:
+                continue
             url = media["media_link"]
 
             if not url and "remote_media_link" in media:
@@ -47,6 +42,9 @@ class FoolfuukaExtractor(Extractor):
                 media["media"].rpartition(".")
             yield Message.Url, url, post
 
+    def metadata(self):
+        """ """
+
     def posts(self):
         """Return an iterable with all relevant posts"""
 
@@ -66,23 +64,27 @@ class FoolfuukaThreadExtractor(FoolfuukaExtractor):
     subcategory = "thread"
     directory_fmt = ("{category}", "{board[shortname]}",
                      "{thread_num}{title:? - //}")
-    archive_fmt = "{board[shortname]}_{num}_{timestamp}"
     pattern_fmt = r"/([^/?#]+)/thread/(\d+)"
 
     def __init__(self, match):
         FoolfuukaExtractor.__init__(self, match)
         self.board, self.thread = match.groups()
+        self.data = None
 
-    def posts(self):
+    def metadata(self):
         url = self.root + "/_/api/chan/thread/"
         params = {"board": self.board, "num": self.thread}
-        data = self.request(url, params=params).json()[self.thread]
+        self.data = self.request(url, params=params).json()[self.thread]
+        return self.data["op"]
 
-        # sort post objects by key
+    def posts(self):
-        posts = sorted(data.get("posts", {}).items())
+        posts = self.data.get("posts")
-        posts = map(operator.itemgetter(1), posts)
+        if posts:
-
+            posts = list(posts.values())
-        return itertools.chain((data["op"],), posts)
+            posts.sort(key=lambda p: p["timestamp"])
+        else:
+            posts = ()
+        return itertools.chain((self.data["op"],), posts)
 
 
 class FoolfuukaBoardExtractor(FoolfuukaExtractor):
@@ -115,6 +117,59 @@ class FoolfuukaBoardExtractor(FoolfuukaExtractor):
                 yield Message.Queue, thread["url"], thread
 
 
+class FoolfuukaSearchExtractor(FoolfuukaExtractor):
+    """Base extractor for search results on FoolFuuka based boards/archives"""
+    subcategory = "search"
+    directory_fmt = ("{category}", "search", "{search}")
+    pattern_fmt = r"/([^/?#]+)/search((?:/[^/?#]+/[^/?#]+)+)"
+    request_interval = 1.0
+
+    def __init__(self, match):
+        FoolfuukaExtractor.__init__(self, match)
+        board, search = match.groups()
+
+        self.params = params = {}
+        args = search.split("/")
+        key = None
+
+        for arg in args:
+            if key:
+                params[key] = text.unescape(arg)
+                key = None
+            else:
+                key = arg
+        if board != "_":
+            params["boards"] = board
+
+    def metadata(self):
+        return {"search": self.params.get("text", "")}
+
+    def posts(self):
+        url = self.root + "/_/api/chan/search/"
+        params = self.params.copy()
+        params["page"] = text.parse_int(params.get("page"), 1)
+        if "filter" not in params:
+            params["filter"] = "text"
+
+        while True:
+            try:
+                data = self.request(url, params=params).json()
+            except ValueError:
+                return
+
+            if isinstance(data, dict):
+                if data.get("error"):
+                    return
+                posts = data["0"]["posts"]
+            elif isinstance(data, list):
+                posts = data[0]["posts"]
+            else:
+                return
+
+            yield from posts
+            params["page"] += 1
+
+
 EXTRACTORS = {
     "4plebs": {
         "name": "_4plebs",
@@ -124,6 +179,7 @@ EXTRACTORS = {
             "url": "07452944164b602502b02b24521f8cee5c484d2a",
         }),
         "test-board": ("https://archive.4plebs.org/tg/",),
+        "test-search": ("https://archive.4plebs.org/_/search/text/test/",),
     },
     "archivedmoe": {
         "root": "https://archived.moe",
@@ -137,6 +193,7 @@ EXTRACTORS = {
             }),
         ),
         "test-board": ("https://archived.moe/gd/",),
+        "test-search": ("https://archived.moe/_/search/text/test/",),
     },
     "archiveofsins": {
         "root": "https://archiveofsins.com",
@@ -146,6 +203,7 @@ EXTRACTORS = {
             "content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4",
         }),
         "test-board": ("https://archiveofsins.com/h/",),
+        "test-search": ("https://archiveofsins.com/_/search/text/test/",),
     },
     "b4k": {
         "root": "https://arch.b4k.co",
@@ -154,6 +212,7 @@ EXTRACTORS = {
             "url": "d309713d2f838797096b3e9cb44fe514a9c9d07a",
         }),
         "test-board": ("https://arch.b4k.co/meta/",),
+        "test-search": ("https://arch.b4k.co/_/search/text/test/",),
     },
     "desuarchive": {
         "root": "https://desuarchive.org",
@@ -161,6 +220,7 @@ EXTRACTORS = {
             "url": "3ae1473f6916ac831efe5cc4d4e7d3298ce79406",
         }),
         "test-board": ("https://desuarchive.org/a/",),
+        "test-search": ("https://desuarchive.org/_/search/text/test/",),
     },
     "fireden": {
         "root": "https://boards.fireden.net",
@@ -168,6 +228,7 @@ EXTRACTORS = {
             "url": "3adfe181ee86a8c23021c705f623b3657a9b0a43",
         }),
         "test-board": ("https://boards.fireden.net/sci/",),
+        "test-search": ("https://boards.fireden.net/_/search/text/test/",),
     },
     "nyafuu": {
         "root": "https://archive.nyafuu.org",
@@ -176,6 +237,7 @@ EXTRACTORS = {
             "url": "bbe6f82944a45e359f5c8daf53f565913dc13e4f",
         }),
         "test-board": ("https://archive.nyafuu.org/c/",),
+        "test-search": ("https://archive.nyafuu.org/_/search/text/test/",),
     },
     "rbt": {
         "root": "https://rbt.asia",
@@ -189,6 +251,7 @@ EXTRACTORS = {
             }),
         ),
         "test-board": ("https://rbt.asia/g/",),
+        "test-search": ("https://rbt.asia/_/search/text/test/",),
     },
     "thebarchive": {
         "root": "https://thebarchive.com",
@@ -197,6 +260,7 @@ EXTRACTORS = {
             "url": "e8b18001307d130d67db31740ce57c8561b5d80c",
         }),
         "test-board": ("https://thebarchive.com/b/",),
+        "test-search": ("https://thebarchive.com/_/search/text/test/",),
     },
     "_ckey": "childclass",
 }
@@ -204,4 +268,5 @@ EXTRACTORS = {
 generate_extractors(EXTRACTORS, globals(), (
     FoolfuukaThreadExtractor,
     FoolfuukaBoardExtractor,
+    FoolfuukaSearchExtractor,
 ))