github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[foolfuuka] add 'search' extractors (#1174)

Browse Source
This commit is contained in:
Mike Fährmann
2021-01-01 23:46:21 +01:00
parent fb64183d53
commit 56b460dcea
2 changed files with 91 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
docs/supportedsites.rst
View File

@@ -8,21 +8,21 @@ Site URL Capabilities
35PHOTO https://35photo.pro/ |35photo-C|
3dbooru http://behoimi.org/ Pools, Popular Images, Posts, Tag Searches
4chan https://www.4chan.org/ Boards, Threads
4plebs https://archive.4plebs.org/ Boards, Threads
4plebs https://archive.4plebs.org/ Boards, Search Results, Threads
500px https://500px.com/ Galleries, individual Images, User Profiles
8kun https://8kun.top/ Boards, Threads
8muses https://comics.8muses.com/ Albums
Adobe Portfolio https://www.myportfolio.com/ Galleries
Adult Empire https://www.adultempire.com/ Galleries
arch.b4k.co https://arch.b4k.co/ Boards, Threads
Archive of Sins https://archiveofsins.com/ Boards, Threads
Archived.Moe https://archived.moe/ Boards, Threads
arch.b4k.co https://arch.b4k.co/ Boards, Search Results, Threads
Archive of Sins https://archiveofsins.com/ Boards, Search Results, Threads
Archived.Moe https://archived.moe/ Boards, Search Results, Threads
ArtStation https://www.artstation.com/ |artstation-C|
baraag https://baraag.net/ Images from Statuses, User Profiles `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
Behance https://www.behance.net/ Collections, Galleries, User Profiles
Blogger https://www.blogger.com/ Blogs, Posts, Search Results
Danbooru https://danbooru.donmai.us/ Pools, Popular Images, Posts, Tag Searches Supported
Desuarchive https://desuarchive.org/ Boards, Threads
Desuarchive https://desuarchive.org/ Boards, Search Results, Threads
DeviantArt https://www.deviantart.com/ |deviantart-C| `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
Doki Reader https://kobato.hologfx.com/reader/ Chapters, Manga
Dynasty Reader https://dynasty-scans.com/ Chapters, individual Images, Search Results
@@ -32,7 +32,7 @@ Eka's Portal https://aryion.com/ Galleries, Posts
ExHentai https://exhentai.org/ Favorites, Galleries, Search Results Supported
Fallen Angels Scans https://www.fascans.com/ Chapters, Manga
Fashion Nova https://www.fashionnova.com/ Collections, Products
Fireden https://boards.fireden.net/ Boards, Threads
Fireden https://boards.fireden.net/ Boards, Search Results, Threads
Flickr https://www.flickr.com/ |flickr-C| `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
Fur Affinity https://www.furaffinity.net/ |furaffinity-C| `Cookies <https://github.com/mikf/gallery-dl#cookies>`__
Fuskator https://fuskator.com/ Galleries, Search Results
@@ -89,7 +89,7 @@ Niconico Seiga https://seiga.nicovideo.jp/ individual Images, User
nijie https://nijie.info/ |nijie-C| Required
Nozomi.la https://nozomi.la/ Posts, Search Results, Tag Searches
NSFWalbum.com https://nsfwalbum.com/ Albums
Nyafuu Archive https://archive.nyafuu.org/ Boards, Threads
Nyafuu Archive https://archive.nyafuu.org/ Boards, Search Results, Threads
Patreon https://www.patreon.com/ Creators, Posts, User Profiles `Cookies <https://github.com/mikf/gallery-dl#cookies>`__
Pawoo https://pawoo.net/ Images from Statuses, User Profiles `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
Photobucket https://photobucket.com/ Albums, individual Images
@@ -104,7 +104,7 @@ PowerManga https://read.powermanga.org/ Chapters, Manga
Pururin https://pururin.io/ Galleries
Read Comic Online https://readcomiconline.to/ Comic Issues, Comics
Realbooru https://realbooru.com/ Pools, Posts, Tag Searches
RebeccaBlackTech https://rbt.asia/ Boards, Threads
RebeccaBlackTech https://rbt.asia/ Boards, Search Results, Threads
Reddit https://www.reddit.com/ |reddit-C| `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
RedGIFs https://redgifs.com/ individual Images, Search Results, User Profiles
rule #34 https://rule34.paheal.net/ Posts, Tag Searches
@@ -122,7 +122,7 @@ SlideShare https://www.slideshare.net/ Presentations
SmugMug https://www.smugmug.com/ |smugmug-C| `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
Speaker Deck https://speakerdeck.com/ Presentations
SubscribeStar https://www.subscribestar.com/ Posts, User Profiles Supported
The /b/ Archive https://thebarchive.com/ Boards, Threads
The /b/ Archive https://thebarchive.com/ Boards, Search Results, Threads
Tsumino https://www.tsumino.com/ Galleries, Search Results Supported
Tumblr https://www.tumblr.com/ Likes, Posts, Tag Searches, User Profiles `OAuth <https://github.com/mikf/gallery-dl#oauth>`__
Twitter https://twitter.com/ |twitter-C| Supported

99
gallery_dl/extractor/foolfuuka.py
View File

@@ -11,12 +11,12 @@
from .common import Extractor, Message, generate_extractors
from .. import text
import itertools
import operator
class FoolfuukaExtractor(Extractor):
"""Base extractor for FoolFuuka based boards/archives"""
basecategory = "foolfuuka"
archive_fmt = "{board[shortname]}_{num}_{timestamp}"
external = "default"
def __init__(self, match):
@@ -26,16 +26,11 @@ class FoolfuukaExtractor(Extractor):
self.remote = self._remote_direct
def items(self):
op = True
yield Message.Version, 1
yield Message.Directory, self.metadata()
for post in self.posts():
if op:
yield Message.Directory, post
op = False
if not post["media"]:
continue
media = post["media"]
if not media:
continue
url = media["media_link"]
if not url and "remote_media_link" in media:
@@ -47,6 +42,9 @@ class FoolfuukaExtractor(Extractor):
media["media"].rpartition(".")
yield Message.Url, url, post
def metadata(self):
""" """
def posts(self):
"""Return an iterable with all relevant posts"""
@@ -66,23 +64,27 @@ class FoolfuukaThreadExtractor(FoolfuukaExtractor):
subcategory = "thread"
directory_fmt = ("{category}", "{board[shortname]}",
"{thread_num}{title:? - //}")
archive_fmt = "{board[shortname]}_{num}_{timestamp}"
pattern_fmt = r"/([^/?#]+)/thread/(\d+)"
def __init__(self, match):
FoolfuukaExtractor.__init__(self, match)
self.board, self.thread = match.groups()
self.data = None
def posts(self):
def metadata(self):
url = self.root + "/_/api/chan/thread/"
params = {"board": self.board, "num": self.thread}
data = self.request(url, params=params).json()[self.thread]
self.data = self.request(url, params=params).json()[self.thread]
return self.data["op"]
# sort post objects by key
posts = sorted(data.get("posts", {}).items())
posts = map(operator.itemgetter(1), posts)
return itertools.chain((data["op"],), posts)
def posts(self):
posts = self.data.get("posts")
if posts:
posts = list(posts.values())
posts.sort(key=lambda p: p["timestamp"])
else:
posts = ()
return itertools.chain((self.data["op"],), posts)
class FoolfuukaBoardExtractor(FoolfuukaExtractor):
@@ -115,6 +117,59 @@ class FoolfuukaBoardExtractor(FoolfuukaExtractor):
yield Message.Queue, thread["url"], thread
class FoolfuukaSearchExtractor(FoolfuukaExtractor):
"""Base extractor for search results on FoolFuuka based boards/archives"""
subcategory = "search"
directory_fmt = ("{category}", "search", "{search}")
pattern_fmt = r"/([^/?#]+)/search((?:/[^/?#]+/[^/?#]+)+)"
request_interval = 1.0
def __init__(self, match):
FoolfuukaExtractor.__init__(self, match)
board, search = match.groups()
self.params = params = {}
args = search.split("/")
key = None
for arg in args:
if key:
params[key] = text.unescape(arg)
key = None
else:
key = arg
if board != "_":
params["boards"] = board
def metadata(self):
return {"search": self.params.get("text", "")}
def posts(self):
url = self.root + "/_/api/chan/search/"
params = self.params.copy()
params["page"] = text.parse_int(params.get("page"), 1)
if "filter" not in params:
params["filter"] = "text"
while True:
try:
data = self.request(url, params=params).json()
except ValueError:
return
if isinstance(data, dict):
if data.get("error"):
return
posts = data["0"]["posts"]
elif isinstance(data, list):
posts = data[0]["posts"]
else:
return
yield from posts
params["page"] += 1
EXTRACTORS = {
"4plebs": {
"name": "_4plebs",
@@ -124,6 +179,7 @@ EXTRACTORS = {
"url": "07452944164b602502b02b24521f8cee5c484d2a",
}),
"test-board": ("https://archive.4plebs.org/tg/",),
"test-search": ("https://archive.4plebs.org/_/search/text/test/",),
},
"archivedmoe": {
"root": "https://archived.moe",
@@ -137,6 +193,7 @@ EXTRACTORS = {
}),
),
"test-board": ("https://archived.moe/gd/",),
"test-search": ("https://archived.moe/_/search/text/test/",),
},
"archiveofsins": {
"root": "https://archiveofsins.com",
@@ -146,6 +203,7 @@ EXTRACTORS = {
"content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4",
}),
"test-board": ("https://archiveofsins.com/h/",),
"test-search": ("https://archiveofsins.com/_/search/text/test/",),
},
"b4k": {
"root": "https://arch.b4k.co",
@@ -154,6 +212,7 @@ EXTRACTORS = {
"url": "d309713d2f838797096b3e9cb44fe514a9c9d07a",
}),
"test-board": ("https://arch.b4k.co/meta/",),
"test-search": ("https://arch.b4k.co/_/search/text/test/",),
},
"desuarchive": {
"root": "https://desuarchive.org",
@@ -161,6 +220,7 @@ EXTRACTORS = {
"url": "3ae1473f6916ac831efe5cc4d4e7d3298ce79406",
}),
"test-board": ("https://desuarchive.org/a/",),
"test-search": ("https://desuarchive.org/_/search/text/test/",),
},
"fireden": {
"root": "https://boards.fireden.net",
@@ -168,6 +228,7 @@ EXTRACTORS = {
"url": "3adfe181ee86a8c23021c705f623b3657a9b0a43",
}),
"test-board": ("https://boards.fireden.net/sci/",),
"test-search": ("https://boards.fireden.net/_/search/text/test/",),
},
"nyafuu": {
"root": "https://archive.nyafuu.org",
@@ -176,6 +237,7 @@ EXTRACTORS = {
"url": "bbe6f82944a45e359f5c8daf53f565913dc13e4f",
}),
"test-board": ("https://archive.nyafuu.org/c/",),
"test-search": ("https://archive.nyafuu.org/_/search/text/test/",),
},
"rbt": {
"root": "https://rbt.asia",
@@ -189,6 +251,7 @@ EXTRACTORS = {
}),
),
"test-board": ("https://rbt.asia/g/",),
"test-search": ("https://rbt.asia/_/search/text/test/",),
},
"thebarchive": {
"root": "https://thebarchive.com",
@@ -197,6 +260,7 @@ EXTRACTORS = {
"url": "e8b18001307d130d67db31740ce57c8561b5d80c",
}),
"test-board": ("https://thebarchive.com/b/",),
"test-search": ("https://thebarchive.com/_/search/text/test/",),
},
"_ckey": "childclass",
}
@@ -204,4 +268,5 @@ EXTRACTORS = {
generate_extractors(EXTRACTORS, globals(), (
FoolfuukaThreadExtractor,
FoolfuukaBoardExtractor,
FoolfuukaSearchExtractor,
))