github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[vichan] add generic extractors for vichan imageboards

Browse Source 
includes 8kun.top, smuglo.li, and wikieat.club
This commit is contained in:
Mike Fährmann
2022-10-21 14:40:45 +02:00
parent 04d3ebdfb4
commit a7d23f1484
7 changed files with 187 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
docs/supportedsites.md
View File

@@ -55,12 +55,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Boards, Threads</td>
<td></td>
</tr>
<tr>
<td>8kun</td>
<td>https://8kun.top/</td>
<td>Boards, Threads</td>
<td></td>
</tr>
<tr>
<td>8muses</td>
<td>https://comics.8muses.com/</td>
@@ -763,12 +757,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Presentations</td>
<td></td>
</tr>
<tr>
<td>Smugloli</td>
<td>https://smuglo.li/</td>
<td>Boards, Threads</td>
<td></td>
</tr>
<tr>
<td>SmugMug</td>
<td>https://www.smugmug.com/</td>
@@ -907,12 +895,6 @@ Consider all sites to be NSFW unless otherwise known.
<td>Artists, Artist Listings, Artworks, individual Images</td>
<td></td>
</tr>
<tr>
<td>Wikieat</td>
<td>https://wikieat.club/</td>
<td>Boards, Threads</td>
<td></td>
</tr>
<tr>
<td>xHamster</td>
<td>https://xhamster.com/</td>
@@ -1112,6 +1094,28 @@ Consider all sites to be NSFW unless otherwise known.
<td></td>
</tr>
<tr>
<td colspan="4"><strong>vichan Imageboards</strong></td>
</tr>
<tr>
<td>8kun</td>
<td>https://8kun.top/</td>
<td>Boards, Threads</td>
<td></td>
</tr>
<tr>
<td>Wikieat</td>
<td>https://wikieat.club/</td>
<td>Boards, Threads</td>
<td></td>
</tr>
<tr>
<td>Smugloli</td>
<td>https://smuglo.li/</td>
<td>Boards, Threads</td>
<td></td>
</tr>
<tr>
<td colspan="4"><strong>Moebooru and MyImouto</strong></td>
</tr>

100
gallery_dl/extractor/8kun.py
View File

@@ -1,100 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2020-2022 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://8kun.top/"""
from .common import Extractor, Message
from .. import text
class _8kunThreadExtractor(Extractor):
"""Extractor for 8kun threads"""
category = "8kun"
subcategory = "thread"
directory_fmt = ("{category}", "{board}", "{thread} {title}")
filename_fmt = "{time}{num:?-//} {filename}.{extension}"
archive_fmt = "{board}_{thread}_{tim}"
pattern = r"(?:https?://)?8kun\.top/([^/]+)/res/(\d+)"
test = (
("https://8kun.top/test/res/65248.html", {
"pattern": r"https://media\.8kun\.top/file_store/\w{64}\.\w+",
"count": ">= 8",
}),
# old-style file URLs (#1101)
# ("https://8kun.top/d/res/13258.html", {
# "pattern": r"https://media\.8kun\.top/d/src/\d+(-\d)?\.\w+",
# "range": "1-20",
# }),
)
def __init__(self, match):
Extractor.__init__(self, match)
self.board, self.thread = match.groups()
def items(self):
url = "https://8kun.top/{}/res/{}.json".format(self.board, self.thread)
posts = self.request(url).json()["posts"]
title = posts[0].get("sub") or text.remove_html(posts[0]["com"])
process = self._process
data = {
"board" : self.board,
"thread": self.thread,
"title" : text.unescape(title)[:50],
"num" : 0,
}
yield Message.Directory, data
for post in posts:
if "filename" in post:
yield process(post, data)
if "extra_files" in post:
for post["num"], filedata in enumerate(
post["extra_files"], 1):
yield process(post, filedata)
@staticmethod
def _process(post, data):
post.update(data)
post["extension"] = post["ext"][1:]
tim = post["tim"]
url = ("https://media.8kun.top/" +
("file_store/" if len(tim) > 16 else post["board"] + "/src/") +
tim + post["ext"])
return Message.Url, url, post
class _8kunBoardExtractor(Extractor):
"""Extractor for 8kun boards"""
category = "8kun"
subcategory = "board"
pattern = r"(?:https?://)?8kun\.top/([^/?#]+)/(?:index|\d+)\.html"
test = (
("https://8kun.top/v/index.html", {
"pattern": _8kunThreadExtractor.pattern,
"count": ">= 100",
}),
("https://8kun.top/v/2.html"),
("https://8kun.top/v/index.html?PageSpeed=noscript"),
)
def __init__(self, match):
Extractor.__init__(self, match)
self.board = match.group(1)
def items(self):
url = "https://8kun.top/{}/threads.json".format(self.board)
threads = self.request(url).json()
for page in threads:
for thread in page["threads"]:
url = "https://8kun.top/{}/res/{}.html".format(
self.board, thread["no"])
thread["page"] = page["page"]
thread["_extractor"] = _8kunThreadExtractor
yield Message.Queue, url, thread

4
gallery_dl/extractor/__init__.py
View File

@@ -17,7 +17,6 @@ modules = [
"4chan",
"500px",
"8chan",
"8kun",
"8muses",
"adultempire",
"architizer",
@@ -142,6 +141,7 @@ modules = [
"twitter",
"unsplash",
"vanillarock",
"vichan",
"vk",
"vsco",
"wallhaven",
@@ -151,7 +151,6 @@ modules = [
"webtoons",
"weibo",
"wikiart",
"wikieat",
"xhamster",
"xvideos",
"zerochan",
@@ -161,7 +160,6 @@ modules = [
"foolslide",
"mastodon",
"shopify",
"smugloli",
"lolisafe",
"imagehosts",
"directlink",

98
gallery_dl/extractor/smugloli.py
View File

@@ -1,98 +0,0 @@
# -*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://smuglo.li/"""
from .common import Extractor, Message
from .. import text
class SmugloliThreadExtractor(Extractor):
"""Extractor for smugloli threads"""
category = "smugloli"
subcategory = "thread"
directory_fmt = ("{category}", "{board}", "{thread} {title}")
filename_fmt = "{time}{num:?-//} {filename}.{extension}"
archive_fmt = "{board}_{thread}_{tim}"
pattern = r"(?:https?://)?(smuglo(?:\.li|li\.net))/([^/]+)/res/(\d+)\.html"
test = (
("https://smuglo.li/a/res/1154380.html", {
"pattern": r"https://smug.+/a/src/\d+(-\d)?\.\w+",
"count": ">= 18",
"keyword": {
"board": "a",
"thread": "1154380",
"title": "Mob Psycho 100 Season 3",
},
}),
("https://smugloli.net/a/res/1145409.html"),
)
def __init__(self, match):
Extractor.__init__(self, match)
self.domain, self.board, self.thread = match.groups()
def items(self):
url = "https://{}/{}/res/{}.json".format(
self.domain, self.board, self.thread)
posts = self.request(url).json()["posts"]
title = posts[0].get("sub") or text.remove_html(posts[0]["com"])
process = self._process
data = {
"board" : self.board,
"thread": self.thread,
"title" : text.unescape(title)[:50],
"num" : 0,
}
yield Message.Directory, data
for post in posts:
if "filename" in post:
yield process(post, data)
if "extra_files" in post:
for post["num"], filedata in enumerate(
post["extra_files"], 1):
yield process(post, filedata)
def _process(self, post, data):
post.update(data)
post["extension"] = post["ext"][1:]
post["url"] = "https://{}/{}/src/{}{}".format(
self.domain, post["board"], post["tim"], post["ext"])
return Message.Url, post["url"], post
class SmugloliBoardExtractor(Extractor):
"""Extractor for smugloli boards"""
category = "smugloli"
subcategory = "board"
pattern = (r"(?:https?://)?(smuglo(?:\.li|li\.net))"
r"/([^/?#]+)(?:/(?:index|catalog|\d{1,2})\.html)?/?$")
test = (
("https://smuglo.li/a/", {
"pattern": SmugloliThreadExtractor.pattern,
"count": ">= 100",
}),
("https://smuglo.li/a/1.html"),
("https://smuglo.li/cute/catalog.html"),
)
def __init__(self, match):
Extractor.__init__(self, match)
self.domain, self.board = match.groups()
def items(self):
url = "https://{}/{}/threads.json".format(self.domain, self.board)
threads = self.request(url).json()
for page in threads:
for thread in page["threads"]:
url = "{}/{}/res/{}.html".format(
self.domain, self.board, thread["no"])
thread["page"] = page["page"]
thread["_extractor"] = SmugloliThreadExtractor
yield Message.Queue, url, thread

163
gallery_dl/extractor/vichan.py Normal file
View File

@@ -0,0 +1,163 @@
# -*- coding: utf-8 -*-
# Copyright 2022 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for vichan imageboards"""
from .common import BaseExtractor, Message
from .. import text
class VichanExtractor(BaseExtractor):
"""Base class for vichan extractors"""
basecategory = "vichan"
BASE_PATTERN = VichanExtractor.update({
"8kun": {
"root": "https://8kun.top",
"pattern": r"8kun\.top",
},
"wikieat": {
"root": "https://wikieat.club",
"pattern": r"wikieat\.club",
},
"smugloli": {
"root": None,
"pattern": r"smuglo(?:\.li|li\.net)",
},
})
class VichanThreadExtractor(VichanExtractor):
"""Extractor for vichan threads"""
subcategory = "thread"
directory_fmt = ("{category}", "{board}", "{thread} {title}")
filename_fmt = "{time}{num:?-//} {filename}.{extension}"
archive_fmt = "{board}_{thread}_{tim}"
pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)"
test = (
("https://8kun.top/test/res/65248.html", {
"pattern": r"https://media\.128ducks\.com/file_store/\w{64}\.\w+",
"count": ">= 8",
}),
# old-style file URLs (#1101)
# ("https://8kun.top/d/res/13258.html", {
# "pattern": r"https://media\.128ducks\.com/d/src/\d+(-\d)?\.\w+",
# "range": "1-20",
# }),
("https://wikieat.club/cel/res/25321.html", {
"pattern": r"https://wikieat\.club/cel/src/\d+(-\d)?\.\w+",
"count": ">= 200",
}),
("https://smuglo.li/a/res/1154380.html", {
"pattern": r"https://smug.+/a/src/\d+(-\d)?\.\w+",
"count": ">= 18",
"keyword": {
"board": "a",
"thread": "1154380",
"title": "Mob Psycho 100 Season 3",
},
}),
("https://smugloli.net/a/res/1145409.html"),
)
def __init__(self, match):
VichanExtractor.__init__(self, match)
index = match.lastindex
self.board = match.group(index-1)
self.thread = match.group(index)
def items(self):
url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread)
posts = self.request(url).json()["posts"]
title = posts[0].get("sub") or text.remove_html(posts[0]["com"])
process = (self._process_8kun if self.category == "8kun" else
self._process)
data = {
"board" : self.board,
"thread": self.thread,
"title" : text.unescape(title)[:50],
"num" : 0,
}
yield Message.Directory, data
for post in posts:
if "filename" in post:
yield process(post, data)
if "extra_files" in post:
for post["num"], filedata in enumerate(
post["extra_files"], 1):
yield process(post, filedata)
def _process(self, post, data):
post.update(data)
post["extension"] = post["ext"][1:]
post["url"] = "{}/{}/src/{}{}".format(
self.root, post["board"], post["tim"], post["ext"])
return Message.Url, post["url"], post
@staticmethod
def _process_8kun(post, data):
post.update(data)
post["extension"] = post["ext"][1:]
tim = post["tim"]
if len(tim) > 16:
post["url"] = "https://media.128ducks.com/file_store/{}{}".format(
tim, post["ext"])
else:
post["url"] = "https://media.128ducks.com/{}/src/{}{}".format(
post["board"], tim, post["ext"])
return Message.Url, post["url"], post
class VichanBoardExtractor(VichanExtractor):
"""Extractor for vichan boards"""
subcategory = "board"
pattern = BASE_PATTERN + r"/([^/?#]+)/(?:index|catalog|\d+|$)"
test = (
("https://8kun.top/v/index.html", {
"pattern": VichanThreadExtractor.pattern,
"count": ">= 100",
}),
("https://8kun.top/v/2.html"),
("https://8kun.top/v/index.html?PageSpeed=noscript"),
("https://wikieat.club/cel/index.html", {
"pattern": VichanThreadExtractor.pattern,
"count": ">= 100",
}),
("https://wikieat.club/cel/catalog.html"),
("https://wikieat.club/cel/2.html"),
("https://smuglo.li/a/", {
"pattern": VichanThreadExtractor.pattern,
"count": ">= 100",
}),
("https://smuglo.li/a/1.html"),
("https://smugloli.net/cute/catalog.html"),
)
def __init__(self, match):
VichanExtractor.__init__(self, match)
self.board = match.group(match.lastindex)
def items(self):
url = "{}/{}/threads.json".format(self.root, self.board)
threads = self.request(url).json()
for page in threads:
for thread in page["threads"]:
url = "{}/{}/res/{}.html".format(
self.root, self.board, thread["no"])
thread["page"] = page["page"]
thread["_extractor"] = VichanThreadExtractor
yield Message.Queue, url, thread

95
gallery_dl/extractor/wikieat.py
View File

@@ -1,95 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2021 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extractors for https://wikieat.club/"""
from .common import Extractor, Message
from .. import text
class WikieatThreadExtractor(Extractor):
"""Extractor for Wikieat threads"""
category = "wikieat"
subcategory = "thread"
directory_fmt = ("{category}", "{board}", "{thread} {title}")
filename_fmt = "{time}{num:?-//} {filename}.{extension}"
archive_fmt = "{board}_{thread}_{tim}"
pattern = r"(?:https?://)?wikieat\.club/([^/]+)/res/(\d+)"
test = ("https://wikieat.club/cel/res/25321.html", {
"pattern": r"https://wikieat\.club/cel/src/\d+(-\d)?\.\w+",
"count": ">= 200",
})
def __init__(self, match):
Extractor.__init__(self, match)
self.board, self.thread = match.groups()
def items(self):
url = "https://wikieat.club/{}/res/{}.json".format(
self.board, self.thread)
posts = self.request(url).json()["posts"]
title = posts[0].get("sub") or text.remove_html(posts[0]["com"])
process = self._process
data = {
"board" : self.board,
"thread": self.thread,
"title" : text.unescape(title)[:50],
"num" : 0,
}
yield Message.Directory, data
for post in posts:
if "filename" in post:
yield process(post, data)
if "extra_files" in post:
for post["num"], filedata in enumerate(
post["extra_files"], 1):
yield process(post, filedata)
@staticmethod
def _process(post, data):
post.update(data)
post["extension"] = post["ext"][1:]
tim = post["tim"]
url = ("https://wikieat.club/" +
post["board"] + "/src/" +
tim + post["ext"])
return Message.Url, url, post
class WikieatBoardExtractor(Extractor):
"""Extractor for Wikieat boards"""
category = "wikieat"
subcategory = "board"
pattern = (r"(?:https?://)?wikieat\.club"
r"/([^/?#]+)/(?:index|catalog|\d+)\.html")
test = (
("https://wikieat.club/cel/index.html", {
"pattern": WikieatThreadExtractor.pattern,
"count": ">= 100",
}),
("https://wikieat.club/cel/catalog.html"),
("https://wikieat.club/cel/2.html")
)
def __init__(self, match):
Extractor.__init__(self, match)
self.board = match.group(1)
def items(self):
url = "https://wikieat.club/{}/threads.json".format(self.board)
threads = self.request(url).json()
for page in threads:
for thread in page["threads"]:
url = "https://wikieat.club/{}/res/{}.html".format(
self.board, thread["no"])
thread["page"] = page["page"]
thread["_extractor"] = WikieatThreadExtractor
yield Message.Queue, url, thread

1
scripts/supportedsites.py
View File

@@ -254,6 +254,7 @@ BASE_MAP = {
"gelbooru_v02": "Gelbooru Beta 0.2",
"lolisafe" : "lolisafe and chibisafe",
"moebooru" : "Moebooru and MyImouto",
"vichan" : "vichan Imageboards",
}
_OAUTH = '<a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a>'