github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[8chan] support '/last/' thread URLs (#6318)

Browse Source
This commit is contained in:
Mike Fährmann
2024-10-13 20:42:41 +02:00
parent c7f0d8945b
commit 93265db9b3
2 changed files with 36 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
gallery_dl/extractor/8chan.py
View File

@@ -64,16 +64,14 @@ class _8chanThreadExtractor(_8chanExtractor):
"{threadId} {subject[:50]}") "{threadId} {subject[:50]}")
filename_fmt = "{postId}{num:?-//} {filename[:200]}.{extension}" filename_fmt = "{postId}{num:?-//} {filename[:200]}.{extension}"
archive_fmt = "{boardUri}_{postId}_{num}" archive_fmt = "{boardUri}_{postId}_{num}"
pattern = BASE_PATTERN + r"/([^/?#]+)/res/(\d+)" pattern = BASE_PATTERN + r"/([^/?#]+)/(?:res|last)/(\d+)"
example = "https://8chan.moe/a/res/12345.html" example = "https://8chan.moe/a/res/12345.html"
def __init__(self, match):
_8chanExtractor.__init__(self, match)
_, self.board, self.thread = match.groups()
def items(self): def items(self):
_, board, thread = self.groups
# fetch thread data # fetch thread data
url = "{}/{}/res/{}.".format(self.root, self.board, self.thread) url = "{}/{}/res/{}.".format(self.root, board, thread)
self.session.headers["Referer"] = url + "html" self.session.headers["Referer"] = url + "html"
thread = self.request(url + "json").json() thread = self.request(url + "json").json()
thread["postId"] = thread["threadId"] thread["postId"] = thread["threadId"]
@@ -106,25 +104,22 @@ class _8chanBoardExtractor(_8chanExtractor):
pattern = BASE_PATTERN + r"/([^/?#]+)/(?:(\d+)\.html)?$" pattern = BASE_PATTERN + r"/([^/?#]+)/(?:(\d+)\.html)?$"
example = "https://8chan.moe/a/" example = "https://8chan.moe/a/"
def __init__(self, match):
_8chanExtractor.__init__(self, match)
_, self.board, self.page = match.groups()
def items(self): def items(self):
page = text.parse_int(self.page, 1) _, board, pnum = self.groups
url = "{}/{}/{}.json".format(self.root, self.board, page) pnum = text.parse_int(pnum, 1)
board = self.request(url).json() url = "{}/{}/{}.json".format(self.root, board, pnum)
threads = board["threads"] data = self.request(url).json()
threads = data["threads"]
while True: while True:
for thread in threads: for thread in threads:
thread["_extractor"] = _8chanThreadExtractor thread["_extractor"] = _8chanThreadExtractor
url = "{}/{}/res/{}.html".format( url = "{}/{}/res/{}.html".format(
self.root, self.board, thread["threadId"]) self.root, board, thread["threadId"])
yield Message.Queue, url, thread yield Message.Queue, url, thread
page += 1 pnum += 1
if page > board["pageCount"]: if pnum > data["pageCount"]:
return return
url = "{}/{}/{}.json".format(self.root, self.board, page) url = "{}/{}/{}.json".format(self.root, board, pnum)
threads = self.request(url).json()["threads"] threads = self.request(url).json()["threads"]

48
test/results/8chan.py
View File

@@ -10,11 +10,10 @@ _8chan = getattr(gallery_dl.extractor, "8chan")
__tests__ = ( __tests__ = (
{ {
"#url" : "https://8chan.moe/vhs/res/4.html", "#url" : "https://8chan.moe/vhs/res/4.html",
"#category": ("", "8chan", "thread"), "#class": _8chan._8chanThreadExtractor,
"#class" : _8chan._8chanThreadExtractor, "#pattern": r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$",
"#pattern" : r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$", "#count" : 14,
"#count" : 14,
"archived" : False, "archived" : False,
"autoSage" : False, "autoSage" : False,
@@ -51,41 +50,40 @@ __tests__ = (
}, },
{ {
"#url" : "https://8chan.se/vhs/res/4.html", "#url" : "https://8chan.moe/vhs/last/4.html",
"#category": ("", "8chan", "thread"), "#class": _8chan._8chanThreadExtractor,
"#class" : _8chan._8chanThreadExtractor,
}, },
{ {
"#url" : "https://8chan.cc/vhs/res/4.html", "#url" : "https://8chan.se/vhs/res/4.html",
"#category": ("", "8chan", "thread"), "#class": _8chan._8chanThreadExtractor,
"#class" : _8chan._8chanThreadExtractor,
}, },
{ {
"#url" : "https://8chan.moe/vhs/", "#url" : "https://8chan.cc/vhs/res/4.html",
"#category": ("", "8chan", "board"), "#class": _8chan._8chanThreadExtractor,
"#class" : _8chan._8chanBoardExtractor,
}, },
{ {
"#url" : "https://8chan.moe/vhs/2.html", "#url" : "https://8chan.moe/vhs/",
"#category": ("", "8chan", "board"), "#class": _8chan._8chanBoardExtractor,
"#class" : _8chan._8chanBoardExtractor,
"#pattern" : _8chan._8chanThreadExtractor.pattern,
"#count" : range(24, 32),
}, },
{ {
"#url" : "https://8chan.se/vhs/", "#url" : "https://8chan.moe/vhs/2.html",
"#category": ("", "8chan", "board"), "#class": _8chan._8chanBoardExtractor,
"#class" : _8chan._8chanBoardExtractor, "#pattern": _8chan._8chanThreadExtractor.pattern,
"#count" : range(24, 32),
}, },
{ {
"#url" : "https://8chan.cc/vhs/", "#url" : "https://8chan.se/vhs/",
"#category": ("", "8chan", "board"), "#class": _8chan._8chanBoardExtractor,
"#class" : _8chan._8chanBoardExtractor, },
{
"#url" : "https://8chan.cc/vhs/",
"#class": _8chan._8chanBoardExtractor,
}, },
) )