github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[warosu] fix extraction (#4634)

Browse Source
This commit is contained in:
Mike Fährmann
2023-10-13 23:03:39 +02:00
parent c4c4e4d2f4
commit 13ce3a9acb
2 changed files with 74 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
gallery_dl/extractor/warosu.py
View File

@@ -47,7 +47,7 @@ class WarosuThreadExtractor(Extractor):
def metadata(self, page):
boardname = text.extr(page, "<title>", "</title>")
title = text.extr(page, 'filetitle" itemprop="name">', '<')
title = text.unescape(text.extr(page, "class=filetitle>", "<"))
return {
"board" : self.board,
"board_name": boardname.rpartition(" - ")[2],
@@ -57,39 +57,37 @@ class WarosuThreadExtractor(Extractor):
def posts(self, page):
"""Build a list of all post objects"""
page = text.extr(page, '<div class="content">', '<table>')
needle = '<table itemscope itemtype="http://schema.org/Comment">'
page = text.extr(page, "<div class=content", "</form>")
needle = "<table>"
return [self.parse(post) for post in page.split(needle)]
def parse(self, post):
"""Build post object by extracting data from an HTML post"""
data = self._extract_post(post)
if "<span>File:" in post:
if "<span> File:" in post:
self._extract_image(post, data)
part = data["image"].rpartition("/")[2]
data["tim"], _, data["extension"] = part.partition(".")
data["ext"] = "." + data["extension"]
return data
@staticmethod
def _extract_post(post):
def _extract_post(self, post):
extr = text.extract_from(post)
return {
"no" : extr('id="p', '"'),
"name": extr('<span itemprop="name">', "</span>"),
"time": extr('<span class="posttime" title="', '000">'),
"now" : extr("", "<"),
"no" : extr("id=p", ">"),
"name": extr("class=postername>", "<").strip(),
"time": extr("class=posttime title=", "000>"),
"now" : extr("", "<").strip(),
"com" : text.unescape(text.remove_html(extr(
'<blockquote><p itemprop="text">', '</p></blockquote>'
).strip())),
"<blockquote>", "</blockquote>").strip())),
}
@staticmethod
def _extract_image(post, data):
def _extract_image(self, post, data):
extr = text.extract_from(post)
data["fsize"] = extr("<span>File: ", ", ")
data["fsize"] = extr("<span> File: ", ", ")
data["w"] = extr("", "x")
data["h"] = extr("", ", ")
data["filename"] = text.unquote(extr("", "<").rpartition(".")[0])
extr("<br />", "")
data["image"] = "https:" + extr('<a href="', '"')
data["filename"] = text.unquote(extr(
"", "<").rstrip().rpartition(".")[0])
extr("<br>", "")
data["image"] = self.root + extr("<a href=", ">")