[mangahere] fix extraction
This commit is contained in:
Mike Fährmann
@@ -10,7 +10,6 @@
|
||||
|
||||
from .common import ChapterExtractor, MangaExtractor
|
||||
from .. import text
|
||||
from ..cache import memcache
|
||||
import re
|
||||
|
||||
|
||||
@@ -18,7 +17,8 @@ class MangahereBase():
|
||||
"""Base class for mangahere extractors"""
|
||||
category = "mangahere"
|
||||
root = "https://www.mangahere.cc"
|
||||
url_fmt = root + "/manga/{}/{}.html"
|
||||
mobile_root = "https://m.mangahere.cc"
|
||||
url_fmt = mobile_root + "/manga/{}/{}.html"
|
||||
|
||||
|
||||
class MangahereMangaExtractor(MangahereBase, MangaExtractor):
|
||||
@@ -27,15 +27,15 @@ class MangahereMangaExtractor(MangahereBase, MangaExtractor):
|
||||
r"/manga/([^/]+)/?(?:#.*)?$")
|
||||
test = (
|
||||
("https://www.mangahere.cc/manga/aria/", {
|
||||
"url": "e8971b1605d9888d978ebb2895adb1c7c37d663c",
|
||||
"keyword": "951eef36a3775525a31ca78c9d9cea546f4cf2f5",
|
||||
"url": "23ad9256f7392de5973b79a36f6875e9fdcb7563",
|
||||
"keyword": "79e326641e7d5d2fed43a1eb9949471b8162a9e0",
|
||||
}),
|
||||
("http://www.mangahere.cc/manga/hiyokoi#50", {
|
||||
"url": "6df27c0e105d9ee0b78a7aa77340d0891e6c7fc6",
|
||||
"keyword": "9542283639bd082fabf3a14b6695697d3ef15111",
|
||||
("https://www.mangahere.cc/manga/hiyokoi/#50", {
|
||||
"url": "654850570aa03825cd57e2ae2904af489602c523",
|
||||
"keyword": "c8084d89a9ea6cf40353093669f9601a39bf5ca2",
|
||||
}),
|
||||
("http://www.mangahere.co/manga/aria/"),
|
||||
("http://m.mangahere.co/manga/aria/"),
|
||||
("https://www.mangahere.co/manga/aria/"),
|
||||
("https://m.mangahere.co/manga/aria/"),
|
||||
)
|
||||
|
||||
def __init__(self, match):
|
||||
@@ -44,25 +44,38 @@ class MangahereMangaExtractor(MangahereBase, MangaExtractor):
|
||||
|
||||
def chapters(self, page):
|
||||
results = []
|
||||
pos = page.index('<div class="detail_list">')
|
||||
manga, pos = text.extract(page, '<h3>Read ', ' Online</h3>', pos)
|
||||
manga, pos = text.extract(page, '<meta name="og:title" content="', '"')
|
||||
manga = text.unescape(manga)
|
||||
|
||||
page = text.extract(
|
||||
page, 'id="chapterlist"', 'class="detail-main-list-more"', pos)[0]
|
||||
pos = 0
|
||||
while True:
|
||||
url, pos = text.extract(
|
||||
page, '<a class="color_0077" href="', '"', pos)
|
||||
url, pos = text.extract(page, ' href="', '"', pos)
|
||||
if not url:
|
||||
return results
|
||||
chapter, dot, minor = url[:-1].rpartition("/c")[2].partition(".")
|
||||
volume, pos = text.extract(page, 'span class="mr6">', '<', pos)
|
||||
title, pos = text.extract(page, '/span>', '<', pos)
|
||||
date, pos = text.extract(page, 'class="right">', '</span>', pos)
|
||||
results.append((text.urljoin("http:", url), {
|
||||
"manga": manga, "title": title, "date": date,
|
||||
"volume": text.parse_int(volume.rpartition(" ")[2]),
|
||||
info, pos = text.extract(page, 'class="title3">', '<', pos)
|
||||
date, pos = text.extract(page, 'class="title2">', '<', pos)
|
||||
|
||||
match = re.match(
|
||||
r"(?:Vol\.0*(\d+) )?Ch\.0*(\d+)(\S*)(?: - (.*))?", info)
|
||||
if match:
|
||||
volume, chapter, minor, title = match.groups()
|
||||
else:
|
||||
chapter, _, minor = url[:-1].rpartition("/c")[2].partition(".")
|
||||
minor = "." + minor
|
||||
volume = 0
|
||||
title = ""
|
||||
|
||||
results.append((text.urljoin(self.root, url), {
|
||||
"manga": manga,
|
||||
"title": text.unescape(title) if title else "",
|
||||
"volume": text.parse_int(volume),
|
||||
"chapter": text.parse_int(chapter),
|
||||
"chapter_minor": dot + minor,
|
||||
"lang": "en", "language": "English",
|
||||
"chapter_minor": minor,
|
||||
"date": date,
|
||||
"lang": "en",
|
||||
"language": "English",
|
||||
}))
|
||||
|
||||
|
||||
@@ -72,7 +85,7 @@ class MangahereChapterExtractor(MangahereBase, ChapterExtractor):
|
||||
r"([^/]+(?:/v0*(\d+))?/c([^/?&#]+))")
|
||||
test = (
|
||||
("https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", {
|
||||
"keyword": "0e1cee6dd377da02ad51aa810ba65db3e811aef9",
|
||||
"keyword": "6407556817bd1fd2bdc8dee3fd2a718f5724ddc0",
|
||||
"content": "708d475f06893b88549cbd30df1e3f9428f2c884",
|
||||
}),
|
||||
("http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/"),
|
||||
@@ -81,27 +94,25 @@ class MangahereChapterExtractor(MangahereBase, ChapterExtractor):
|
||||
|
||||
def __init__(self, match):
|
||||
self.part, self.volume, self.chapter = match.groups()
|
||||
# remove ".html" for the first chapter page to avoid redirects
|
||||
url = self.url_fmt.format(self.part, "")[:-5]
|
||||
ChapterExtractor.__init__(self, url)
|
||||
ChapterExtractor.__init__(self, self.url_fmt.format(self.part, 1))
|
||||
|
||||
def get_metadata(self, page):
|
||||
"""Collect metadata for extractor-job"""
|
||||
manga, pos = text.extract(page, '<title>', '</title>')
|
||||
mid , pos = text.extract(page, '.net/store/manga/', '/', pos)
|
||||
pages, pos = text.extract(page, ' class="wid60"', '</select>', pos)
|
||||
count = re.findall(r">(\d+)<", pages)[-1]
|
||||
manga = re.match((r"(.+) \d+(\.\d+)? - Read .+ Chapter "
|
||||
r"\d+(\.\d+)? Online"), manga).group(1)
|
||||
pos = page.index("</select>")
|
||||
count , pos = text.extract(page, ">", "<", pos - 20)
|
||||
manga_id , pos = text.extract(page, "series_id = ", ";", pos)
|
||||
chapter_id, pos = text.extract(page, "chapter_id = ", ";", pos)
|
||||
manga , pos = text.extract(page, '"name":"', '"', pos)
|
||||
chapter, dot, minor = self.chapter.partition(".")
|
||||
|
||||
return {
|
||||
"manga": text.unescape(manga),
|
||||
"manga_id": text.parse_int(mid),
|
||||
"title": self._get_title_map(mid).get(self.chapter),
|
||||
"manga_id": text.parse_int(manga_id),
|
||||
"title": self._get_title(),
|
||||
"volume": text.parse_int(self.volume),
|
||||
"chapter": text.parse_int(chapter),
|
||||
"chapter_minor": dot + minor,
|
||||
"chapter_id": text.parse_int(chapter_id),
|
||||
"count": text.parse_int(count),
|
||||
"lang": "en",
|
||||
"language": "English",
|
||||
@@ -110,23 +121,22 @@ class MangahereChapterExtractor(MangahereBase, ChapterExtractor):
|
||||
def get_images(self, page):
|
||||
"""Yield all image-urls for this chapter"""
|
||||
pnum = 1
|
||||
|
||||
while True:
|
||||
for url in text.extract_iter(page, '<img src="', '"'):
|
||||
if "/store/manga/" in url:
|
||||
yield url, None
|
||||
url, pos = text.extract(page, '<img src="', '"')
|
||||
yield url, None
|
||||
url, pos = text.extract(page, ' src="', '"', pos)
|
||||
yield url, None
|
||||
pnum += 2
|
||||
page = self.request(self.url_fmt.format(self.part, pnum)).text
|
||||
|
||||
@memcache(keyarg=1)
|
||||
def _get_title_map(self, manga_id):
|
||||
url = "{}/get_chapters{}.js".format(self.root, manga_id)
|
||||
def _get_title(self):
|
||||
url = "{}/manga/{}/".format(self.root, self.part)
|
||||
page = self.request(url).text
|
||||
|
||||
chapters = {}
|
||||
for info in text.extract_iter(page, '["', '"]'):
|
||||
title, _, url = info.partition('","')
|
||||
title = title.partition(": ")[2]
|
||||
num = url.rpartition("c")[2].rstrip("/")
|
||||
chapters[num] = text.unescape(title)
|
||||
|
||||
return chapters
|
||||
try:
|
||||
pos = page.index(self.part) + len(self.part)
|
||||
pos = page.index(self.part, pos) + len(self.part)
|
||||
return text.extract(page, ' title="', '"', pos)[0]
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
@@ -23,7 +23,6 @@ TRAVIS_SKIP = {
|
||||
|
||||
# temporary issues, etc.
|
||||
BROKEN = {
|
||||
"mangahere",
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user