github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[mangahere] fix extraction

Browse Source
This commit is contained in:
Mike Fährmann
2019-02-10 22:09:29 +01:00
parent 84ae72b8d8
commit 1f3422c28b
2 changed files with 58 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
gallery_dl/extractor/mangahere.py
View File

@@ -10,7 +10,6 @@
from .common import ChapterExtractor, MangaExtractor from .common import ChapterExtractor, MangaExtractor
from .. import text from .. import text
from ..cache import memcache
import re import re
@@ -18,7 +17,8 @@ class MangahereBase():
"""Base class for mangahere extractors""" """Base class for mangahere extractors"""
category = "mangahere" category = "mangahere"
root = "https://www.mangahere.cc" root = "https://www.mangahere.cc"
url_fmt = root + "/manga/{}/{}.html" mobile_root = "https://m.mangahere.cc"
url_fmt = mobile_root + "/manga/{}/{}.html"
class MangahereMangaExtractor(MangahereBase, MangaExtractor): class MangahereMangaExtractor(MangahereBase, MangaExtractor):
@@ -27,15 +27,15 @@ class MangahereMangaExtractor(MangahereBase, MangaExtractor):
r"/manga/([^/]+)/?(?:#.*)?$") r"/manga/([^/]+)/?(?:#.*)?$")
test = ( test = (
("https://www.mangahere.cc/manga/aria/", { ("https://www.mangahere.cc/manga/aria/", {
"url": "e8971b1605d9888d978ebb2895adb1c7c37d663c", "url": "23ad9256f7392de5973b79a36f6875e9fdcb7563",
"keyword": "951eef36a3775525a31ca78c9d9cea546f4cf2f5", "keyword": "79e326641e7d5d2fed43a1eb9949471b8162a9e0",
}), }),
("http://www.mangahere.cc/manga/hiyokoi#50", { ("https://www.mangahere.cc/manga/hiyokoi/#50", {
"url": "6df27c0e105d9ee0b78a7aa77340d0891e6c7fc6", "url": "654850570aa03825cd57e2ae2904af489602c523",
"keyword": "9542283639bd082fabf3a14b6695697d3ef15111", "keyword": "c8084d89a9ea6cf40353093669f9601a39bf5ca2",
}), }),
("http://www.mangahere.co/manga/aria/"), ("https://www.mangahere.co/manga/aria/"),
("http://m.mangahere.co/manga/aria/"), ("https://m.mangahere.co/manga/aria/"),
) )
def __init__(self, match): def __init__(self, match):
@@ -44,25 +44,38 @@ class MangahereMangaExtractor(MangahereBase, MangaExtractor):
def chapters(self, page): def chapters(self, page):
results = [] results = []
pos = page.index('<div class="detail_list">') manga, pos = text.extract(page, '<meta name="og:title" content="', '"')
manga, pos = text.extract(page, '<h3>Read ', ' Online</h3>', pos)
manga = text.unescape(manga) manga = text.unescape(manga)
page = text.extract(
page, 'id="chapterlist"', 'class="detail-main-list-more"', pos)[0]
pos = 0
while True: while True:
url, pos = text.extract( url, pos = text.extract(page, ' href="', '"', pos)
page, '<a class="color_0077" href="', '"', pos)
if not url: if not url:
return results return results
chapter, dot, minor = url[:-1].rpartition("/c")[2].partition(".") info, pos = text.extract(page, 'class="title3">', '<', pos)
volume, pos = text.extract(page, 'span class="mr6">', '<', pos) date, pos = text.extract(page, 'class="title2">', '<', pos)
title, pos = text.extract(page, '/span>', '<', pos)
date, pos = text.extract(page, 'class="right">', '</span>', pos) match = re.match(
results.append((text.urljoin("http:", url), { r"(?:Vol\.0*(\d+) )?Ch\.0*(\d+)(\S*)(?: - (.*))?", info)
"manga": manga, "title": title, "date": date, if match:
"volume": text.parse_int(volume.rpartition(" ")[2]), volume, chapter, minor, title = match.groups()
else:
chapter, _, minor = url[:-1].rpartition("/c")[2].partition(".")
minor = "." + minor
volume = 0
title = ""
results.append((text.urljoin(self.root, url), {
"manga": manga,
"title": text.unescape(title) if title else "",
"volume": text.parse_int(volume),
"chapter": text.parse_int(chapter), "chapter": text.parse_int(chapter),
"chapter_minor": dot + minor, "chapter_minor": minor,
"lang": "en", "language": "English", "date": date,
"lang": "en",
"language": "English",
})) }))
@@ -72,7 +85,7 @@ class MangahereChapterExtractor(MangahereBase, ChapterExtractor):
r"([^/]+(?:/v0*(\d+))?/c([^/?&#]+))") r"([^/]+(?:/v0*(\d+))?/c([^/?&#]+))")
test = ( test = (
("https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", { ("https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", {
"keyword": "0e1cee6dd377da02ad51aa810ba65db3e811aef9", "keyword": "6407556817bd1fd2bdc8dee3fd2a718f5724ddc0",
"content": "708d475f06893b88549cbd30df1e3f9428f2c884", "content": "708d475f06893b88549cbd30df1e3f9428f2c884",
}), }),
("http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/"), ("http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/"),
@@ -81,27 +94,25 @@ class MangahereChapterExtractor(MangahereBase, ChapterExtractor):
def __init__(self, match): def __init__(self, match):
self.part, self.volume, self.chapter = match.groups() self.part, self.volume, self.chapter = match.groups()
# remove ".html" for the first chapter page to avoid redirects ChapterExtractor.__init__(self, self.url_fmt.format(self.part, 1))
url = self.url_fmt.format(self.part, "")[:-5]
ChapterExtractor.__init__(self, url)
def get_metadata(self, page): def get_metadata(self, page):
"""Collect metadata for extractor-job""" """Collect metadata for extractor-job"""
manga, pos = text.extract(page, '<title>', '</title>') pos = page.index("</select>")
mid , pos = text.extract(page, '.net/store/manga/', '/', pos) count , pos = text.extract(page, ">", "<", pos - 20)
pages, pos = text.extract(page, ' class="wid60"', '</select>', pos) manga_id , pos = text.extract(page, "series_id = ", ";", pos)
count = re.findall(r">(\d+)<", pages)[-1] chapter_id, pos = text.extract(page, "chapter_id = ", ";", pos)
manga = re.match((r"(.+) \d+(\.\d+)? - Read .+ Chapter " manga , pos = text.extract(page, '"name":"', '"', pos)
r"\d+(\.\d+)? Online"), manga).group(1)
chapter, dot, minor = self.chapter.partition(".") chapter, dot, minor = self.chapter.partition(".")
return { return {
"manga": text.unescape(manga), "manga": text.unescape(manga),
"manga_id": text.parse_int(mid), "manga_id": text.parse_int(manga_id),
"title": self._get_title_map(mid).get(self.chapter), "title": self._get_title(),
"volume": text.parse_int(self.volume), "volume": text.parse_int(self.volume),
"chapter": text.parse_int(chapter), "chapter": text.parse_int(chapter),
"chapter_minor": dot + minor, "chapter_minor": dot + minor,
"chapter_id": text.parse_int(chapter_id),
"count": text.parse_int(count), "count": text.parse_int(count),
"lang": "en", "lang": "en",
"language": "English", "language": "English",
@@ -110,23 +121,22 @@ class MangahereChapterExtractor(MangahereBase, ChapterExtractor):
def get_images(self, page): def get_images(self, page):
"""Yield all image-urls for this chapter""" """Yield all image-urls for this chapter"""
pnum = 1 pnum = 1
while True: while True:
for url in text.extract_iter(page, '<img src="', '"'): url, pos = text.extract(page, '<img src="', '"')
if "/store/manga/" in url: yield url, None
yield url, None url, pos = text.extract(page, ' src="', '"', pos)
yield url, None
pnum += 2 pnum += 2
page = self.request(self.url_fmt.format(self.part, pnum)).text page = self.request(self.url_fmt.format(self.part, pnum)).text
@memcache(keyarg=1) def _get_title(self):
def _get_title_map(self, manga_id): url = "{}/manga/{}/".format(self.root, self.part)
url = "{}/get_chapters{}.js".format(self.root, manga_id)
page = self.request(url).text page = self.request(url).text
chapters = {} try:
for info in text.extract_iter(page, '["', '"]'): pos = page.index(self.part) + len(self.part)
title, _, url = info.partition('","') pos = page.index(self.part, pos) + len(self.part)
title = title.partition(": ")[2] return text.extract(page, ' title="', '"', pos)[0]
num = url.rpartition("c")[2].rstrip("/") except ValueError:
chapters[num] = text.unescape(title) return ""
return chapters

1
test/test_results.py
View File

@@ -23,7 +23,6 @@ TRAVIS_SKIP = {
# temporary issues, etc. # temporary issues, etc.
BROKEN = { BROKEN = {
"mangahere",
} }