github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improved foolslide-based extractors

Browse Source 
- this includes dokireader, fallenangels, jaiminisbox, powermanga,
  sensescans, worldthree, yonkouprod, gomanga, yomanga
- added 'chapter_string', 'chapter_id', 'chapter_minor' and 'count'
  keywords
- changed the 'chapter' keyword to always be just a number
- changed the default directory format
This commit is contained in:
Mike Fährmann
2017-02-16 23:42:30 +01:00
parent 0a6487afe8
commit 9a08f8a097
11 changed files with 103 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
gallery_dl/extractor/dokireader.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mike Fährmann
# Copyright 2016-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -8,19 +8,14 @@
"""Extract manga-chapters from https://kobato.hologfx.com/"""
from .foolslide import FoolslideChapterExtractor
from . import foolslide
class DokireaderChapterExtractor(FoolslideChapterExtractor):
class DokireaderChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from kobato.hologfx.com"""
category = "dokireader"
pattern = [(r"(?:https?://)?(kobato\.hologfx\.com/reader/read/"
r"[^/]+/([a-z]{2})/\d+/\d+)")]
pattern = foolslide.chapter_pattern("kobato\.hologfx\.com/reader")
test = [(("https://kobato.hologfx.com/reader/read/"
"hitoribocchi_no_oo_seikatsu/en/3/34"), {
"keyword": "4ee981ae14c6643f6a03a14c9f2c0d4898202671",
"keyword": "f28811c01b64031671108a4a3d6eea1040816b82",
})]
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

15
gallery_dl/extractor/fallenangels.py
View File

@@ -8,25 +8,20 @@
"""Extract manga-chapters from http://famatg.com/"""
from .foolslide import FoolslideChapterExtractor
from . import foolslide
class FallenangelsChapterExtractor(FoolslideChapterExtractor):
class FallenangelsChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from famatg.com"""
category = "fallenangels"
pattern = [(r"(?:https?://)?((?:manga|truyen)\.famatg\.com/read/"
r"[^/]+/([a-z]{2})/\d+/\d+(?:/\d+)?)")]
pattern = foolslide.chapter_pattern(r"(?:manga|truyen)\.famatg\.com")
test = [
("http://manga.famatg.com/read/chronos_ruler/en/0/20/", {
"url": "a777f93533674744b74c9b57c7dfa7254f5ddbed",
"keyword": "47ac083cac8a3c0aaf0f6b571a9bfb535217fd31",
"keyword": "76e7130a64d96317e3e4dcd55d770c9f6d9cb71d",
}),
("https://truyen.famatg.com/read/madan_no_ou_to_vanadis/vi/0/33/", {
"url": "b46bf1ef0537c3ce42bf2b9e4b62ace41c2299cd",
"keyword": "9eb750934f4f712211f5a7063c2206693b7cedf9",
"keyword": "658cdbecd3a1698f5462c1db437b423b6bcf7dd3",
}),
]
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

56
gallery_dl/extractor/foolslide.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mike Fährmann
# Copyright 2016-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -11,28 +11,46 @@
from .common import Extractor, Message
from .. import text, iso639_1
import json
import re
CHAPTER_RE = (
r"/read/[^/]+"
r"/(?P<lang>[a-z]{2})"
r"/(?P<volume>\d+)"
r"/(?P<chapter>\d+)"
r"(?:/(?P<chapter_minor>\d+))?)"
)
def chapter_pattern(domain_re):
return [r"(?:https?://)?(" + domain_re + CHAPTER_RE]
class FoolslideChapterExtractor(Extractor):
"""Base class for chapter extractors on foolslide based sites"""
subcategory = "chapter"
directory_fmt = ["{category}", "{manga}", "{chapter:>03} - {title}"]
directory_fmt = ["{category}", "{manga}", "{chapter_string}"]
filename_fmt = "{manga}_{chapter:>03}_{page:>03}.{extension}"
scheme = "https"
single = True
def __init__(self, url, lang):
def __init__(self, match, url=None):
Extractor.__init__(self)
self.url = url
self.lang = lang
self.url = url or self.scheme + "://" + match.group(1)
self.data = match.groupdict(default="")
def items(self):
page = self.request(self.url, encoding="utf-8",
method="post", data={"adult": "true"}).text
data = self.get_job_metadata(page)
imgs = self.get_images(page)
data["count"] = len(imgs)
data["chapter_id"] = imgs[0]["chapter_id"]
yield Message.Version, 1
yield Message.Directory, data
for data["page"], image in enumerate(self.get_images(page), 1):
for data["page"], image in enumerate(imgs, 1):
try:
url = image["url"]
del image["url"]
@@ -45,23 +63,19 @@ class FoolslideChapterExtractor(Extractor):
def get_job_metadata(self, page):
"""Collect metadata for extractor-job"""
_ , pos = text.extract(page, '<h1 class="tbtitle dnone">', '')
manga , pos = text.extract(page, 'title="', '"', pos)
chapter , pos = text.extract(page, '">', '</a>', pos)
_ , pos = text.extract(page, '<h1 class="tbtitle dnone">', '')
manga , pos = text.extract(page, 'title="', '"', pos)
chapter, pos = text.extract(page, 'title="', '"', pos)
chapter = text.unescape(chapter)
parts = chapter.split(":", maxsplit=1)
match = re.match(r"(?:Vol.(\d+) )?(?:Chapter (\d+)$|(.+))", parts[0])
volume = match.group(1) or ""
chapter = match.group(2) or match.group(3).strip()
title = parts[1].strip() if len(parts) > 1 else ""
return {
"manga": text.unescape(manga),
"chapter": chapter,
"volume": volume,
"lang": self.lang,
"language": iso639_1.code_to_language(self.lang),
"title": text.unescape(parts[1].strip() if len(parts) > 1 else ""),
}
self.data["manga"] = text.unescape(manga)
self.data["title"] = title
self.data["language"] = iso639_1.code_to_language(self.data["lang"])
self.data["chapter_string"] = chapter
return self.data
def get_images(self, page):
"""Return a list of all images in this chapter"""

25
gallery_dl/extractor/gomanga.py
View File

@@ -8,20 +8,21 @@
"""Extract manga-chapters from https://gomanga.co/"""
from .foolslide import FoolslideChapterExtractor
from . import foolslide
class GomangaChapterExtractor(FoolslideChapterExtractor):
class GomangaChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from gomanga.co"""
category = "gomanga"
pattern = [(r"(?:https?://)?(?:www\.)?(gomanga\.co/reader/read/"
r"[^/]+/([a-z]{2})/\d+/\d+)")]
test = [("https://gomanga.co/reader/read/mata-kata-omou/en/0/1/page/11", {
"url": "5088d75bb44327fc503c85b52b1d6a371b8057f2",
"keyword": "63f4d2cbbcaf3e7b5c48e71c4c4d453d9a399a26",
})]
pattern = foolslide.chapter_pattern(r"(?:www\.)?gomanga\.co/reader")
test = [
("https://gomanga.co/reader/read/mata-kata-omou/en/0/1/page/11", {
"url": "5088d75bb44327fc503c85b52b1d6a371b8057f2",
"keyword": "f534cfc4c3aef87cb0439e2a37e2ebee98077e92",
}),
("https://gomanga.co/reader/read/pastel/en/31/144/", {
"url": "9cc2052fbf36344c573c754c5abe533a14b3e280",
"keyword": "a2ef55d26984c64baf026382f889bb013d01dc4f",
}),
]
single = False
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

15
gallery_dl/extractor/jaiminisbox.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mike Fährmann
# Copyright 2016-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -8,19 +8,14 @@
"""Extract manga-chapters from https://jaiminisbox.com/"""
from .foolslide import FoolslideChapterExtractor
from . import foolslide
class JaiminisboxChapterExtractor(FoolslideChapterExtractor):
class JaiminisboxChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from jaiminisbox.com"""
category = "jaiminisbox"
pattern = [(r"(?:https?://)?(?:www\.)?(jaiminisbox.com/reader/read/"
r"[^/]+/([a-z]{2})/\d+/\d+)")]
pattern = foolslide.chapter_pattern(r"(?:www\.)?jaiminisbox.com/reader")
test = [("https://jaiminisbox.com/reader/read/uratarou/en/0/1/", {
"url": "f021de7f31ee3a3f688fdf3e8183aef4226c2b50",
"keyword": "836e94f68b78159cc10d12b72c981c276ff45b3f",
"keyword": "d187df3e3b6dbe09ec163626f6fd7c57133ab163",
})]
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

27
gallery_dl/extractor/powermanga.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2015, 2016 Mike Fährmann
# Copyright 2015-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -8,31 +8,14 @@
"""Extract manga-chapters from http://powermanga.org/"""
from .foolslide import FoolslideChapterExtractor
from .. import text
import re
from . import foolslide
class PowermangaChapterExtractor(FoolslideChapterExtractor):
class PowermangaChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from powermanga.org"""
category = "powermanga"
pattern = [
(r"(?:https?://)?read(?:er)?\.powermanga\.org/read/"
r"(.+/([a-z]{2})/\d+/\d+)(?:/page)?"),
(r"(?:https?://)?(?:www\.)?(p)owermanga\.org/((?:[^-]+-)+[^-]+/?)"),
]
pattern = foolslide.chapter_pattern(r"read(?:er)?\.powermanga\.org")
test = [("https://read.powermanga.org/read/one_piece/en/0/803/page/1", {
"url": "e6179c1565068f99180620281f86bdd25be166b4",
"keyword": "51cabad8995727334e5ca9773c18d709b3868f02",
"keyword": "203ea5d0ef7759f4517316f0678f3592fc27cdbe",
})]
def __init__(self, match):
if match.group(1) == "p":
url = "https://powermanga.org/" + match.group(2)
page = self.request(url).text
pos = page.index("class='small-button smallblack'>Download</a>")
url = text.extract(page, "<a href='", "'", pos)[0]
match = re.match(self.pattern[0], url)
extra = "er" if "://reader" in match.string else ""
url = "https://read" + extra + ".powermanga.org/read/" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

18
gallery_dl/extractor/sensescans.py
View File

@@ -8,25 +8,27 @@
"""Extract manga-chapters from http://sensescans.com/"""
from .foolslide import FoolslideChapterExtractor
from . import foolslide
class SensescansChapterExtractor(FoolslideChapterExtractor):
class SensescansChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from sensescans.com"""
category = "sensescans"
pattern = [(r"(?:https?://)?(?:www\.|reader\.)?sensescans\.com/"
r"(?:reader/)?read/([^/]+/([a-z]{2})/\d+/\d+)")]
pattern = [(r"(?:https?://)?(?:www\.|reader\.)?sensescans\.com"
r"(?:/reader)?(" + foolslide.CHAPTER_RE)]
test = [
(("http://reader.sensescans.com/read/"
"magi__labyrinth_of_magic/en/33/319/page/1"), {
"url": "cc192cbeed36127d374926c50c3a4bd06092b760",
"keyword": "77f906f04bf49d3bd636e8c92d85dc25aa361754"}),
"keyword": "f80104d205e3f537ca11fef9e761393c956768f0",
}),
(("http://sensescans.com/reader/read/"
"magi__labyrinth_of_magic/en/33/319/page/1"), {
"url": "cc192cbeed36127d374926c50c3a4bd06092b760",
"keyword": "77f906f04bf49d3bd636e8c92d85dc25aa361754"}),
"keyword": "f80104d205e3f537ca11fef9e761393c956768f0",
}),
]
def __init__(self, match):
url = "http://sensescans.com/reader/read/" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))
url = "http://sensescans.com/reader" + match.group(1)
super().__init__(match, url)

31
gallery_dl/extractor/worldthree.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mike Fährmann
# Copyright 2016-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -8,20 +8,23 @@
"""Extract manga-chapters from http://www.slide.world-three.org/"""
from .foolslide import FoolslideChapterExtractor
from . import foolslide
class WorldthreeChapterExtractor(FoolslideChapterExtractor):
class WorldthreeChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from slide.world-three.org"""
category = "worldthree"
pattern = [(r"(?:https?://)?(?:www\.)?(slide\.world-three\.org/read/"
r"[^/]+/([a-z]{2})/\d+/\d+)")]
test = [(("http://www.slide.world-three.org/"
"read/black_bullet/en/2/7/page/1"), {
"url": "be2f04f6e2d311b35188094cfd3e768583271584",
"keyword": "6d77d9fc806d76d881491a52ccd8dfd875c47d05",
})]
def __init__(self, match):
url = "http://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))
pattern = foolslide.chapter_pattern("(?:www\.)?slide\.world-three\.org")
test = [
(("http://www.slide.world-three.org"
"/read/black_bullet/en/2/7/page/1"), {
"url": "be2f04f6e2d311b35188094cfd3e768583271584",
"keyword": "25fd070bc93ee8ad316f5b7d1bd9011c9bcf0616",
}),
(("http://www.slide.world-three.org"
"/read/idolmster_cg_shuffle/en/0/4/2/"), {
"url": "6028ea5ca282744f925dfad92eeb98509f9cc78c",
"keyword": "10e3dc961ac2c9395f4d1f3ad3b9ad84113e7366",
}),
]
scheme = "http"

12
gallery_dl/extractor/yomanga.py
View File

@@ -8,20 +8,14 @@
"""Extract manga-chapters from https://yomanga.co/"""
from .foolslide import FoolslideChapterExtractor
from . import foolslide
class YomangaChapterExtractor(FoolslideChapterExtractor):
class YomangaChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from yomanga.co"""
category = "yomanga"
pattern = [(r"(?:https?://)?(?:www\.)?(yomanga\.co/reader/read/"
r"[^/]+/([a-z]{2})/\d+/\d+)")]
pattern = foolslide.chapter_pattern(r"(?:www\.)?yomanga\.co/reader")
test = [("https://yomanga.co/reader/read/uwakoi/en/0/2/", {
"url": "4b5d8fc5902f03647cc876cf6643849e5bc05455",
"keyword": "1b9ac4217146421dbcb2a1108693054c56554a9d",
})]
single = False
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

13
gallery_dl/extractor/yonkouprod.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2016, 2017 Mike Fährmann
# Copyright 2016-2017 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -8,19 +8,14 @@
"""Extract manga-chapters from https://yonkouprod.com/"""
from .foolslide import FoolslideChapterExtractor
from . import foolslide
class YonkouprodChapterExtractor(FoolslideChapterExtractor):
class YonkouprodChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from yonkouprod.com"""
category = "yonkouprod"
pattern = [(r"(?:https?://)?(?:www\.)?(yonkouprod\.com/reader/read/"
r"[^/]+/([a-z]{2})/\d+/\d+)")]
pattern = foolslide.chapter_pattern(r"(?:www\.)?yonkouprod\.com/reader")
test = [("http://yonkouprod.com/reader/read/fairy-tail/en/0/512/", {
"url": "7647850e2b1ad11c2baa9628755bf7f186350a0b",
"keyword": "d079c718d6620478fa72a700fdd027f9a0f0760b",
})]
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

2
test/test_extractors.py
View File

@@ -48,7 +48,7 @@ def _generate_test(extr, tcase):
# enable selective testing for direct calls
skip = ["exhentai", "kissmanga", "mangafox"]
skip = ["exhentai", "kissmanga", "mangafox", "yomanga"]
if __name__ == '__main__' and len(sys.argv) > 1:
extractors = [
extr for extr in extractor.extractors()