github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improved foolslide-based extractors

Browse Source 
- this includes dokireader, fallenangels, jaiminisbox, powermanga,
  sensescans, worldthree, yonkouprod, gomanga, yomanga
- added 'chapter_string', 'chapter_id', 'chapter_minor' and 'count'
  keywords
- changed the 'chapter' keyword to always be just a number
- changed the default directory format
This commit is contained in:
Mike Fährmann
2017-02-16 23:42:30 +01:00
parent 0a6487afe8
commit 9a08f8a097
11 changed files with 103 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
gallery_dl/extractor/dokireader.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2016 Mike Fährmann # Copyright 2016-2017 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@@ -8,19 +8,14 @@
"""Extract manga-chapters from https://kobato.hologfx.com/""" """Extract manga-chapters from https://kobato.hologfx.com/"""
from .foolslide import FoolslideChapterExtractor from . import foolslide
class DokireaderChapterExtractor(FoolslideChapterExtractor): class DokireaderChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from kobato.hologfx.com""" """Extractor for manga-chapters from kobato.hologfx.com"""
category = "dokireader" category = "dokireader"
pattern = [(r"(?:https?://)?(kobato\.hologfx\.com/reader/read/" pattern = foolslide.chapter_pattern("kobato\.hologfx\.com/reader")
r"[^/]+/([a-z]{2})/\d+/\d+)")]
test = [(("https://kobato.hologfx.com/reader/read/" test = [(("https://kobato.hologfx.com/reader/read/"
"hitoribocchi_no_oo_seikatsu/en/3/34"), { "hitoribocchi_no_oo_seikatsu/en/3/34"), {
"keyword": "4ee981ae14c6643f6a03a14c9f2c0d4898202671", "keyword": "f28811c01b64031671108a4a3d6eea1040816b82",
})] })]
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

15
gallery_dl/extractor/fallenangels.py
View File

@@ -8,25 +8,20 @@
"""Extract manga-chapters from http://famatg.com/""" """Extract manga-chapters from http://famatg.com/"""
from .foolslide import FoolslideChapterExtractor from . import foolslide
class FallenangelsChapterExtractor(FoolslideChapterExtractor): class FallenangelsChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from famatg.com""" """Extractor for manga-chapters from famatg.com"""
category = "fallenangels" category = "fallenangels"
pattern = [(r"(?:https?://)?((?:manga|truyen)\.famatg\.com/read/" pattern = foolslide.chapter_pattern(r"(?:manga|truyen)\.famatg\.com")
r"[^/]+/([a-z]{2})/\d+/\d+(?:/\d+)?)")]
test = [ test = [
("http://manga.famatg.com/read/chronos_ruler/en/0/20/", { ("http://manga.famatg.com/read/chronos_ruler/en/0/20/", {
"url": "a777f93533674744b74c9b57c7dfa7254f5ddbed", "url": "a777f93533674744b74c9b57c7dfa7254f5ddbed",
"keyword": "47ac083cac8a3c0aaf0f6b571a9bfb535217fd31", "keyword": "76e7130a64d96317e3e4dcd55d770c9f6d9cb71d",
}), }),
("https://truyen.famatg.com/read/madan_no_ou_to_vanadis/vi/0/33/", { ("https://truyen.famatg.com/read/madan_no_ou_to_vanadis/vi/0/33/", {
"url": "b46bf1ef0537c3ce42bf2b9e4b62ace41c2299cd", "url": "b46bf1ef0537c3ce42bf2b9e4b62ace41c2299cd",
"keyword": "9eb750934f4f712211f5a7063c2206693b7cedf9", "keyword": "658cdbecd3a1698f5462c1db437b423b6bcf7dd3",
}), }),
] ]
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

56
gallery_dl/extractor/foolslide.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2016 Mike Fährmann # Copyright 2016-2017 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@@ -11,28 +11,46 @@
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, iso639_1 from .. import text, iso639_1
import json import json
import re
CHAPTER_RE = (
r"/read/[^/]+"
r"/(?P<lang>[a-z]{2})"
r"/(?P<volume>\d+)"
r"/(?P<chapter>\d+)"
r"(?:/(?P<chapter_minor>\d+))?)"
)
def chapter_pattern(domain_re):
return [r"(?:https?://)?(" + domain_re + CHAPTER_RE]
class FoolslideChapterExtractor(Extractor): class FoolslideChapterExtractor(Extractor):
"""Base class for chapter extractors on foolslide based sites""" """Base class for chapter extractors on foolslide based sites"""
subcategory = "chapter" subcategory = "chapter"
directory_fmt = ["{category}", "{manga}", "{chapter:>03} - {title}"] directory_fmt = ["{category}", "{manga}", "{chapter_string}"]
filename_fmt = "{manga}_{chapter:>03}_{page:>03}.{extension}" filename_fmt = "{manga}_{chapter:>03}_{page:>03}.{extension}"
scheme = "https"
single = True single = True
def __init__(self, url, lang): def __init__(self, match, url=None):
Extractor.__init__(self) Extractor.__init__(self)
self.url = url self.url = url or self.scheme + "://" + match.group(1)
self.lang = lang self.data = match.groupdict(default="")
def items(self): def items(self):
page = self.request(self.url, encoding="utf-8", page = self.request(self.url, encoding="utf-8",
method="post", data={"adult": "true"}).text method="post", data={"adult": "true"}).text
data = self.get_job_metadata(page) data = self.get_job_metadata(page)
imgs = self.get_images(page)
data["count"] = len(imgs)
data["chapter_id"] = imgs[0]["chapter_id"]
yield Message.Version, 1 yield Message.Version, 1
yield Message.Directory, data yield Message.Directory, data
for data["page"], image in enumerate(self.get_images(page), 1): for data["page"], image in enumerate(imgs, 1):
try: try:
url = image["url"] url = image["url"]
del image["url"] del image["url"]
@@ -45,23 +63,19 @@ class FoolslideChapterExtractor(Extractor):
def get_job_metadata(self, page): def get_job_metadata(self, page):
"""Collect metadata for extractor-job""" """Collect metadata for extractor-job"""
_ , pos = text.extract(page, '<h1 class="tbtitle dnone">', '') _ , pos = text.extract(page, '<h1 class="tbtitle dnone">', '')
manga , pos = text.extract(page, 'title="', '"', pos) manga , pos = text.extract(page, 'title="', '"', pos)
chapter , pos = text.extract(page, '">', '</a>', pos) chapter, pos = text.extract(page, 'title="', '"', pos)
chapter = text.unescape(chapter)
parts = chapter.split(":", maxsplit=1) parts = chapter.split(":", maxsplit=1)
match = re.match(r"(?:Vol.(\d+) )?(?:Chapter (\d+)$|(.+))", parts[0]) title = parts[1].strip() if len(parts) > 1 else ""
volume = match.group(1) or ""
chapter = match.group(2) or match.group(3).strip()
return { self.data["manga"] = text.unescape(manga)
"manga": text.unescape(manga), self.data["title"] = title
"chapter": chapter, self.data["language"] = iso639_1.code_to_language(self.data["lang"])
"volume": volume, self.data["chapter_string"] = chapter
"lang": self.lang, return self.data
"language": iso639_1.code_to_language(self.lang),
"title": text.unescape(parts[1].strip() if len(parts) > 1 else ""),
}
def get_images(self, page): def get_images(self, page):
"""Return a list of all images in this chapter""" """Return a list of all images in this chapter"""

25
gallery_dl/extractor/gomanga.py
View File

@@ -8,20 +8,21 @@
"""Extract manga-chapters from https://gomanga.co/""" """Extract manga-chapters from https://gomanga.co/"""
from .foolslide import FoolslideChapterExtractor from . import foolslide
class GomangaChapterExtractor(FoolslideChapterExtractor): class GomangaChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from gomanga.co""" """Extractor for manga-chapters from gomanga.co"""
category = "gomanga" category = "gomanga"
pattern = [(r"(?:https?://)?(?:www\.)?(gomanga\.co/reader/read/" pattern = foolslide.chapter_pattern(r"(?:www\.)?gomanga\.co/reader")
r"[^/]+/([a-z]{2})/\d+/\d+)")] test = [
test = [("https://gomanga.co/reader/read/mata-kata-omou/en/0/1/page/11", { ("https://gomanga.co/reader/read/mata-kata-omou/en/0/1/page/11", {
"url": "5088d75bb44327fc503c85b52b1d6a371b8057f2", "url": "5088d75bb44327fc503c85b52b1d6a371b8057f2",
"keyword": "63f4d2cbbcaf3e7b5c48e71c4c4d453d9a399a26", "keyword": "f534cfc4c3aef87cb0439e2a37e2ebee98077e92",
})] }),
("https://gomanga.co/reader/read/pastel/en/31/144/", {
"url": "9cc2052fbf36344c573c754c5abe533a14b3e280",
"keyword": "a2ef55d26984c64baf026382f889bb013d01dc4f",
}),
]
single = False single = False
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

15
gallery_dl/extractor/jaiminisbox.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2016 Mike Fährmann # Copyright 2016-2017 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@@ -8,19 +8,14 @@
"""Extract manga-chapters from https://jaiminisbox.com/""" """Extract manga-chapters from https://jaiminisbox.com/"""
from .foolslide import FoolslideChapterExtractor from . import foolslide
class JaiminisboxChapterExtractor(FoolslideChapterExtractor): class JaiminisboxChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from jaiminisbox.com""" """Extractor for manga-chapters from jaiminisbox.com"""
category = "jaiminisbox" category = "jaiminisbox"
pattern = [(r"(?:https?://)?(?:www\.)?(jaiminisbox.com/reader/read/" pattern = foolslide.chapter_pattern(r"(?:www\.)?jaiminisbox.com/reader")
r"[^/]+/([a-z]{2})/\d+/\d+)")]
test = [("https://jaiminisbox.com/reader/read/uratarou/en/0/1/", { test = [("https://jaiminisbox.com/reader/read/uratarou/en/0/1/", {
"url": "f021de7f31ee3a3f688fdf3e8183aef4226c2b50", "url": "f021de7f31ee3a3f688fdf3e8183aef4226c2b50",
"keyword": "836e94f68b78159cc10d12b72c981c276ff45b3f", "keyword": "d187df3e3b6dbe09ec163626f6fd7c57133ab163",
})] })]
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

27
gallery_dl/extractor/powermanga.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2015, 2016 Mike Fährmann # Copyright 2015-2017 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@@ -8,31 +8,14 @@
"""Extract manga-chapters from http://powermanga.org/""" """Extract manga-chapters from http://powermanga.org/"""
from .foolslide import FoolslideChapterExtractor from . import foolslide
from .. import text
import re
class PowermangaChapterExtractor(FoolslideChapterExtractor): class PowermangaChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from powermanga.org""" """Extractor for manga-chapters from powermanga.org"""
category = "powermanga" category = "powermanga"
pattern = [ pattern = foolslide.chapter_pattern(r"read(?:er)?\.powermanga\.org")
(r"(?:https?://)?read(?:er)?\.powermanga\.org/read/"
r"(.+/([a-z]{2})/\d+/\d+)(?:/page)?"),
(r"(?:https?://)?(?:www\.)?(p)owermanga\.org/((?:[^-]+-)+[^-]+/?)"),
]
test = [("https://read.powermanga.org/read/one_piece/en/0/803/page/1", { test = [("https://read.powermanga.org/read/one_piece/en/0/803/page/1", {
"url": "e6179c1565068f99180620281f86bdd25be166b4", "url": "e6179c1565068f99180620281f86bdd25be166b4",
"keyword": "51cabad8995727334e5ca9773c18d709b3868f02", "keyword": "203ea5d0ef7759f4517316f0678f3592fc27cdbe",
})] })]
def __init__(self, match):
if match.group(1) == "p":
url = "https://powermanga.org/" + match.group(2)
page = self.request(url).text
pos = page.index("class='small-button smallblack'>Download</a>")
url = text.extract(page, "<a href='", "'", pos)[0]
match = re.match(self.pattern[0], url)
extra = "er" if "://reader" in match.string else ""
url = "https://read" + extra + ".powermanga.org/read/" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

18
gallery_dl/extractor/sensescans.py
View File

@@ -8,25 +8,27 @@
"""Extract manga-chapters from http://sensescans.com/""" """Extract manga-chapters from http://sensescans.com/"""
from .foolslide import FoolslideChapterExtractor from . import foolslide
class SensescansChapterExtractor(FoolslideChapterExtractor): class SensescansChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from sensescans.com""" """Extractor for manga-chapters from sensescans.com"""
category = "sensescans" category = "sensescans"
pattern = [(r"(?:https?://)?(?:www\.|reader\.)?sensescans\.com/" pattern = [(r"(?:https?://)?(?:www\.|reader\.)?sensescans\.com"
r"(?:reader/)?read/([^/]+/([a-z]{2})/\d+/\d+)")] r"(?:/reader)?(" + foolslide.CHAPTER_RE)]
test = [ test = [
(("http://reader.sensescans.com/read/" (("http://reader.sensescans.com/read/"
"magi__labyrinth_of_magic/en/33/319/page/1"), { "magi__labyrinth_of_magic/en/33/319/page/1"), {
"url": "cc192cbeed36127d374926c50c3a4bd06092b760", "url": "cc192cbeed36127d374926c50c3a4bd06092b760",
"keyword": "77f906f04bf49d3bd636e8c92d85dc25aa361754"}), "keyword": "f80104d205e3f537ca11fef9e761393c956768f0",
}),
(("http://sensescans.com/reader/read/" (("http://sensescans.com/reader/read/"
"magi__labyrinth_of_magic/en/33/319/page/1"), { "magi__labyrinth_of_magic/en/33/319/page/1"), {
"url": "cc192cbeed36127d374926c50c3a4bd06092b760", "url": "cc192cbeed36127d374926c50c3a4bd06092b760",
"keyword": "77f906f04bf49d3bd636e8c92d85dc25aa361754"}), "keyword": "f80104d205e3f537ca11fef9e761393c956768f0",
}),
] ]
def __init__(self, match): def __init__(self, match):
url = "http://sensescans.com/reader/read/" + match.group(1) url = "http://sensescans.com/reader" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2)) super().__init__(match, url)

31
gallery_dl/extractor/worldthree.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2016 Mike Fährmann # Copyright 2016-2017 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@@ -8,20 +8,23 @@
"""Extract manga-chapters from http://www.slide.world-three.org/""" """Extract manga-chapters from http://www.slide.world-three.org/"""
from .foolslide import FoolslideChapterExtractor from . import foolslide
class WorldthreeChapterExtractor(FoolslideChapterExtractor): class WorldthreeChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from slide.world-three.org""" """Extractor for manga-chapters from slide.world-three.org"""
category = "worldthree" category = "worldthree"
pattern = [(r"(?:https?://)?(?:www\.)?(slide\.world-three\.org/read/" pattern = foolslide.chapter_pattern("(?:www\.)?slide\.world-three\.org")
r"[^/]+/([a-z]{2})/\d+/\d+)")] test = [
test = [(("http://www.slide.world-three.org/" (("http://www.slide.world-three.org"
"read/black_bullet/en/2/7/page/1"), { "/read/black_bullet/en/2/7/page/1"), {
"url": "be2f04f6e2d311b35188094cfd3e768583271584", "url": "be2f04f6e2d311b35188094cfd3e768583271584",
"keyword": "6d77d9fc806d76d881491a52ccd8dfd875c47d05", "keyword": "25fd070bc93ee8ad316f5b7d1bd9011c9bcf0616",
})] }),
(("http://www.slide.world-three.org"
def __init__(self, match): "/read/idolmster_cg_shuffle/en/0/4/2/"), {
url = "http://" + match.group(1) "url": "6028ea5ca282744f925dfad92eeb98509f9cc78c",
FoolslideChapterExtractor.__init__(self, url, match.group(2)) "keyword": "10e3dc961ac2c9395f4d1f3ad3b9ad84113e7366",
}),
]
scheme = "http"

12
gallery_dl/extractor/yomanga.py
View File

@@ -8,20 +8,14 @@
"""Extract manga-chapters from https://yomanga.co/""" """Extract manga-chapters from https://yomanga.co/"""
from .foolslide import FoolslideChapterExtractor from . import foolslide
class YomangaChapterExtractor(FoolslideChapterExtractor): class YomangaChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from yomanga.co""" """Extractor for manga-chapters from yomanga.co"""
category = "yomanga" category = "yomanga"
pattern = [(r"(?:https?://)?(?:www\.)?(yomanga\.co/reader/read/" pattern = foolslide.chapter_pattern(r"(?:www\.)?yomanga\.co/reader")
r"[^/]+/([a-z]{2})/\d+/\d+)")]
test = [("https://yomanga.co/reader/read/uwakoi/en/0/2/", { test = [("https://yomanga.co/reader/read/uwakoi/en/0/2/", {
"url": "4b5d8fc5902f03647cc876cf6643849e5bc05455", "url": "4b5d8fc5902f03647cc876cf6643849e5bc05455",
"keyword": "1b9ac4217146421dbcb2a1108693054c56554a9d",
})] })]
single = False single = False
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

13
gallery_dl/extractor/yonkouprod.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright 2016, 2017 Mike Fährmann # Copyright 2016-2017 Mike Fährmann
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as # it under the terms of the GNU General Public License version 2 as
@@ -8,19 +8,14 @@
"""Extract manga-chapters from https://yonkouprod.com/""" """Extract manga-chapters from https://yonkouprod.com/"""
from .foolslide import FoolslideChapterExtractor from . import foolslide
class YonkouprodChapterExtractor(FoolslideChapterExtractor): class YonkouprodChapterExtractor(foolslide.FoolslideChapterExtractor):
"""Extractor for manga-chapters from yonkouprod.com""" """Extractor for manga-chapters from yonkouprod.com"""
category = "yonkouprod" category = "yonkouprod"
pattern = [(r"(?:https?://)?(?:www\.)?(yonkouprod\.com/reader/read/" pattern = foolslide.chapter_pattern(r"(?:www\.)?yonkouprod\.com/reader")
r"[^/]+/([a-z]{2})/\d+/\d+)")]
test = [("http://yonkouprod.com/reader/read/fairy-tail/en/0/512/", { test = [("http://yonkouprod.com/reader/read/fairy-tail/en/0/512/", {
"url": "7647850e2b1ad11c2baa9628755bf7f186350a0b", "url": "7647850e2b1ad11c2baa9628755bf7f186350a0b",
"keyword": "d079c718d6620478fa72a700fdd027f9a0f0760b", "keyword": "d079c718d6620478fa72a700fdd027f9a0f0760b",
})] })]
def __init__(self, match):
url = "https://" + match.group(1)
FoolslideChapterExtractor.__init__(self, url, match.group(2))

2
test/test_extractors.py
View File

@@ -48,7 +48,7 @@ def _generate_test(extr, tcase):
# enable selective testing for direct calls # enable selective testing for direct calls
skip = ["exhentai", "kissmanga", "mangafox"] skip = ["exhentai", "kissmanga", "mangafox", "yomanga"]
if __name__ == '__main__' and len(sys.argv) > 1: if __name__ == '__main__' and len(sys.argv) > 1:
extractors = [ extractors = [
extr for extr in extractor.extractors() extr for extr in extractor.extractors()