github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[hentaifoundry] update and simplify

Browse Source
This commit is contained in:
Mike Fährmann
2020-10-12 21:15:07 +02:00
parent 5b844a72b7
commit 783e0af26d
1 changed files with 97 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
gallery_dl/extractor/hentaifoundry.py
View File

@@ -9,7 +9,9 @@
"""Extractors for https://www.hentai-foundry.com/""" """Extractors for https://www.hentai-foundry.com/"""
from .common import Extractor, Message from .common import Extractor, Message
from .. import text, util, exception from .. import text, util
BASE_PATTERN = r"(?:https?://)?(?:www\.)?hentai-foundry\.com"
class HentaifoundryExtractor(Extractor): class HentaifoundryExtractor(Extractor):
@@ -21,22 +23,21 @@ class HentaifoundryExtractor(Extractor):
root = "https://www.hentai-foundry.com" root = "https://www.hentai-foundry.com"
per_page = 25 per_page = 25
def __init__(self, match, user="", page=1): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
self.user = match.group(1)
self.page_url = "" self.page_url = ""
self.user = user
self.start_post = 0 self.start_post = 0
self.start_page = text.parse_int(page, 1) self.start_page = 1
def items(self): def items(self):
data = self.get_job_metadata() self._init_site_filters()
yield Message.Version, 1 data = self.metadata()
yield Message.Directory, data
self.set_filters() for post_url in util.advance(self.posts(), self.start_post):
for page_url in util.advance(self._pagination(), self.start_post): image = self._parse_post(post_url)
image = self.get_image_metadata(page_url)
image.update(data) image.update(data)
yield Message.Directory, image
yield Message.Url, image["src"], image yield Message.Url, image["src"], image
def skip(self, num): def skip(self, num):
@@ -45,24 +46,25 @@ class HentaifoundryExtractor(Extractor):
self.start_post += posts self.start_post += posts
return num return num
def get_job_metadata(self): def metadata(self):
"""Collect metadata for extractor-job"""
self.request(self.root + "/?enterAgree=1")
return {"user": self.user} return {"user": self.user}
def _pagination(self, begin='thumbTitle"><a href="', end='"'): def posts(self):
return self._pagination(self.page_url)
def _pagination(self, url, begin='thumbTitle"><a href="', end='"'):
num = self.start_page num = self.start_page
while True: while True:
page = self.request("{}/page/{}".format(self.page_url, num)).text page = self.request("{}/page/{}".format(url, num)).text
yield from text.extract_iter(page, begin, end) yield from text.extract_iter(page, begin, end)
if 'class="pager"' not in page or 'class="last hidden"' in page: if 'class="pager"' not in page or 'class="last hidden"' in page:
return return
num += 1 num += 1
def get_image_metadata(self, path): def _parse_post(self, path):
"""Collect url and metadata from an image page""" """Collect url and metadata from an image post"""
url = text.urljoin(self.root, path) url = text.urljoin(self.root, path)
page = self.request(url).text page = self.request(url).text
extr = text.extract_from(page, page.index('id="picBox"')) extr = text.extract_from(page, page.index('id="picBox"'))
@@ -89,7 +91,7 @@ class HentaifoundryExtractor(Extractor):
return text.nameext_from_url(data["src"], data) return text.nameext_from_url(data["src"], data)
def get_story_metadata(self, html): def _parse_story(self, html):
"""Collect url and metadata for a story""" """Collect url and metadata for a story"""
extr = text.extract_from(html) extr = text.extract_from(html)
data = { data = {
@@ -116,68 +118,66 @@ class HentaifoundryExtractor(Extractor):
return text.nameext_from_url(data["src"], data) return text.nameext_from_url(data["src"], data)
def set_filters(self): def _init_site_filters(self):
"""Set site-internal filters to show all images""" """Set site-internal filters to show all images"""
token = text.unquote(text.extract( url = self.root + "/?enterAgree=1"
self.session.cookies["YII_CSRF_TOKEN"], "%22", "%22")[0]) response = self.request(url, method="HEAD")
data = {
"YII_CSRF_TOKEN": token,
"rating_nudity": 3,
"rating_violence": 3,
"rating_profanity": 3,
"rating_racism": 3,
"rating_sex": 3,
"rating_spoilers": 3,
"rating_yaoi": 1,
"rating_yuri": 1,
"rating_teen": 1,
"rating_guro": 1,
"rating_furry": 1,
"rating_beast": 1,
"rating_male": 1,
"rating_female": 1,
"rating_futa": 1,
"rating_other": 1,
"rating_scat": 1,
"rating_incest": 1,
"rating_rape": 1,
"filter_media": "A",
"filter_order": "date_new",
"filter_type": 0,
}
url = self.root + "/site/filters" url = self.root + "/site/filters"
data = {
"rating_nudity" : "3",
"rating_violence" : "3",
"rating_profanity": "3",
"rating_racism" : "3",
"rating_sex" : "3",
"rating_spoilers" : "3",
"rating_yaoi" : "1",
"rating_yuri" : "1",
"rating_teen" : "1",
"rating_guro" : "1",
"rating_furry" : "1",
"rating_beast" : "1",
"rating_male" : "1",
"rating_female" : "1",
"rating_futa" : "1",
"rating_other" : "1",
"rating_scat" : "1",
"rating_incest" : "1",
"rating_rape" : "1",
"filter_media" : "A",
"filter_order" : "date_new",
"filter_type" : "0",
"YII_CSRF_TOKEN" : text.unquote(text.extract(
response.cookies["YII_CSRF_TOKEN"], "%22", "%22")[0]),
}
self.request(url, method="POST", data=data) self.request(url, method="POST", data=data)
class HentaifoundryUserExtractor(HentaifoundryExtractor): class HentaifoundryUserExtractor(HentaifoundryExtractor):
"""Extractor for all images of a hentai-foundry-user""" """Extractor for a hentaifoundry user profile"""
subcategory = "user" subcategory = "user"
pattern = (r"(?:https?://)?(?:www\.)?hentai-foundry\.com" pattern = BASE_PATTERN + r"/user/([^/?&#]+)/profile"
r"/user/([^/]+)/profile")
test = ("https://www.hentai-foundry.com/user/Tenpura/profile",) test = ("https://www.hentai-foundry.com/user/Tenpura/profile",)
def __init__(self, match):
HentaifoundryExtractor.__init__(self, match, match.group(1))
def items(self): def items(self):
root = self.root
user = "/user/" + self.user user = "/user/" + self.user
return self._dispatch_extractors(( return self._dispatch_extractors((
(HentaifoundryPicturesExtractor , (HentaifoundryPicturesExtractor ,
self.root + "/pictures" + user), root + "/pictures" + user),
(HentaifoundryScrapsExtractor, (HentaifoundryScrapsExtractor,
self.root + "/pictures" + user + "/scraps"), root + "/pictures" + user + "/scraps"),
(HentaifoundryStoriesExtractor, (HentaifoundryStoriesExtractor,
self.root + "/stories" + user), root + "/stories" + user),
(HentaifoundryFavoriteExtractor, (HentaifoundryFavoriteExtractor,
self.root + user + "/faves/pictures"), root + user + "/faves/pictures"),
), ("pictures",)) ), ("pictures",))
class HentaifoundryPicturesExtractor(HentaifoundryExtractor): class HentaifoundryPicturesExtractor(HentaifoundryExtractor):
"""Extractor for all pictures of a hentaifoundry user""" """Extractor for all pictures of a hentaifoundry user"""
subcategory = "pictures" subcategory = "pictures"
pattern = (r"(?:https?://)?(?:www\.)?hentai-foundry\.com" pattern = BASE_PATTERN + r"/pictures/user/([^/?&#]+)(?:/page/(\d+))?/?$"
r"/pictures/user/([^/]+)(?:/page/(\d+))?/?$")
test = ( test = (
("https://www.hentai-foundry.com/pictures/user/Tenpura", { ("https://www.hentai-foundry.com/pictures/user/Tenpura", {
"url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28", "url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28",
@@ -186,22 +186,15 @@ class HentaifoundryPicturesExtractor(HentaifoundryExtractor):
) )
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__( HentaifoundryExtractor.__init__(self, match)
self, match, match.group(1), match.group(2))
self.page_url = "{}/pictures/user/{}".format(self.root, self.user) self.page_url = "{}/pictures/user/{}".format(self.root, self.user)
def get_job_metadata(self):
page = self.request(self.page_url + "?enterAgree=1").text
count = text.extract(page, ">Pictures (", ")")[0]
return {"user": self.user, "count": text.parse_int(count)}
class HentaifoundryScrapsExtractor(HentaifoundryExtractor): class HentaifoundryScrapsExtractor(HentaifoundryExtractor):
"""Extractor for scrap images of a hentai-foundry-user""" """Extractor for scraps of a hentaifoundry user"""
subcategory = "scraps" subcategory = "scraps"
directory_fmt = ("{category}", "{user}", "Scraps") directory_fmt = ("{category}", "{user}", "Scraps")
pattern = (r"(?:https?://)?(?:www\.)?hentai-foundry\.com" pattern = BASE_PATTERN + r"/pictures/user/([^/?&#]+)/scraps"
r"/pictures/user/([^/]+)/scraps(?:/page/(\d+))?")
test = ( test = (
("https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps", { ("https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps", {
"url": "7cd9c6ec6258c4ab8c44991f7731be82337492a7", "url": "7cd9c6ec6258c4ab8c44991f7731be82337492a7",
@@ -211,24 +204,17 @@ class HentaifoundryScrapsExtractor(HentaifoundryExtractor):
) )
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__( HentaifoundryExtractor.__init__(self, match)
self, match, match.group(1), match.group(2))
self.page_url = "{}/pictures/user/{}/scraps".format( self.page_url = "{}/pictures/user/{}/scraps".format(
self.root, self.user) self.root, self.user)
def get_job_metadata(self):
page = self.request(self.page_url + "?enterAgree=1").text
count = text.extract(page, ">Scraps (", ")")[0]
return {"user": self.user, "count": text.parse_int(count)}
class HentaifoundryFavoriteExtractor(HentaifoundryExtractor): class HentaifoundryFavoriteExtractor(HentaifoundryExtractor):
"""Extractor for favorite images of a hentai-foundry-user""" """Extractor for favorite images of a hentaifoundry user"""
subcategory = "favorite" subcategory = "favorite"
directory_fmt = ("{category}", "{user}", "Favorites") directory_fmt = ("{category}", "{user}", "Favorites")
archive_fmt = "f_{user}_{index}" archive_fmt = "f_{user}_{index}"
pattern = (r"(?:https?://)?(?:www\.)?hentai-foundry\.com" pattern = BASE_PATTERN + r"/user/([^/?&#]+)/faves/pictures"
r"/user/([^/]+)/faves/pictures(?:/page/(\d+))?")
test = ( test = (
("https://www.hentai-foundry.com/user/Tenpura/faves/pictures", { ("https://www.hentai-foundry.com/user/Tenpura/faves/pictures", {
"url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b", "url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b",
@@ -238,8 +224,7 @@ class HentaifoundryFavoriteExtractor(HentaifoundryExtractor):
) )
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__( HentaifoundryExtractor.__init__(self, match)
self, match, match.group(1), match.group(2))
self.page_url = "{}/user/{}/faves/pictures".format( self.page_url = "{}/user/{}/faves/pictures".format(
self.root, self.user) self.root, self.user)
@@ -249,21 +234,18 @@ class HentaifoundryRecentExtractor(HentaifoundryExtractor):
subcategory = "recent" subcategory = "recent"
directory_fmt = ("{category}", "Recent Pictures", "{date}") directory_fmt = ("{category}", "Recent Pictures", "{date}")
archive_fmt = "r_{index}" archive_fmt = "r_{index}"
pattern = (r"(?:https?://)?(?:www\.)?hentai-foundry\.com" pattern = BASE_PATTERN + r"/pictures/recent/(\d\d\d\d-\d\d-\d\d)"
r"/pictures/recent/(\d+-\d+-\d+)(?:/page/(\d+))?")
test = ("http://www.hentai-foundry.com/pictures/recent/2018-09-20", { test = ("http://www.hentai-foundry.com/pictures/recent/2018-09-20", {
"pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/]+/\d+/", "pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?&#]+/\d+/",
"range": "20-30", "range": "20-30",
}) })
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__(self, match, "", match.group(2)) HentaifoundryExtractor.__init__(self, match)
self.date = match.group(1) self.page_url = "{}/pictures/recent/{}".format(self.root, self.user)
self.page_url = "{}/pictures/recent/{}".format(self.root, self.date)
def get_job_metadata(self): def metadata(self):
self.request(self.root + "/?enterAgree=1") return {"date": self.user}
return {"date": self.date}
class HentaifoundryPopularExtractor(HentaifoundryExtractor): class HentaifoundryPopularExtractor(HentaifoundryExtractor):
@@ -271,15 +253,14 @@ class HentaifoundryPopularExtractor(HentaifoundryExtractor):
subcategory = "popular" subcategory = "popular"
directory_fmt = ("{category}", "Popular Pictures") directory_fmt = ("{category}", "Popular Pictures")
archive_fmt = "p_{index}" archive_fmt = "p_{index}"
pattern = (r"(?:https?://)?(?:www\.)?hentai-foundry\.com" pattern = BASE_PATTERN + r"/pictures/popular()"
r"/pictures/popular(?:/page/(\d+))?")
test = ("http://www.hentai-foundry.com/pictures/popular", { test = ("http://www.hentai-foundry.com/pictures/popular", {
"pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/]+/\d+/", "pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?&#]+/\d+/",
"range": "20-30", "range": "20-30",
}) })
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__(self, match, "", match.group(1)) HentaifoundryExtractor.__init__(self, match)
self.page_url = self.root + "/pictures/popular" self.page_url = self.root + "/pictures/popular"
@@ -287,7 +268,7 @@ class HentaifoundryImageExtractor(HentaifoundryExtractor):
"""Extractor for a single image from hentaifoundry.com""" """Extractor for a single image from hentaifoundry.com"""
subcategory = "image" subcategory = "image"
pattern = (r"(?:https?://)?(?:www\.|pictures\.)?hentai-foundry\.com" pattern = (r"(?:https?://)?(?:www\.|pictures\.)?hentai-foundry\.com"
r"/(?:pictures/user|[^/])/([^/]+)/(\d+)") r"/(?:pictures/user|[^/?&#])/([^/?&#]+)/(\d+)")
test = ( test = (
(("https://www.hentai-foundry.com" (("https://www.hentai-foundry.com"
"/pictures/user/Tenpura/407501/shimakaze"), { "/pictures/user/Tenpura/407501/shimakaze"), {
@@ -309,36 +290,30 @@ class HentaifoundryImageExtractor(HentaifoundryExtractor):
"width" : 495, "width" : 495,
}, },
}), }),
("https://www.hentai-foundry.com/pictures/user/Tenpura/340853/", { ("https://www.hentai-foundry.com/pictures/user/Tenpura/340853/"),
"exception": exception.HttpError,
}),
("https://pictures.hentai-foundry.com" ("https://pictures.hentai-foundry.com"
"/t/Tenpura/407501/Tenpura-407501-shimakaze.png"), "/t/Tenpura/407501/Tenpura-407501-shimakaze.png"),
) )
skip = Extractor.skip
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__(self, match, match.group(1)) HentaifoundryExtractor.__init__(self, match)
self.index = match.group(2) self.index = match.group(2)
def items(self): def items(self):
post_url = "{}/pictures/user/{}/{}/?enterAgree=1".format( post_url = "{}/pictures/user/{}/{}/?enterAgree=1".format(
self.root, self.user, self.index) self.root, self.user, self.index)
data = self.get_image_metadata(post_url) image = self._parse_post(post_url)
data["user"] = self.user image["user"] = self.user
yield Message.Directory, image
yield Message.Version, 1 yield Message.Url, image["src"], image
yield Message.Directory, data
yield Message.Url, data["src"], data
def skip(self, _):
return 0
class HentaifoundryStoriesExtractor(HentaifoundryExtractor): class HentaifoundryStoriesExtractor(HentaifoundryExtractor):
"""Extractor for stories of a hentai-foundry user""" """Extractor for stories of a hentaifoundry user"""
subcategory = "stories" subcategory = "stories"
pattern = (r"(?:https?://)?(?:www\.)?hentai-foundry\.com" archive_fmt = "s_{index}"
r"/stories/user/([^/]+)(?:/page/(\d+))?/?$") pattern = BASE_PATTERN + r"/stories/user/([^/?&#]+)(?:/page/(\d+))?/?$"
test = ("https://www.hentai-foundry.com/stories/user/SnowWolf35", { test = ("https://www.hentai-foundry.com/stories/user/SnowWolf35", {
"count": ">= 35", "count": ">= 35",
"keyword": { "keyword": {
@@ -358,42 +333,37 @@ class HentaifoundryStoriesExtractor(HentaifoundryExtractor):
}, },
}) })
def __init__(self, match):
HentaifoundryExtractor.__init__(self, match, match.group(1))
self.page_url = "{}/stories/user/{}".format(self.root, self.user)
def items(self): def items(self):
self.get_job_metadata() self._init_site_filters()
self.set_filters() for story_html in util.advance(self.stories(), self.start_post):
stories = self._pagination('<div class="storyRow">', '</tr></table>') story = self._parse_story(story_html)
for story_html in util.advance(stories, self.start_post):
story = self.get_story_metadata(story_html)
yield Message.Directory, story yield Message.Directory, story
yield Message.Url, story["src"], story yield Message.Url, story["src"], story
def stories(self):
url = "{}/stories/user/{}".format(self.root, self.user)
return self._pagination(url, '<div class="storyRow">', '</tr></table>')
class HentaifoundryStoryExtractor(HentaifoundryExtractor): class HentaifoundryStoryExtractor(HentaifoundryExtractor):
"""Extractor for a hentaifoundry story""" """Extractor for a hentaifoundry story"""
subcategory = "story" subcategory = "story"
pattern = (r"(?:https?://)?(?:www\.)?hentai-foundry\.com" archive_fmt = "s_{index}"
r"/stories/user/([^/]+)/(\d+)") pattern = BASE_PATTERN + r"/stories/user/([^/?&#]+)/(\d+)"
test = (("https://www.hentai-foundry.com/stories/user/SnowWolf35" test = (("https://www.hentai-foundry.com/stories/user/SnowWolf35"
"/26416/Overwatch-High-Chapter-Voting-Location"), { "/26416/Overwatch-High-Chapter-Voting-Location"), {
"url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8", "url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8",
"keyword": {"title": "Overwatch High Chapter Voting Location"}, "keyword": {"title": "Overwatch High Chapter Voting Location"},
}) })
skip = Extractor.skip
def __init__(self, match): def __init__(self, match):
HentaifoundryExtractor.__init__(self, match, match.group(1)) HentaifoundryExtractor.__init__(self, match)
self.index = match.group(2) self.index = match.group(2)
def items(self): def items(self):
story_url = "{}/stories/user/{}/{}/x?enterAgree=1".format( story_url = "{}/stories/user/{}/{}/x?enterAgree=1".format(
self.root, self.user, self.index) self.root, self.user, self.index)
page = self.request(story_url).text story = self._parse_story(self.request(story_url).text)
story = self.get_story_metadata(page)
yield Message.Directory, story yield Message.Directory, story
yield Message.Url, story["src"], story yield Message.Url, story["src"], story
def skip(self, _):
return 0