github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[idolcomplex] update to site layout changes

Browse Source
This commit is contained in:
Mike Fährmann
2023-11-27 18:27:08 +01:00
parent 95c1dfb089
commit 013ca21543
1 changed files with 74 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
gallery_dl/extractor/idolcomplex.py
View File

@@ -15,15 +15,17 @@ from .. import text, util, exception
import collections import collections
import re import re
BASE_PATTERN = r"(?:https?://)?idol\.sankakucomplex\.com(?:/[a-z]{2})?"
class IdolcomplexExtractor(SankakuExtractor): class IdolcomplexExtractor(SankakuExtractor):
"""Base class for idolcomplex extractors""" """Base class for idolcomplex extractors"""
category = "idolcomplex" category = "idolcomplex"
root = "https://idol.sankakucomplex.com"
cookies_domain = "idol.sankakucomplex.com" cookies_domain = "idol.sankakucomplex.com"
cookies_names = ("login", "pass_hash") cookies_names = ("_idolcomplex_session",)
root = "https://" + cookies_domain
referer = False referer = False
request_interval = 5.0 request_interval = (4.0, 6.0)
def __init__(self, match): def __init__(self, match):
SankakuExtractor.__init__(self, match) SankakuExtractor.__init__(self, match)
@@ -32,14 +34,16 @@ class IdolcomplexExtractor(SankakuExtractor):
self.start_post = 0 self.start_post = 0
def _init(self): def _init(self):
self.extags = self.config("tags", False) self.find_tags = re.compile(
r'tag-type-([^"]+)">\s*<div [^>]+>\s*<a href="/\?tags=([^"]+)'
).findall
def items(self): def items(self):
self.login() self.login()
data = self.metadata() data = self.metadata()
for post_id in util.advance(self.post_ids(), self.start_post): for post_id in util.advance(self.post_ids(), self.start_post):
post = self._parse_post(post_id) post = self._extract_post(post_id)
url = post["file_url"] url = post["file_url"]
post.update(data) post.update(data)
text.nameext_from_url(url, post) text.nameext_from_url(url, post)
@@ -67,63 +71,75 @@ class IdolcomplexExtractor(SankakuExtractor):
def _login_impl(self, username, password): def _login_impl(self, username, password):
self.log.info("Logging in as %s", username) self.log.info("Logging in as %s", username)
url = self.root + "/user/authenticate" url = self.root + "/users/login"
page = self.request(url).text
headers = {
"Referer": url,
}
url = self.root + (text.extr(page, '<form action="', '"') or
"/en/user/authenticate")
data = { data = {
"authenticity_token": text.unescape(text.extr(
page, 'name="authenticity_token" value="', '"')),
"url" : "", "url" : "",
"user[name]" : username, "user[name]" : username,
"user[password]": password, "user[password]": password,
"commit" : "Login", "commit" : "Login",
} }
response = self.request(url, method="POST", data=data) response = self.request(url, method="POST", headers=headers, data=data)
if not response.history or response.url != self.root + "/user/home": if not response.history or response.url.endswith("/user/home"):
raise exception.AuthenticationError() raise exception.AuthenticationError()
cookies = response.history[0].cookies return {c.name: c.value for c in response.history[0].cookies}
return {c: cookies[c] for c in self.cookies_names}
def _parse_post(self, post_id): def _extract_post(self, post_id):
"""Extract metadata of a single post""" url = self.root + "/posts/" + post_id
url = self.root + "/post/show/" + post_id
page = self.request(url, retries=10).text page = self.request(url, retries=10).text
extr = text.extract extr = text.extract_from(page)
tags , pos = extr(page, "<title>", " | ") tags = extr("<title>", " | ")
vavg , pos = extr(page, "itemprop=ratingValue>", "<", pos) vavg = extr('itemprop="ratingValue">', "<")
vcnt , pos = extr(page, "itemprop=reviewCount>", "<", pos) vcnt = extr('itemprop="reviewCount">', "<")
_ , pos = extr(page, "Posted: <", "", pos) pid = extr(">Post ID:", "<")
created, pos = extr(page, ' title="', '"', pos) created = extr(' title="', '"')
rating = extr(page, "<li>Rating: ", "<", pos)[0]
file_url, pos = extr(page, '<li>Original: <a href="', '"', pos) file_url = extr('>Original:', 'id=')
if file_url: if file_url:
width , pos = extr(page, '>', 'x', pos) file_url = extr(' href="', '"')
height, pos = extr(page, '', ' ', pos) width = extr(">", "x")
height = extr("", " ")
else: else:
width , pos = extr(page, '<object width=', ' ', pos) width = extr('<object width=', ' ')
height, pos = extr(page, 'height=', '>', pos) height = extr('height=', '>')
file_url = extr(page, '<embed src="', '"', pos)[0] file_url = extr('<embed src="', '"')
rating = extr(">Rating:", "<br")
data = { data = {
"id": text.parse_int(post_id), "id" : text.parse_int(pid),
"md5": file_url.rpartition("/")[2].partition(".")[0], "md5" : file_url.rpartition("/")[2].partition(".")[0],
"tags": text.unescape(tags), "tags" : text.unescape(tags),
"vote_average": text.parse_float(vavg), "vote_average": text.parse_float(vavg),
"vote_count": text.parse_int(vcnt), "vote_count" : text.parse_int(vcnt),
"created_at": created, "created_at" : created,
"rating": (rating or "?")[0].lower(), "date" : text.parse_datetime(
"file_url": "https:" + text.unescape(file_url), created, "%Y-%m-%d %H:%M:%S.%f"),
"width": text.parse_int(width), "rating" : text.remove_html(rating).lower(),
"height": text.parse_int(height), "file_url" : "https:" + text.unescape(file_url),
"width" : text.parse_int(width),
"height" : text.parse_int(height),
} }
if self.extags: tags = collections.defaultdict(list)
tags = collections.defaultdict(list) tags_list = []
tags_html = text.extr(page, '<ul id=tag-sidebar>', '</ul>') tags_html = text.extr(page, '<ul id="tag-sidebar"', '</ul>')
pattern = re.compile(r'tag-type-([^>]+)><a href="/\?tags=([^"]+)') for tag_type, tag_name in self.find_tags(tags_html or ""):
for tag_type, tag_name in pattern.findall(tags_html or ""): tags[tag_type].append(text.unquote(tag_name))
tags[tag_type].append(text.unquote(tag_name)) for key, value in tags.items():
for key, value in tags.items(): data["tags_" + key] = " ".join(value)
data["tags_" + key] = " ".join(value) tags_list += value
data["tags"] = " ".join(tags_list)
return data return data
@@ -178,15 +194,16 @@ class IdolcomplexTagExtractor(IdolcomplexExtractor):
while True: while True:
page = self.request(self.root, params=params, retries=10).text page = self.request(self.root, params=params, retries=10).text
pos = page.find("<div id=more-popular-posts-link>") + 1 pos = ((page.find('id="more-popular-posts-link"') + 1) or
yield from text.extract_iter(page, '" id=p', '>', pos) (page.find('<span class="thumb') + 1))
yield from text.extract_iter(page, ' href="/posts/', '"', pos)
next_url = text.extract(page, 'next-page-url="', '"', pos)[0] next_url = text.extract(page, 'next-page-url="', '"', pos)[0]
if not next_url: if not next_url:
return return
next_params = text.parse_query(text.unescape( next_params = text.parse_query(text.unescape(text.unescape(
next_url).lstrip("?/")) next_url).lstrip("?/")))
if "next" in next_params: if "next" in next_params:
# stop if the same "next" value occurs twice in a row (#265) # stop if the same "next" value occurs twice in a row (#265)
@@ -201,8 +218,8 @@ class IdolcomplexPoolExtractor(IdolcomplexExtractor):
subcategory = "pool" subcategory = "pool"
directory_fmt = ("{category}", "pool", "{pool}") directory_fmt = ("{category}", "pool", "{pool}")
archive_fmt = "p_{pool}_{id}" archive_fmt = "p_{pool}_{id}"
pattern = r"(?:https?://)?idol\.sankakucomplex\.com/pool/show/(\d+)" pattern = r"(?:https?://)?idol\.sankakucomplex\.com/pools?/show/(\d+)"
example = "https://idol.sankakucomplex.com/pool/show/12345" example = "https://idol.sankakucomplex.com/pools/show/12345"
per_page = 24 per_page = 24
def __init__(self, match): def __init__(self, match):
@@ -219,15 +236,17 @@ class IdolcomplexPoolExtractor(IdolcomplexExtractor):
return {"pool": self.pool_id} return {"pool": self.pool_id}
def post_ids(self): def post_ids(self):
url = self.root + "/pool/show/" + self.pool_id url = self.root + "/pools/show/" + self.pool_id
params = {"page": self.start_page} params = {"page": self.start_page}
while True: while True:
page = self.request(url, params=params, retries=10).text page = self.request(url, params=params, retries=10).text
ids = list(text.extract_iter(page, '" id=p', '>')) pos = page.find('id="pool-show"') + 1
post_ids = list(text.extract_iter(
page, ' href="/posts/', '"', pos))
yield from ids yield from post_ids
if len(ids) < self.per_page: if len(post_ids) < self.per_page:
return return
params["page"] += 1 params["page"] += 1
@@ -236,8 +255,8 @@ class IdolcomplexPostExtractor(IdolcomplexExtractor):
"""Extractor for single images from idol.sankakucomplex.com""" """Extractor for single images from idol.sankakucomplex.com"""
subcategory = "post" subcategory = "post"
archive_fmt = "{id}" archive_fmt = "{id}"
pattern = r"(?:https?://)?idol\.sankakucomplex\.com/post/show/(\d+)" pattern = BASE_PATTERN + r"/posts?/(?:show/)?([0-9a-f]+)"
example = "https://idol.sankakucomplex.com/post/show/12345" example = "https://idol.sankakucomplex.com/posts/0123456789abcdef"
def __init__(self, match): def __init__(self, match):
IdolcomplexExtractor.__init__(self, match) IdolcomplexExtractor.__init__(self, match)