github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[sankaku] always use correct file-url

Browse Source
This commit is contained in:
Mike Fährmann
2015-11-10 00:55:01 +01:00
parent 3560935318
commit 3510faaed5
1 changed files with 27 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
gallery_dl/extractor/sankaku.py
View File

@@ -8,7 +8,7 @@
"""Extract images from https://chan.sankakucomplex.com/""" """Extract images from https://chan.sankakucomplex.com/"""
from .common import Extractor, Message from .common import AsynchronousExtractor, Message
from .. import text from .. import text
import os.path import os.path
@@ -22,24 +22,25 @@ info = {
], ],
} }
class SankakuExtractor(Extractor): class SankakuExtractor(AsynchronousExtractor):
url = "https://chan.sankakucomplex.com/" url = "https://chan.sankakucomplex.com/"
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self) AsynchronousExtractor.__init__(self)
self.tags = text.unquote(match.group(1)) self.tags = text.unquote(match.group(1))
self.session.headers["User-Agent"] = ( self.session.headers["User-Agent"] = (
"Mozilla/5.0 Gecko/20100101 Firefox/40.0" "Mozilla/5.0 Gecko/20100101 Firefox/40.0"
) )
def items(self): def items(self):
yield Message.Version, 1
data = self.get_job_metadata() data = self.get_job_metadata()
yield Message.Version, 1
yield Message.Headers, self.session.headers
yield Message.Directory, data yield Message.Directory, data
for image in self.get_images(): for image in self.get_images():
data.update(image) image.update(data)
yield Message.Url, image["file-url"], data yield Message.Url, image["file-url"], image
def get_job_metadata(self): def get_job_metadata(self):
"""Collect metadata for extractor-job""" """Collect metadata for extractor-job"""
@@ -49,33 +50,40 @@ class SankakuExtractor(Extractor):
} }
def get_images(self): def get_images(self):
image = {}
params = { params = {
"tags": self.tags, "tags": self.tags,
"page": 1, "page": 1,
} }
while True: while True:
pos = 0
count = 0 count = 0
page = self.request(self.url, params=params).text page = self.request(self.url, params=params).text
pos = text.extract(page, '<div id=more-popular-posts-link>', '')[1]
while True: while True:
image["id"], pos = text.extract(page, image_id, pos = text.extract(page,
'<span class="thumb blacklisted" id=p', '>', pos) '<span class="thumb blacklisted" id=p', '>', pos)
if not image["id"]: if not image_id:
break break
url , pos = text.extract(page, ' src="//c.sankakucomplex.com/', '"', pos) image = self.get_image_metadata(image_id)
tags, pos = text.extract(page, ' title="', '"', pos)
self.get_image_metadata(image, url)
count += 1 count += 1
yield image yield image
if count < 20: if count < 20:
return return
params["page"] += 1 params["page"] += 1
@staticmethod def get_image_metadata(self, image_id):
def get_image_metadata(image, url): url = "https://chan.sankakucomplex.com/post/show/" + image_id
image["file-url"] = "https://cs.sankakucomplex.com/data/" + url[13:] page = self.request(url).text
filename = text.filename_from_url(url) image_url, pos = text.extract(page, '<li>Original: <a href="', '"')
width , pos = text.extract(page, '>', 'x', pos)
height , pos = text.extract(page, '', ' ', pos)
filename = text.filename_from_url(image_url)
name, ext = os.path.splitext(filename) name, ext = os.path.splitext(filename)
image["name"] = image["md5"] = name return {
image["extension"] = ext[1:] "id": image_id,
"file-url": "https:" + image_url,
"width": width,
"height": height,
"md5": name,
"name": name,
"extension": ext[1:],
}