github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[4chan] more metadata entries (size, width, height)

Browse Source
This commit is contained in:
Mike Fährmann
2015-05-04 14:57:47 +02:00
parent f82cd5c7f7
commit 5993177ae3
1 changed files with 19 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
gallery_dl/extractor/4chan.py
View File

@@ -18,51 +18,55 @@ info = {
"directory": ["{category}", "{board}-{thread-id}"], "directory": ["{category}", "{board}-{thread-id}"],
"filename": "{timestamp}-{name}", "filename": "{timestamp}-{name}",
"pattern": [ "pattern": [
r"(?:https?://)?boards\.4chan\.org/([^/]+)/thread/(\d+)(?:/([^#]*))?", r"(?:https?://)?boards\.4chan\.org/([^/]+)/thread/(\d+).*",
], ],
} }
class FourChanExtractor(SequentialExtractor): class FourChanExtractor(SequentialExtractor):
url_fmt = "https://boards.4chan.org/{board}/res/{thread-id}.html" url_fmt = "https://boards.4chan.org/{0}/res/{1}.html"
regex = ( regex = (
r'<a href="(//i.4cdn.org/[^/]+/(\d+)\.([^"]+))"' r'<a (?:title="(?P<orig_name>[^"]+)" )?href="'
r' target="_blank">([^<]+)</a>' r'(?P<url>//i.4cdn.org/[^/]+/(?P<timestamp>\d+)\.(?P<extension>[^"]+))'
r'" target="_blank">(?P<name>[^<]+)</a> '
r'\((?P<size>[^,]+), (?P<width>\d+)x(?P<height>\d+)\)'
) )
def __init__(self, match, config): def __init__(self, match, config):
SequentialExtractor.__init__(self, config) SequentialExtractor.__init__(self, config)
self.metadata = self.get_job_metadata(match) self.match = match
self.metadata = None
def items(self): def items(self):
yield Message.Version, 1 yield Message.Version, 1
yield Message.Directory, self.metadata
url = self.url_fmt.format(**self.metadata) url = self.url_fmt.format(*self.match.groups())
text = self.request(url).text text = self.request(url).text
self.metadata = self.get_job_metadata(text)
yield Message.Directory, self.metadata
for match in re.finditer(self.regex, text): for match in re.finditer(self.regex, text):
yield Message.Url, self.get_file_url(match), self.get_file_metadata(match) yield Message.Url, self.get_file_url(match), self.get_file_metadata(match)
@staticmethod def get_job_metadata(self, text):
def get_job_metadata(match):
"""Collect metadata for extractor-job""" """Collect metadata for extractor-job"""
board, thread_id, title = match.groups() board, thread_id = self.match.groups()
title, _ = self.extract(text, '"description" content="', ' - &quot;/')
return { return {
"category": info["category"], "category": info["category"],
"board": board, "board": board,
"thread-id": thread_id, "thread-id": thread_id,
"title": title, "title": unquote(title),
} }
def get_file_metadata(self, match): def get_file_metadata(self, match):
"""Collect metadata for a downloadable file""" """Collect metadata for a downloadable file"""
data = self.metadata data = self.metadata
data["timestamp"] = match.group(2) data.update(match.groupdict(default=""))
data["extension"] = match.group(3) data["name"] = unquote(data["orig_name"] or data["name"])
data["name"] = unquote(match.group(4))
return data return data
@staticmethod @staticmethod
def get_file_url(match): def get_file_url(match):
"""Extract download-url from 'match'""" """Extract download-url from 'match'"""
return "https:" + match.group(1) return "https:" + match.group("url")