github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into 1.4-dev

Browse Source
This commit is contained in:
Mike Fährmann
2018-04-21 12:15:40 +02:00
parent cc36f88586 a2020c736e
commit f471161920
17 changed files with 278 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
gallery_dl/extractor/__init__.py
View File

@@ -77,7 +77,6 @@ modules = [
"senmanga",
"sensescans",
"slideshare",
"spectrumnexus",
"subapics",
"thebarchive",
"tumblr",

2
gallery_dl/extractor/artstation.py
View File

@@ -137,7 +137,7 @@ class ArtstationUserExtractor(ArtstationExtractor):
class ArtstationAlbumExtractor(ArtstationExtractor):
"""Extractor for all projects of an artstation user"""
"""Extractor for all projects in an artstation album"""
subcategory = "album"
directory_fmt = ["{category}", "{userinfo[username]}", "Albums",
"{album[id]} - {album[title]}"]

166
gallery_dl/extractor/hentaifoundry.py
View File

@@ -9,71 +9,45 @@
"""Extract images from https://www.hentai-foundry.com/"""
from .common import Extractor, Message
from .. import text, exception
from .. import text, util, exception
from urllib.parse import urljoin
class HentaifoundryUserExtractor(Extractor):
"""Extractor for all images of a hentai-foundry-user"""
class HentaifoundryExtractor(Extractor):
category = "hentaifoundry"
subcategory = "user"
directory_fmt = ["{category}", "{artist}"]
filename_fmt = "{category}_{index}_{title}.{extension}"
archive_fmt = "{index}"
pattern = [r"(?:https?://)?(?:www\.)?hentai-foundry\.com/"
r"(?:pictures/user/([^/]+)/?$|user/([^/]+)/profile)"]
test = [
("https://www.hentai-foundry.com/pictures/user/Tenpura", {
"url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28",
"keyword": "f8fecc8aa89978ecf402ec221243978fe791bd54",
}),
("http://www.hentai-foundry.com/user/asdq/profile", {
"exception": exception.NotFoundError,
}),
]
base_url = "https://www.hentai-foundry.com/pictures/user/"
root = "https://www.hentai-foundry.com"
per_page = 25
def __init__(self, match):
def __init__(self, artist):
Extractor.__init__(self)
self.artist = match.group(1) or match.group(2)
self.artist = artist
self.artist_url = "{}/pictures/user/{}".format(self.root, self.artist)
self.start_post = 0
def items(self):
data, token = self.get_job_metadata()
self.set_filters(token)
data = self.get_job_metadata()
yield Message.Version, 1
yield Message.Directory, data
for url, image in self.get_images(data["count"]):
for page_url in util.advance(self.get_image_pages(), self.start_post):
url, image = self.get_image_metadata(page_url)
image.update(data)
if not image["extension"]:
image["extension"] = "jpg"
yield Message.Url, url, image
def get_images(self, count):
"""Yield url and keywords for all images of one artist"""
num = 1
needle = 'thumbTitle"><a href="/pictures/user/'
for _ in range((count-1) // 25 + 1):
pos = 0
url = self.base_url + self.artist + "/page/" + str(num)
page = self.request(url).text
for _ in range(25):
part, pos = text.extract(page, needle, '"', pos)
if not part:
return
yield self.get_image_metadata(self.base_url + part)
num += 1
def get_image_pages(self):
"""Yield urls all image pages of one artist"""
def get_job_metadata(self):
"""Collect metadata for extractor-job"""
url = self.base_url + self.artist + "?enterAgree=1"
response = self.request(url, fatal=False)
if response.status_code == 404:
raise exception.NotFoundError("user")
page = response.text
token, pos = text.extract(page, 'hidden" value="', '"')
count, pos = text.extract(page, 'class="active" >Pictures (', ')', pos)
return {"artist": self.artist, "count": text.parse_int(count)}, token
def get_image_metadata(self, url):
"""Collect metadata for an image"""
page = self.request(url).text
page = self.request(urljoin(self.root, url)).text
index = url.rsplit("/", 2)[1]
title, pos = text.extract(
page, 'Pictures</a> &raquo; <span>', '<')
@@ -83,8 +57,65 @@ class HentaifoundryUserExtractor(Extractor):
text.nameext_from_url(part, data)
return "https://pictures.hentai-foundry.com" + part, data
def set_filters(self, token):
class HentaifoundryUserExtractor(HentaifoundryExtractor):
"""Extractor for all images of a hentai-foundry-user"""
subcategory = "user"
pattern = [r"(?:https?://)?(?:www\.)?hentai-foundry\.com/"
r"(?:pictures/user/([^/]+)(?:/(?:page/(\d+))?)?$"
r"|user/([^/]+)/profile)"]
test = [
("https://www.hentai-foundry.com/pictures/user/Tenpura", {
"url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28",
"keyword": "f8fecc8aa89978ecf402ec221243978fe791bd54",
}),
("http://www.hentai-foundry.com/user/asdq/profile", {
"exception": exception.NotFoundError,
}),
("https://www.hentai-foundry.com/pictures/user/Tenpura/page/3", None),
]
def __init__(self, match):
HentaifoundryExtractor.__init__(self, match.group(1) or match.group(3))
self.start_page = text.parse_int(match.group(2), 1)
self._skipped = (self.start_page - 1) * self.per_page
def skip(self, num):
pages, posts = divmod(num, self.per_page)
self.start_page += pages
self.start_post += posts
self._skipped += num
return num
def get_image_pages(self):
num = self.start_page
while True:
url = "{}/page/{}".format(self.artist_url, num)
page = self.request(url).text
yield from text.extract_iter(page, 'thumbTitle"><a href="', '"')
if 'class="pager"' not in page or 'class="last hidden"' in page:
return
num += 1
def get_job_metadata(self):
response = self.request(self.artist_url + "?enterAgree=1", fatal=False)
if response.status_code == 404:
raise exception.NotFoundError("user")
count = text.parse_int(text.extract(
response.text, 'class="active" >Pictures (', ')')[0])
if self._skipped >= count:
raise exception.StopExtraction()
self.set_filters()
return {"artist": self.artist, "count": count}
def set_filters(self):
"""Set site-internal filters to show all images"""
token = text.extract(
self.session.cookies["YII_CSRF_TOKEN"], "%22", "%22")[0]
formdata = {
"YII_CSRF_TOKEN": token,
"rating_nudity": 3,
@@ -114,55 +145,30 @@ class HentaifoundryUserExtractor(Extractor):
method="post", data=formdata, allow_empty=True)
class HentaifoundryImageExtractor(Extractor):
class HentaifoundryImageExtractor(HentaifoundryExtractor):
"""Extractor for a single image from hentaifoundry.com"""
category = "hentaifoundry"
subcategory = "image"
directory_fmt = ["{category}", "{artist}"]
filename_fmt = "{category}_{index}_{title}.{extension}"
archive_fmt = "{index}"
pattern = [(r"(?:https?://)?(?:www\.|pictures\.)?hentai-foundry\.com/"
r"(?:pictures/user/([^/]+)/(\d+)"
r"|[^/]/([^/]+)/(\d+))")]
test = [
(("http://www.hentai-foundry.com/"
"pictures/user/Tenpura/407501/shimakaze"), {
(("http://www.hentai-foundry.com"
"/pictures/user/Tenpura/407501/shimakaze"), {
"url": "fbf2fd74906738094e2575d2728e8dc3de18a8a3",
"keyword": "85b8e26fa93d00ae1333cb7b418078f1792dc4a8",
"keyword": "2956321893e9187edde4aeac6bed889449692e6a",
"content": "91bf01497c39254b6dfb234a18e8f01629c77fd1",
}),
("http://www.hentai-foundry.com/pictures/user/Tenpura/340853/", {
"exception": exception.NotFoundError,
"exception": exception.HttpError,
}),
]
def __init__(self, match):
Extractor.__init__(self)
self.artist = match.group(1) or match.group(3)
HentaifoundryExtractor.__init__(self, match.group(1) or match.group(3))
self.index = match.group(2) or match.group(4)
def items(self):
url, data = self.get_image_metadata()
yield Message.Version, 1
yield Message.Directory, data
yield Message.Url, url, data
def get_image_pages(self):
return ("{}/{}?enterAgree=1".format(self.artist_url, self.index),)
def get_image_metadata(self):
"""Collect metadata for an image"""
url = "https://www.hentai-foundry.com/pictures/user/{}/{}".format(
self.artist, self.index)
response = self.request(url + "?enterAgree=1", fatal=False)
if response.status_code == 404:
raise exception.NotFoundError("image")
extr = text.extract
page = response.text
artist, pos = extr(page, '<a href="/pictures/user/', '"')
title , pos = extr(page, 'Pictures</a> &raquo; <span>', '<', pos)
url , pos = extr(page, '//pictures.hentai-foundry.com', '"', pos)
data = {
"artist": artist,
"index": text.parse_int(self.index),
"title": text.unescape(title),
}
text.nameext_from_url(url, data)
return "https://pictures.hentai-foundry.com" + url, data
def get_job_metadata(self):
return {"artist": self.artist, "index": text.parse_int(self.index)}

2
gallery_dl/extractor/hitomi.py
View File

@@ -24,7 +24,7 @@ class HitomiGalleryExtractor(ChapterExtractor):
test = [
("https://hitomi.la/galleries/867789.html", {
"url": "cb759868d090fe0e2655c3e29ebf146054322b6d",
"keyword": "b1e66ff971fc8cb80240a687f508f3b74053f799",
"keyword": "85e453d01ee7f137669e75a764ccdc65ca092ad2",
}),
("https://hitomi.la/reader/867789.html", None),
]

2
gallery_dl/extractor/luscious.py
View File

@@ -74,7 +74,7 @@ class LusciousAlbumExtractor(AsynchronousExtractor):
"""Collect image-urls and -metadata"""
num = 1
if 'class="read-more-btn"' in page:
if 'class="search_filter' in page:
url = "{}/pictures/album/x_{}/sorted/oldest/page/1/".format(
self.root, self.gid)
page = self.request(url).text

88
gallery_dl/extractor/oauth.py
View File

@@ -50,14 +50,7 @@ class OAuthBase(Extractor):
data = self.client.recv(1024).decode()
path = data.split(" ", 2)[1]
query = path.partition("?")[2]
return {
key: urllib.parse.unquote(value)
for key, _, value in [
part.partition("=")
for part in query.split("&")
]
}
return text.parse_query(path.partition("?")[2])
def send(self, msg):
"""Send 'msg' to the socket opened in 'recv()'"""
@@ -69,8 +62,7 @@ class OAuthBase(Extractor):
"""Open 'url' in browser amd return response parameters"""
import webbrowser
url += "?" + urllib.parse.urlencode(params)
browser = self.config("browser", True)
if not browser or not webbrowser.open(url):
if not self.config("browser", True) or not webbrowser.open(url):
print("Please open this URL in your browser:")
print(url, end="\n\n", flush=True)
return self.recv()
@@ -80,7 +72,7 @@ class OAuthBase(Extractor):
"""Perform the OAuth 1.0a authorization flow"""
del self.session.params["oauth_token"]
# Get a Request Token
# get a request token
params = {"oauth_callback": self.redirect_uri}
data = self.session.get(request_token_url, params=params).text
@@ -88,25 +80,29 @@ class OAuthBase(Extractor):
self.session.params["oauth_token"] = token = data["oauth_token"]
self.session.token_secret = data["oauth_token_secret"]
# Get the User's Authorization
# get the user's authorization
params = {"oauth_token": token, "perms": "read"}
data = self.open(authorize_url, params)
# Exchange the Request Token for an Access Token
# exchange the request token for an access token
data = self.session.get(access_token_url, params=data).text
data = text.parse_query(data)
self.send(OAUTH1_MSG_TEMPLATE.format(
category=self.subcategory,
token=data["oauth_token"],
token_secret=data["oauth_token_secret"]))
token_secret=data["oauth_token_secret"],
))
def _oauth2_authorization_code_grant(
self, client_id, client_secret, auth_url, token_url, scope):
self, client_id, client_secret, auth_url, token_url,
scope="read", key="refresh_token", auth=True):
"""Perform an OAuth2 authorization code grant"""
state = "gallery-dl:{}:{}".format(
self.subcategory, util.OAuthSession.nonce(8))
state = "gallery-dl_{}_{}".format(
self.subcategory,
util.OAuthSession.nonce(8)
)
auth_params = {
"client_id": client_id,
@@ -117,25 +113,33 @@ class OAuthBase(Extractor):
"scope": scope,
}
# receive 'code'
# receive an authorization code
params = self.open(auth_url, auth_params)
# check auth response
# check authorization response
if state != params.get("state"):
self.send("'state' mismatch: expected {}, got {}.".format(
state, params.get("state")))
state, params.get("state")
))
return
if "error" in params:
self.send(params["error"])
return
# exchange 'code' for 'refresh_token'
# exchange the authorization code for a token
data = {
"grant_type": "authorization_code",
"code": params["code"],
"redirect_uri": self.redirect_uri,
}
auth = (client_id, client_secret)
if auth:
auth = (client_id, client_secret)
else:
auth = None
data["client_id"] = client_id
data["client_secret"] = client_secret
data = self.session.post(token_url, data=data, auth=auth).json()
# check token response
@@ -143,10 +147,13 @@ class OAuthBase(Extractor):
self.send(data["error"])
return
# display refresh token
# display token
part = key.partition("_")[0]
self.send(OAUTH2_MSG_TEMPLATE.format(
category=self.subcategory,
token=data["refresh_token"]
key=part,
Key=part.capitalize(),
token=data[key],
))
@@ -165,7 +172,7 @@ class OAuthDeviantart(OAuthBase):
"client-secret", deviantart.DeviantartAPI.CLIENT_SECRET),
"https://www.deviantart.com/oauth2/authorize",
"https://www.deviantart.com/oauth2/token",
"browse",
scope="browse",
)
@@ -191,6 +198,29 @@ class OAuthFlickr(OAuthBase):
)
class OAuthPinterest(OAuthBase):
subcategory = "pinterest"
pattern = ["oauth:pinterest$"]
redirect_uri = "https://mikf.github.io/gallery-dl/oauth-redirect.html"
def items(self):
yield Message.Version, 1
client_id = self.oauth_config("client-id")
client_secret = self.oauth_config("client-secret")
if not client_id or not client_secret:
self.log.error("'client-id' and 'client-secret' required")
return
self._oauth2_authorization_code_grant(
client_id, client_secret,
"https://api.pinterest.com/oauth/",
"https://api.pinterest.com/v1/oauth/token",
scope="read_public", key="access_token", auth=False,
)
class OAuthReddit(OAuthBase):
subcategory = "reddit"
pattern = ["oauth:reddit$"]
@@ -204,7 +234,7 @@ class OAuthReddit(OAuthBase):
"",
"https://www.reddit.com/api/v1/authorize",
"https://www.reddit.com/api/v1/access_token",
"read",
scope="read",
)
@@ -253,18 +283,18 @@ Example:
OAUTH2_MSG_TEMPLATE = """
Your Refresh Token is
Your {Key} Token is
{token}
Put this value into your configuration file as
'extractor.{category}.refesh-token'.
'extractor.{category}.{key}-token'.
Example:
{{
"extractor": {{
"{category}": {{
"refresh-token": "{token}"
"{key}-token": "{token}"
}}
}}
}}

70
gallery_dl/extractor/pinterest.py
View File

@@ -46,6 +46,9 @@ class PinterestPinExtractor(PinterestExtractor):
"content": "d3e24bc9f7af585e8c23b9136956bd45a4d9b947",
}),
("https://www.pinterest.com/pin/858146903966145188/", {
"exception": exception.StopExtraction,
}),
("https://www.pinterest.com/pin/85814690396614518/", {
"exception": exception.NotFoundError,
}),
]
@@ -135,52 +138,57 @@ class PinterestPinitExtractor(PinterestExtractor):
class PinterestAPI():
"""Minimal interface for the pinterest API"""
def __init__(self, extractor, access_token="AV2U9Oe6dyC2vfPugUnBvJ7Duxg9"
"FHCJPXPZIvRDXv9hvwBALwAAAAA"):
access_token = extractor.config("access-token", access_token)
def __init__(self, extractor, access_token=None):
self.log = extractor.log
self.session = extractor.session
self.session.params["access_token"] = access_token
self.access_token = (
access_token or
extractor.config("access-token") or
"AfyIXxi1MJ6et0NlIl_vBchHbex-FSWylPyr2GJE2uu3W8A97QAAAAA"
)
def pin(self, pin_id, fields="id,image,note"):
"""Query information about a pin"""
endpoint = "pins/{}/".format(pin_id)
params = {"fields": fields}
response = self.session.get(
"https://api.pinterest.com/v1/pins/{pin}/".format(pin=pin_id),
params=params
)
return self._parse(response)["data"]
return self._call(endpoint, params)["data"]
def board(self, user, board, fields="id,name,counts"):
"""Query information about a board"""
endpoint = "boards/{}/{}/".format(user, board)
params = {"fields": fields}
response = self.session.get(
"https://api.pinterest.com/v1/boards/{user}/{board}/"
.format(user=user, board=board), params=params
)
return self._parse(response)["data"]
return self._call(endpoint, params)["data"]
def board_pins(self, user, board, fields="id,image,note"):
def board_pins(self, user, board, fields="id,image,note", limit=100):
"""Yield all pins of a specific board"""
params = {"fields": fields}
url = ("https://api.pinterest.com/v1/boards/{user}/{board}/pins/"
.format(user=user, board=board))
endpoint = "boards/{}/{}/pins/".format(user, board)
params = {"fields": fields, "limit": limit}
return self._pagination(endpoint, params)
def _call(self, endpoint, params):
params["access_token"] = self.access_token
url = "https://api.pinterest.com/v1/" + endpoint
response = self.session.get(url, params=params)
status = response.status_code
data = response.json()
if 200 <= status < 400 and data.get("data"):
return data
msg = data.get("message", "")
if status == 404:
msg = msg.partition(" ")[0].lower()
raise exception.NotFoundError(msg)
self.log.error("API request failed: %s", msg or "")
raise exception.StopExtraction()
def _pagination(self, endpoint, params):
while True:
response = self._parse(self.session.get(url, params=params))
response = self._call(endpoint, params)
yield from response["data"]
cursor = response["page"]["cursor"]
if not cursor:
return
params["cursor"] = cursor
@staticmethod
def _parse(response):
"""Parse an API response"""
data = response.json()
if "data" not in data or data["data"] is None:
try:
msg = data["message"].partition(" ")[0].lower()
except KeyError:
msg = ""
raise exception.NotFoundError(msg)
return data

2
gallery_dl/extractor/slideshare.py
View File

@@ -12,7 +12,7 @@ from .common import Extractor, Message
from .. import text
class SlideshareExtractor(Extractor):
class SlidesharePresentationExtractor(Extractor):
"""Extractor for images from a presentation on slideshare.net"""
category = "slideshare"
subcategory = "presentation"

74
gallery_dl/extractor/spectrumnexus.py
View File

@@ -1,74 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2015-2018 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
"""Extract manga pages from http://www.thespectrum.net/manga_scans/"""
from .common import ChapterExtractor, MangaExtractor
from .. import text
class SpectrumnexusMangaExtractor(MangaExtractor):
"""Extractor for manga from thespectrum.net"""
category = "spectrumnexus"
pattern = [r"(?:https?://)?(view\.thespectrum\.net/series/[^.]+\.html)#?$"]
reverse = False
test = [("http://view.thespectrum.net/series/kare-kano-volume-01.html", {
"url": "b2b175aad5ef1701cc4aee7c24f1ca3a93aba9cb",
"keyword": "5ed9d5c7c69d2d03417c853c4e8eae30f1e5febf",
})]
def chapters(self, page):
results = []
manga = text.extract(page, '<title>', ' &#183; ')[0]
page = text.extract(page, 'class="selectchapter"', '</select>')[0]
for chapter in text.extract_iter(page, '<option value="', '"'):
results.append((self.url + "?ch=" + chapter.replace(" ", "+"), {
"manga": manga, "chapter_string": chapter,
}))
return results
class SpectrumnexusChapterExtractor(ChapterExtractor):
"""Extractor for manga-chapters or -volumes from thespectrum.net"""
category = "spectrumnexus"
directory_fmt = ["{category}", "{manga}", "{chapter_string}"]
filename_fmt = "{manga}_{chapter_string}_{page:>03}.{extension}"
archive_fmt = "{manga}_{chapter_string}_{page}"
pattern = [r"(?:https?://)?view\.thespectrum\.net/series/"
r"([^\.]+\.html)\?ch=(Chapter\+(\d+)|Volume\+(\d+))"]
test = [(("http://view.thespectrum.net/series/"
"toriko.html?ch=Chapter+343&page=1"), {
"url": "c0fc7dc594841217cc622a67edd79f06e9900333",
"keyword": "a8abe126cbc5fc798148b0b155242a470c1ba9d1",
})]
def __init__(self, match):
path, self.chapter_string, self.chapter, self.volume = match.groups()
url = "http://view.thespectrum.net/series/{}?ch={}".format(
path, self.chapter_string)
ChapterExtractor.__init__(self, url)
def get_metadata(self, page):
data = {
"chapter": text.parse_int(self.chapter),
"chapter_string": self.chapter_string.replace("+", " "),
"volume": text.parse_int(self.volume),
}
data = text.extract_all(page, (
('manga', '<title>', ' &#183; SPECTRUM NEXUS </title>'),
('count', '<div class="viewerLabel"> of ', '<'),
), values=data)[0]
data["count"] = text.parse_int(data["count"])
return data
def get_images(self, page):
params = {"page": 1}
while True:
yield text.extract(page, '<img id="mainimage" src="', '"')[0], None
params["page"] += 1
page = self.request(self.url, params=params).text

2
gallery_dl/extractor/tumblr.py
View File

@@ -234,7 +234,7 @@ class TumblrTagExtractor(TumblrExtractor):
class TumblrLikesExtractor(TumblrExtractor):
"""Extractor for images from a tumblr-user by tag"""
"""Extractor for images from a tumblr-user's liked posts"""
subcategory = "likes"
directory_fmt = ["{category}", "{name}", "likes"]
archive_fmt = "f_{blog[name]}_{id}_{num}"