github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve cookie handling during logins

Browse Source
This commit is contained in:
Mike Fährmann
2019-01-30 17:09:32 +01:00
parent 6126615698
commit dd358b4564
7 changed files with 68 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
gallery_dl/extractor/common.py
View File

@@ -116,6 +116,20 @@ class Extractor():
"user-agent", ("Mozilla/5.0 (X11; Linux x86_64; rv:62.0) "
"Gecko/20100101 Firefox/62.0"))
def _set_proxies(self):
"""Update the session's proxy map"""
proxies = self.config("proxy")
if proxies:
if isinstance(proxies, str):
proxies = {"http": proxies, "https": proxies}
if isinstance(proxies, dict):
for scheme, proxy in proxies.items():
if "://" not in proxy:
proxies[scheme] = "http://" + proxy.lstrip("/")
self.session.proxies = proxies
else:
self.log.warning("invalid proxy specifier: %s", proxies)
def _set_cookies(self):
"""Populate the session's cookiejar"""
cookies = self.config("cookies")
@@ -132,31 +146,34 @@ class Extractor():
except OSError as exc:
self.log.warning("cookies: %s", exc)
def _set_proxies(self):
"""Update the session's proxy map"""
proxies = self.config("proxy")
if proxies:
if isinstance(proxies, str):
proxies = {"http": proxies, "https": proxies}
if isinstance(proxies, dict):
for scheme, proxy in proxies.items():
if "://" not in proxy:
proxies[scheme] = "http://" + proxy.lstrip("/")
self.session.proxies = proxies
else:
self.log.warning("invalid proxy specifier: %s", proxies)
def _check_cookies(self, cookienames, domain=None):
def _check_cookies(self, cookienames, *, domain=""):
"""Check if all 'cookienames' are in the session's cookiejar"""
if not domain and self.cookiedomain:
if not domain:
domain = self.cookiedomain
for name in cookienames:
try:
try:
for name in cookienames:
self.session.cookies._find(name, domain)
except KeyError:
return False
except KeyError:
return False
return True
def _update_cookies(self, cookies, *, domain=""):
"""Update the session's cookiejar with 'cookies'"""
if isinstance(cookies, dict):
if not domain:
domain = self.cookiedomain
setcookie = self.session.cookies.set
for name, value in cookies.items():
setcookie(name, value, domain=domain)
else:
try:
cookies = iter(cookies)
except TypeError:
cookies = (cookies,)
setcookie = self.session.cookies.set_cookie
for cookie in cookies:
setcookie(cookie)
class AsynchronousExtractor(Extractor):

18
gallery_dl/extractor/exhentai.py
View File

@@ -62,23 +62,22 @@ class ExhentaiExtractor(Extractor):
if self._check_cookies(self.cookienames):
return
username, password = self._get_auth_info()
if not username:
if username:
self._update_cookies(self._login_impl(username, password))
else:
self.log.info("no username given; using e-hentai.org")
self.root = "https://e-hentai.org"
self.original = False
self.limits = False
self.session.cookies["nw"] = "1"
return
cookies = self._login_impl(username, password)
for key, value in cookies.items():
self.session.cookies.set(
key, value, domain=self.cookiedomain)
@cache(maxage=90*24*60*60, keyarg=1)
def _login_impl(self, username, password):
"""Actual login implementation"""
self.log.info("Logging in as %s", username)
url = "https://forums.e-hentai.org/index.php?act=Login&CODE=01"
headers = {
"Referer": "https://e-hentai.org/bounce_login.php?b=d&bt=1-1",
}
data = {
"CookieDate": "1",
"b": "d",
@@ -87,11 +86,8 @@ class ExhentaiExtractor(Extractor):
"PassWord": password,
"ipb_login_submit": "Login!",
}
headers = {
"Referer": "https://e-hentai.org/bounce_login.php?b=d&bt=1-1"
}
response = self.request(url, method="POST", data=data, headers=headers)
response = self.request(url, method="POST", headers=headers, data=data)
if "You are now logged in as:" not in response.text:
raise exception.AuthenticationError()
return {c: response.cookies[c] for c in self.cookienames}

9
gallery_dl/extractor/luscious.py
View File

@@ -23,10 +23,9 @@ class LusciousExtractor(Extractor):
"""Login and set necessary cookies"""
username, password = self._get_auth_info()
if username:
cookie = self._login_impl(username, password)
self.session.cookies.set_cookie(cookie)
self._update_cookies(self._login_impl(username, password))
@cache(maxage=13*24*60*60, keyarg=1)
@cache(maxage=14*24*60*60, keyarg=1)
def _login_impl(self, username, password):
self.log.info("Logging in as %s", username)
url = "https://members.luscious.net/accounts/login/"
@@ -37,13 +36,13 @@ class LusciousExtractor(Extractor):
"remember": "on",
"next": "" "/",
}
response = self.request(url, method="POST", headers=headers, data=data)
response = self.request(url, method="POST", headers=headers, data=data)
if "/accounts/login/" in response.url or not response.history:
raise exception.AuthenticationError()
for cookie in response.history[0].cookies:
if cookie.name.startswith("sessionid_"):
return cookie
return {cookie.name: cookie.value}
raise exception.AuthenticationError()

21
gallery_dl/extractor/nijie.py
View File

@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2015-2018 Mike Fährmann
# Copyright 2015-2019 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -20,6 +20,7 @@ class NijieExtractor(AsynchronousExtractor):
filename_fmt = "{category}_{artist_id}_{image_id}_p{index:>02}.{extension}"
archive_fmt = "{image_id}_{index}"
cookiedomain = "nijie.info"
cookienames = ("nemail", "nlogin")
root = "https://nijie.info"
view_url = "https://nijie.info/view.php?id="
popup_url = "https://nijie.info/view_popup.php?id="
@@ -84,19 +85,19 @@ class NijieExtractor(AsynchronousExtractor):
})
def login(self):
"""Login and obtain session cookie"""
if not self._check_cookies(("nemail", "nlogin")):
"""Login and obtain session cookies"""
if not self._check_cookies(self.cookienames):
username, password = self._get_auth_info()
self.session.cookies = self._login_impl(username, password)
self._update_cookies(self._login_impl(username, password))
@cache(maxage=30*24*60*60, keyarg=1)
@cache(maxage=150*24*60*60, keyarg=1)
def _login_impl(self, username, password):
"""Actual login implementation"""
self.log.info("Logging in as %s", username)
data = {"email": username, "password": password}
page = self.request(
self.root + "/login_int.php", method="POST", data=data).text
if "//nijie.info/login.php" in page:
url = "{}/login_int.php".format(self.root)
data = {"email": username, "password": password, "save": "on"}
response = self.request(url, method="POST", data=data)
if "//nijie.info/login.php" in response.text:
raise exception.AuthenticationError()
return self.session.cookies

5
gallery_dl/extractor/sankaku.py
View File

@@ -119,15 +119,12 @@ class SankakuExtractor(SharedConfigExtractor):
username, password = self._get_auth_info()
if username:
cookies = self._login_impl((username, self.subdomain), password)
for key, value in cookies.items():
self.session.cookies.set(
key, value, domain=self.cookiedomain)
self._update_cookies(cookies)
else:
self.logged_in = False
@cache(maxage=90*24*60*60, keyarg=1)
def _login_impl(self, usertuple, password):
"""Actual login implementation"""
username = usertuple[0]
self.log.info("Logging in as %s", username)
url = self.root + "/user/authenticate"

4
gallery_dl/extractor/seiga.py
View File

@@ -51,14 +51,14 @@ class SeigaExtractor(Extractor):
"""Login and set necessary cookies"""
if not self._check_cookies(("user_session",)):
username, password = self._get_auth_info()
self.session.cookies = self._login_impl(username, password)
self._update_cookies(self._login_impl(username, password))
@cache(maxage=7*24*60*60, keyarg=1)
def _login_impl(self, username, password):
"""Actual login implementation"""
self.log.info("Logging in as %s", username)
url = "https://account.nicovideo.jp/api/v1/login"
data = {"mail_tel": username, "password": password}
self.request(url, method="POST", data=data)
if "user_session" not in self.session.cookies:
raise exception.AuthenticationError()

15
gallery_dl/extractor/wallhaven.py
View File

@@ -23,14 +23,11 @@ class WallhavenExtractor(Extractor):
"""Login and set necessary cookies"""
username, password = self._get_auth_info()
if username:
cookie = self._login_impl(username, password)
self.session.cookies.set_cookie(cookie)
self._update_cookies(self._login_impl(username, password))
@cache(maxage=365*24*60*60, keyarg=1)
def _login_impl(self, username, password):
"""Actual login implementation"""
self.log.info("Logging in as %s", username)
url = "{}/auth/login".format(self.root)
page = self.request(url).text
pos = page.index('name="_token"')
@@ -40,12 +37,12 @@ class WallhavenExtractor(Extractor):
"password": password,
"_token": text.extract(page, 'value="', '"', pos)[0]
}
response = self.request(
url, method="POST", data=data, allow_redirects=False)
response = self.request(url, method="POST", data=data)
for cookie in response.cookies:
if cookie.name.startswith("remember_"):
return cookie
if response.history:
for cookie in response.history[0].cookies:
if cookie.name.startswith("remember_"):
return {cookie.name: cookie.value}
raise exception.AuthenticationError()
def get_wallpaper_data(self, wallpaper_id):