improve cookie handling during logins

gallery_dl/extractor/common.py

@@ -116,6 +116,20 @@ class Extractor():
             "user-agent", ("Mozilla/5.0 (X11; Linux x86_64; rv:62.0) "
                            "Gecko/20100101 Firefox/62.0"))
 
+    def _set_proxies(self):
+        """Update the session's proxy map"""
+        proxies = self.config("proxy")
+        if proxies:
+            if isinstance(proxies, str):
+                proxies = {"http": proxies, "https": proxies}
+            if isinstance(proxies, dict):
+                for scheme, proxy in proxies.items():
+                    if "://" not in proxy:
+                        proxies[scheme] = "http://" + proxy.lstrip("/")
+                self.session.proxies = proxies
+            else:
+                self.log.warning("invalid proxy specifier: %s", proxies)
+
     def _set_cookies(self):
         """Populate the session's cookiejar"""
         cookies = self.config("cookies")
@@ -132,31 +146,34 @@ class Extractor():
                 except OSError as exc:
                     self.log.warning("cookies: %s", exc)
 
-    def _set_proxies(self):
+    def _check_cookies(self, cookienames, *, domain=""):
-        """Update the session's proxy map"""
-        proxies = self.config("proxy")
-        if proxies:
-            if isinstance(proxies, str):
-                proxies = {"http": proxies, "https": proxies}
-            if isinstance(proxies, dict):
-                for scheme, proxy in proxies.items():
-                    if "://" not in proxy:
-                        proxies[scheme] = "http://" + proxy.lstrip("/")
-                self.session.proxies = proxies
-            else:
-                self.log.warning("invalid proxy specifier: %s", proxies)
-
-    def _check_cookies(self, cookienames, domain=None):
         """Check if all 'cookienames' are in the session's cookiejar"""
-        if not domain and self.cookiedomain:
+        if not domain:
             domain = self.cookiedomain
-        for name in cookienames:
+        try:
-            try:
+            for name in cookienames:
                 self.session.cookies._find(name, domain)
-            except KeyError:
+        except KeyError:
-                return False
+            return False
         return True
 
+    def _update_cookies(self, cookies, *, domain=""):
+        """Update the session's cookiejar with 'cookies'"""
+        if isinstance(cookies, dict):
+            if not domain:
+                domain = self.cookiedomain
+            setcookie = self.session.cookies.set
+            for name, value in cookies.items():
+                setcookie(name, value, domain=domain)
+        else:
+            try:
+                cookies = iter(cookies)
+            except TypeError:
+                cookies = (cookies,)
+            setcookie = self.session.cookies.set_cookie
+            for cookie in cookies:
+                setcookie(cookie)
+
 
 class AsynchronousExtractor(Extractor):
 

gallery_dl/extractor/exhentai.py

@@ -62,23 +62,22 @@ class ExhentaiExtractor(Extractor):
         if self._check_cookies(self.cookienames):
             return
         username, password = self._get_auth_info()
-        if not username:
+        if username:
+            self._update_cookies(self._login_impl(username, password))
+        else:
             self.log.info("no username given; using e-hentai.org")
             self.root = "https://e-hentai.org"
             self.original = False
             self.limits = False
             self.session.cookies["nw"] = "1"
-            return
-        cookies = self._login_impl(username, password)
-        for key, value in cookies.items():
-            self.session.cookies.set(
-                key, value, domain=self.cookiedomain)
 
     @cache(maxage=90*24*60*60, keyarg=1)
     def _login_impl(self, username, password):
-        """Actual login implementation"""
         self.log.info("Logging in as %s", username)
         url = "https://forums.e-hentai.org/index.php?act=Login&CODE=01"
+        headers = {
+            "Referer": "https://e-hentai.org/bounce_login.php?b=d&bt=1-1",
+        }
         data = {
             "CookieDate": "1",
             "b": "d",
@@ -87,11 +86,8 @@ class ExhentaiExtractor(Extractor):
             "PassWord": password,
             "ipb_login_submit": "Login!",
         }
-        headers = {
-            "Referer": "https://e-hentai.org/bounce_login.php?b=d&bt=1-1"
-        }
-        response = self.request(url, method="POST", data=data, headers=headers)
 
+        response = self.request(url, method="POST", headers=headers, data=data)
         if "You are now logged in as:" not in response.text:
             raise exception.AuthenticationError()
         return {c: response.cookies[c] for c in self.cookienames}

gallery_dl/extractor/luscious.py

@@ -23,10 +23,9 @@ class LusciousExtractor(Extractor):
         """Login and set necessary cookies"""
         username, password = self._get_auth_info()
         if username:
-            cookie = self._login_impl(username, password)
+            self._update_cookies(self._login_impl(username, password))
-            self.session.cookies.set_cookie(cookie)
 
-    @cache(maxage=13*24*60*60, keyarg=1)
+    @cache(maxage=14*24*60*60, keyarg=1)
     def _login_impl(self, username, password):
         self.log.info("Logging in as %s", username)
         url = "https://members.luscious.net/accounts/login/"
@@ -37,13 +36,13 @@ class LusciousExtractor(Extractor):
             "remember": "on",
             "next": ""  "/",
         }
-        response = self.request(url, method="POST", headers=headers, data=data)
 
+        response = self.request(url, method="POST", headers=headers, data=data)
         if "/accounts/login/" in response.url or not response.history:
             raise exception.AuthenticationError()
         for cookie in response.history[0].cookies:
             if cookie.name.startswith("sessionid_"):
-                return cookie
+                return {cookie.name: cookie.value}
         raise exception.AuthenticationError()
 
 

gallery_dl/extractor/nijie.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2015-2018 Mike Fährmann
+# Copyright 2015-2019 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -20,6 +20,7 @@ class NijieExtractor(AsynchronousExtractor):
     filename_fmt = "{category}_{artist_id}_{image_id}_p{index:>02}.{extension}"
     archive_fmt = "{image_id}_{index}"
     cookiedomain = "nijie.info"
+    cookienames = ("nemail", "nlogin")
     root = "https://nijie.info"
     view_url = "https://nijie.info/view.php?id="
     popup_url = "https://nijie.info/view_popup.php?id="
@@ -84,19 +85,19 @@ class NijieExtractor(AsynchronousExtractor):
             })
 
     def login(self):
-        """Login and obtain session cookie"""
+        """Login and obtain session cookies"""
-        if not self._check_cookies(("nemail", "nlogin")):
+        if not self._check_cookies(self.cookienames):
             username, password = self._get_auth_info()
-            self.session.cookies = self._login_impl(username, password)
+            self._update_cookies(self._login_impl(username, password))
 
-    @cache(maxage=30*24*60*60, keyarg=1)
+    @cache(maxage=150*24*60*60, keyarg=1)
     def _login_impl(self, username, password):
-        """Actual login implementation"""
         self.log.info("Logging in as %s", username)
-        data = {"email": username, "password": password}
+        url = "{}/login_int.php".format(self.root)
-        page = self.request(
+        data = {"email": username, "password": password, "save": "on"}
-            self.root + "/login_int.php", method="POST", data=data).text
+
-        if "//nijie.info/login.php" in page:
+        response = self.request(url, method="POST", data=data)
+        if "//nijie.info/login.php" in response.text:
             raise exception.AuthenticationError()
         return self.session.cookies
 

gallery_dl/extractor/sankaku.py

@@ -119,15 +119,12 @@ class SankakuExtractor(SharedConfigExtractor):
         username, password = self._get_auth_info()
         if username:
             cookies = self._login_impl((username, self.subdomain), password)
-            for key, value in cookies.items():
+            self._update_cookies(cookies)
-                self.session.cookies.set(
-                    key, value, domain=self.cookiedomain)
         else:
             self.logged_in = False
 
     @cache(maxage=90*24*60*60, keyarg=1)
     def _login_impl(self, usertuple, password):
-        """Actual login implementation"""
         username = usertuple[0]
         self.log.info("Logging in as %s", username)
         url = self.root + "/user/authenticate"

gallery_dl/extractor/seiga.py

@@ -51,14 +51,14 @@ class SeigaExtractor(Extractor):
         """Login and set necessary cookies"""
         if not self._check_cookies(("user_session",)):
             username, password = self._get_auth_info()
-            self.session.cookies = self._login_impl(username, password)
+            self._update_cookies(self._login_impl(username, password))
 
     @cache(maxage=7*24*60*60, keyarg=1)
     def _login_impl(self, username, password):
-        """Actual login implementation"""
         self.log.info("Logging in as %s", username)
         url = "https://account.nicovideo.jp/api/v1/login"
         data = {"mail_tel": username, "password": password}
+
         self.request(url, method="POST", data=data)
         if "user_session" not in self.session.cookies:
             raise exception.AuthenticationError()

gallery_dl/extractor/wallhaven.py

@@ -23,14 +23,11 @@ class WallhavenExtractor(Extractor):
         """Login and set necessary cookies"""
         username, password = self._get_auth_info()
         if username:
-            cookie = self._login_impl(username, password)
+            self._update_cookies(self._login_impl(username, password))
-            self.session.cookies.set_cookie(cookie)
 
     @cache(maxage=365*24*60*60, keyarg=1)
     def _login_impl(self, username, password):
-        """Actual login implementation"""
         self.log.info("Logging in as %s", username)
-
         url = "{}/auth/login".format(self.root)
         page = self.request(url).text
         pos = page.index('name="_token"')
@@ -40,12 +37,12 @@ class WallhavenExtractor(Extractor):
             "password": password,
             "_token": text.extract(page, 'value="', '"', pos)[0]
         }
-        response = self.request(
+        response = self.request(url, method="POST", data=data)
-            url, method="POST", data=data, allow_redirects=False)
 
-        for cookie in response.cookies:
+        if response.history:
-            if cookie.name.startswith("remember_"):
+            for cookie in response.history[0].cookies:
-                return cookie
+                if cookie.name.startswith("remember_"):
+                    return {cookie.name: cookie.value}
         raise exception.AuthenticationError()
 
     def get_wallpaper_data(self, wallpaper_id):