[twitter] fix login with username & password

It is no longer possible to get an 'authenticity_token' from Twitter's
Javascript-free login form, which got disabled few days ago.

Generating a random 16 byte hex string client-side and sending that as
a cookie alongside the regular login form works just as well.

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Changelog
 
+## Unreleased
+
 ## 1.16.1 - 2020-12-27
 ### Additions
 - [instagram] add `include` option ([#1180](https://github.com/mikf/gallery-dl/issues/1180))

gallery_dl/extractor/twitter.py

@@ -239,30 +239,29 @@ class TwitterExtractor(Extractor):
     def _login_impl(self, username, password):
         self.log.info("Logging in as %s", username)
 
-        url = "https://mobile.twitter.com/i/nojs_router"
+        token = util.generate_csrf_token()
-        params = {"path": "/login"}
+        self.session.cookies.clear()
-        headers = {"Referer": self.root + "/", "Origin": self.root}
+        self.request(self.root + "/login")
-        page = self.request(
-            url, method="POST", params=params, headers=headers, data={}).text
 
-        pos = page.index('name="authenticity_token"')
+        url = self.root + "/sessions"
-        token = text.extract(page, 'value="', '"', pos)[0]
+        cookies = {
-
+            "_mb_tk": token,
-        url = "https://mobile.twitter.com/sessions"
+        }
         data = {
+            "redirect_after_login"      : "/",
+            "remember_me"               : "1",
             "authenticity_token"        : token,
+            "wfa"                       : "1",
+            "ui_metrics"                : "{}",
             "session[username_or_email]": username,
             "session[password]"         : password,
-            "remember_me"               : "1",
-            "wfa"                       : "1",
-            "commit"                    : "+Log+in+",
-            "ui_metrics"                : "",
         }
-        response = self.request(url, method="POST", data=data)
+        response = self.request(
+            url, method="POST", cookies=cookies, data=data)
+
         cookies = {
             cookie.name: cookie.value
             for cookie in self.session.cookies
-            if cookie.domain == self.cookiedomain
         }
 
         if "/error" in response.url or "auth_token" not in cookies:

gallery_dl/version.py

@@ -6,4 +6,4 @@
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
 
-__version__ = "1.16.1"
+__version__ = "1.16.2-dev"