[twitter] improve username & password login procedure (#5445)

- handle more subtasks
- support 2FA
- support email verification codes

gallery_dl/extractor/twitter.py

@@ -1710,23 +1710,24 @@ class TwitterAPI():
 
 @cache(maxage=365*86400, keyarg=1)
 def _login_impl(extr, username, password):
-
-    import re
     import random
 
-    if re.fullmatch(r"[\w.%+-]+@[\w.-]+\.\w{2,}", username):
+    def process(data, params=None):
-        extr.log.warning(
+        response = extr.request(
-            "Login with email is no longer possible. "
+            url, params=params, headers=headers, json=data,
-            "You need to provide your username or phone number instead.")
+            method="POST", fatal=None)
 
-    def process(response):
         try:
             data = response.json()
         except ValueError:
             data = {"errors": ({"message": "Invalid response"},)}
         else:
             if response.status_code < 400:
-                return data["flow_token"]
+                try:
+                    return (data["flow_token"],
+                            data["subtasks"][0]["subtask_id"])
+                except LookupError:
+                    pass
 
         errors = []
         for error in data.get("errors") or ():
@@ -1735,9 +1736,13 @@ def _login_impl(extr, username, password):
         extr.log.debug(response.text)
         raise exception.AuthenticationError(", ".join(errors))
 
-    extr.cookies.clear()
+    cookies = extr.cookies
+    cookies.clear()
     api = TwitterAPI(extr)
     api._authenticate_guest()
+
+    url = "https://api.twitter.com/1.1/onboarding/task.json"
+    params = {"flow_name": "login"}
     headers = api.headers
 
     extr.log.info("Logging in as %s", username)
@@ -1794,31 +1799,18 @@ def _login_impl(extr, username, password):
             "web_modal": 1,
         },
     }
-    url = "https://api.twitter.com/1.1/onboarding/task.json?flow_name=login"
-    response = extr.request(url, method="POST", headers=headers, json=data)
 
-    data = {
+    flow_token, subtask = process(data, params)
-        "flow_token": process(response),
+    while not cookies.get("auth_token"):
-        "subtask_inputs": [
+        if subtask == "LoginJsInstrumentationSubtask":
-            {
+            data = {
-                "subtask_id": "LoginJsInstrumentationSubtask",
                 "js_instrumentation": {
                     "response": "{}",
                     "link": "next_link",
                 },
-            },
+            }
-        ],
+        elif subtask == "LoginEnterUserIdentifierSSO":
-    }
+            data = {
-    url = "https://api.twitter.com/1.1/onboarding/task.json"
-    response = extr.request(
-        url, method="POST", headers=headers, json=data, fatal=None)
-
-    # username
-    data = {
-        "flow_token": process(response),
-        "subtask_inputs": [
-            {
-                "subtask_id": "LoginEnterUserIdentifierSSO",
                 "settings_list": {
                     "setting_responses": [
                         {
@@ -1830,48 +1822,61 @@ def _login_impl(extr, username, password):
                     ],
                     "link": "next_link",
                 },
-            },
+            }
-        ],
+        elif subtask == "LoginEnterPassword":
-    }
+            data = {
-    #  url = "https://api.twitter.com/1.1/onboarding/task.json"
-    extr.sleep(random.uniform(2.0, 4.0), "login (username)")
-    response = extr.request(
-        url, method="POST", headers=headers, json=data, fatal=None)
-
-    # password
-    data = {
-        "flow_token": process(response),
-        "subtask_inputs": [
-            {
-                "subtask_id": "LoginEnterPassword",
                 "enter_password": {
                     "password": password,
                     "link": "next_link",
                 },
-            },
+            }
-        ],
+        elif subtask == "LoginEnterAlternateIdentifierSubtask":
-    }
+            alt = extr.input(
-    #  url = "https://api.twitter.com/1.1/onboarding/task.json"
+                "Alternate Identifier (username, email, phone number): ")
-    extr.sleep(random.uniform(2.0, 4.0), "login (password)")
+            data = {
-    response = extr.request(
+                "enter_text": {
-        url, method="POST", headers=headers, json=data, fatal=None)
+                    "text": alt,
-
+                    "link": "next_link",
-    # account duplication check ?
+                },
-    data = {
+            }
-        "flow_token": process(response),
+        elif subtask == "LoginTwoFactorAuthChallenge":
-        "subtask_inputs": [
+            data = {
-            {
+                "enter_text": {
-                "subtask_id": "AccountDuplicationCheck",
+                    "text": extr.input("2FA Token: "),
+                    "link": "next_link",
+                },
+            }
+        elif subtask == "LoginAcid":
+            data = {
+                "enter_text": {
+                    "text": extr.input("Email Verification Code: "),
+                    "link": "next_link",
+                },
+            }
+        elif subtask == "AccountDuplicationCheck":
+            data = {
                 "check_logged_in_account": {
                     "link": "AccountDuplicationCheck_false",
                 },
-            },
+            }
-        ],
+        elif subtask == "ArkoseLogin":
-    }
+            raise exception.AuthenticationError("Login requires CAPTCHA")
-    #  url = "https://api.twitter.com/1.1/onboarding/task.json"
+        elif subtask == "DenyLoginSubtask":
-    response = extr.request(
+            raise exception.AuthenticationError("Login rejected as suspicious")
-        url, method="POST", headers=headers, json=data, fatal=None)
+        elif subtask == "ArkoseLogin":
-    process(response)
+            raise exception.AuthenticationError("No auth token cookie")
+        else:
+            raise exception.StopExtraction("Unrecognized subtask %s", subtask)
+
+        inputs = {"subtask_id": subtask}
+        inputs.update(data)
+        data = {
+            "flow_token": flow_token,
+            "subtask_inputs": [inputs],
+        }
+
+        extr.sleep(random.uniform(1.0, 3.0), "login ({})".format(subtask))
+        flow_token, subtask = process(data)
 
     return {
         cookie.name: cookie.value