[util] move Cloudflare/DDoS-Guard detection into 'detect_challenge()'

gallery_dl/extractor/common.py

@@ -205,25 +205,10 @@ class Extractor():
 
                 msg = "'{} {}' for '{}'".format(
                     code, response.reason, response.url)
-                server = response.headers.get("Server")
+
-                if server and server.startswith("cloudflare") and \
+                challenge = util.detect_challenge(response)
-                        code in (403, 503):
+                if challenge is not None:
-                    mitigated = response.headers.get("cf-mitigated")
+                    self.log.warning(challenge)
-                    if mitigated and mitigated.lower() == "challenge":
-                        self.log.warning("Cloudflare challenge")
-                        break
-                    content = response.content
-                    if b"_cf_chl_opt" in content or b"jschl-answer" in content:
-                        self.log.warning("Cloudflare challenge")
-                        break
-                    if b'name="captcha-bypass"' in content:
-                        self.log.warning("Cloudflare CAPTCHA")
-                        break
-                elif server and server.startswith("ddos-guard") and \
-                        code == 403:
-                    if b"/ddos-guard/js-challenge/" in response.content:
-                        self.log.warning("DDoS-Guard challenge")
-                        break
 
                 if code == 429 and self._handle_429(response):
                     continue

gallery_dl/util.py

@@ -369,6 +369,31 @@ def extract_headers(response):
     return data
 
 
+def detect_challenge(response):
+    server = response.headers.get("server")
+    if not server:
+        return
+
+    elif server.startswith("cloudflare"):
+        if response.status_code not in (403, 503):
+            return
+
+        mitigated = response.headers.get("cf-mitigated")
+        if mitigated and mitigated.lower() == "challenge":
+            return "Cloudflare challenge"
+
+        content = response.content
+        if b"_cf_chl_opt" in content or b"jschl-answer" in content:
+            return "Cloudflare challenge"
+        elif b'name="captcha-bypass"' in content:
+            return "Cloudflare CAPTCHA"
+
+    elif server.startswith("ddos-guard"):
+        if response.status_code == 403 and \
+                b"/ddos-guard/js-challenge/" in response.content:
+            return "DDoS-Guard challenge"
+
+
 @functools.lru_cache(maxsize=None)
 def git_head():
     try: