From 629db56b120d1d7f06ec30cb95e7cb3aaa33f751 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Thu, 17 Jul 2025 17:17:01 +0200
Subject: [PATCH] [common] detect CF & DG challenges in request_json & _xml
 (#7833)

even when called when fatal=False
---
 gallery_dl/extractor/common.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py
index b4d9ddbb..4f7d4751 100644
--- a/gallery_dl/extractor/common.py
+++ b/gallery_dl/extractor/common.py
@@ -265,20 +265,27 @@ class Extractor():
         return self.request(url, **kwargs).headers.get("location", "")
 
     def request_json(self, url, **kwargs):
+        response = self.request(url, **kwargs)
+
         try:
-            return util.json_loads(self.request(url, **kwargs).text)
+            return util.json_loads(response.text)
         except Exception as exc:
             fatal = kwargs.get("fatal", True)
             if not fatal or fatal is ...:
-                self.log.warning("%s: %s", exc.__class__.__name__, exc)
+                if challenge := util.detect_challenge(response):
+                    self.log.warning(challenge)
+                else:
+                    self.log.warning("%s: %s", exc.__class__.__name__, exc)
                 return {}
             raise
 
     def request_xml(self, url, xmlns=True, **kwargs):
-        text = self.request(url, **kwargs).text
+        response = self.request(url, **kwargs)
 
-        if not xmlns:
-            text = text.replace(" xmlns=", " ns=")
+        if xmlns:
+            text = response.text
+        else:
+            text = response.text.replace(" xmlns=", " ns=")
 
         parser = ElementTree.XMLParser()
         try:
@@ -287,7 +294,10 @@ class Extractor():
         except Exception as exc:
             fatal = kwargs.get("fatal", True)
             if not fatal or fatal is ...:
-                self.log.warning("%s: %s", exc.__class__.__name__, exc)
+                if challenge := util.detect_challenge(response):
+                    self.log.warning(challenge)
+                else:
+                    self.log.warning("%s: %s", exc.__class__.__name__, exc)
                 return ElementTree.Element("")
             raise