github/gallery-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[common] detect CF & DG challenges in request_json & _xml (#7833)

Browse Source 
even when called when fatal=False
This commit is contained in:
Mike Fährmann
2025-07-17 17:17:01 +02:00
parent dd09937d69
commit 629db56b12
1 changed files with 16 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
gallery_dl/extractor/common.py
View File

@@ -265,20 +265,27 @@ class Extractor():
return self.request(url, **kwargs).headers.get("location", "") return self.request(url, **kwargs).headers.get("location", "")
def request_json(self, url, **kwargs): def request_json(self, url, **kwargs):
response = self.request(url, **kwargs)
try: try:
return util.json_loads(self.request(url, **kwargs).text) return util.json_loads(response.text)
except Exception as exc: except Exception as exc:
fatal = kwargs.get("fatal", True) fatal = kwargs.get("fatal", True)
if not fatal or fatal is ...: if not fatal or fatal is ...:
self.log.warning("%s: %s", exc.__class__.__name__, exc) if challenge := util.detect_challenge(response):
self.log.warning(challenge)
else:
self.log.warning("%s: %s", exc.__class__.__name__, exc)
return {} return {}
raise raise
def request_xml(self, url, xmlns=True, **kwargs): def request_xml(self, url, xmlns=True, **kwargs):
text = self.request(url, **kwargs).text response = self.request(url, **kwargs)
if not xmlns: if xmlns:
text = text.replace(" xmlns=", " ns=") text = response.text
else:
text = response.text.replace(" xmlns=", " ns=")
parser = ElementTree.XMLParser() parser = ElementTree.XMLParser()
try: try:
@@ -287,7 +294,10 @@ class Extractor():
except Exception as exc: except Exception as exc:
fatal = kwargs.get("fatal", True) fatal = kwargs.get("fatal", True)
if not fatal or fatal is ...: if not fatal or fatal is ...:
self.log.warning("%s: %s", exc.__class__.__name__, exc) if challenge := util.detect_challenge(response):
self.log.warning(challenge)
else:
self.log.warning("%s: %s", exc.__class__.__name__, exc)
return ElementTree.Element("") return ElementTree.Element("")
raise raise