[zerochan] improve redirect handling, add 'redirects' option (#5891)
This commit is contained in:
@@ -4642,6 +4642,16 @@ Description
|
|||||||
(limited to 100 pages * 24 posts)
|
(limited to 100 pages * 24 posts)
|
||||||
|
|
||||||
|
|
||||||
|
extractor.zerochan.redirects
|
||||||
|
----------------------------
|
||||||
|
Type
|
||||||
|
``bool``
|
||||||
|
Default
|
||||||
|
``false``
|
||||||
|
Description
|
||||||
|
Automatically follow tag redirects.
|
||||||
|
|
||||||
|
|
||||||
extractor.[booru].tags
|
extractor.[booru].tags
|
||||||
----------------------
|
----------------------
|
||||||
Type
|
Type
|
||||||
|
|||||||
@@ -198,11 +198,15 @@ class ZerochanTagExtractor(ZerochanExtractor):
|
|||||||
|
|
||||||
while True:
|
while True:
|
||||||
response = self.request(url, params=params, allow_redirects=False)
|
response = self.request(url, params=params, allow_redirects=False)
|
||||||
|
|
||||||
if response.status_code >= 300:
|
if response.status_code >= 300:
|
||||||
url = text.urljoin(self.root, response.headers["location"])
|
url = text.urljoin(self.root, response.headers["location"])
|
||||||
response = self.request(url, params=params)
|
self.log.warning("HTTP redirect to %s", url)
|
||||||
data = response.json()
|
if self.config("redirects"):
|
||||||
|
continue
|
||||||
|
raise exception.StopExtraction()
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
try:
|
try:
|
||||||
posts = data["items"]
|
posts = data["items"]
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
Reference in New Issue
Block a user