[downloader:http] change '_http_retry' to accept a Python function

and rename '_http_retry_codes' to '_http_retry'

(#3569)
This commit is contained in:
Mike Fährmann
2023-03-09 23:30:15 +01:00
parent 175822e065
commit 67ec91cdbd
2 changed files with 9 additions and 12 deletions

View File

@@ -59,10 +59,7 @@ class NitterExtractor(BaseExtractor):
if url[0] == "/":
url = self.root + url
file = {
"url": url,
"_http_retry_codes": (404,),
}
file = {"url": url, "_http_retry": _retry_on_404}
file["filename"], _, file["extension"] = \
name.rpartition(".")
append(file)
@@ -468,3 +465,7 @@ class NitterTweetExtractor(NitterExtractor):
quoted["user"] = tweet["user"]
return (tweet, quoted)
return (tweet,)
def _retry_on_404(response):
return response.status_code == 404