[common] add '--xff' / 'geo-bypass' option

This commit is contained in:
Mike Fährmann
2026-02-04 18:23:03 +01:00
parent b329e6e472
commit 947085c6e3
6 changed files with 324 additions and 6 deletions

View File

@@ -48,6 +48,7 @@ class Extractor():
tls12 = True
browser = None
useragent = util.USERAGENT_FIREFOX
geobypass = None
request_interval = 0.0
request_interval_min = 0.0
request_interval_429 = 60.0
@@ -517,6 +518,17 @@ class Extractor():
custom_ua is not config.get(("extractor",), "user-agent"):
headers["User-Agent"] = custom_ua
custom_xff = self.config("geo-bypass")
if custom_xff is None or custom_xff == "auto":
custom_xff = self.geobypass
if custom_xff is not None:
if ip := self.utils("/geo").random_ipv4(custom_xff):
headers["X-Forwarded-For"] = ip
self.log.debug("Using fake IP %s as 'X-Forwarded-For'", ip)
else:
self.log.warning("xff: Invalid ISO 3166 country code '%s'",
custom_xff)
if custom_headers := self.config("headers"):
if isinstance(custom_headers, str):
if custom_headers in HEADERS: