From 947085c6e3c4e0aad8daf016b07da9ff5d54c7ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Wed, 4 Feb 2026 18:23:03 +0100 Subject: [PATCH] [common] add '--xff' / 'geo-bypass' option --- docs/configuration.rst | 20 +++ docs/gallery-dl.conf | 1 + docs/options.md | 6 +- gallery_dl/extractor/common.py | 12 ++ gallery_dl/extractor/utils/geo.py | 274 ++++++++++++++++++++++++++++++ gallery_dl/option.py | 17 +- 6 files changed, 324 insertions(+), 6 deletions(-) create mode 100644 gallery_dl/extractor/utils/geo.py diff --git a/docs/configuration.rst b/docs/configuration.rst index a0d2bd7e..3ef0ab0e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -970,6 +970,26 @@ Description instead of the extractor's ``root`` domain. +extractor.*.geo-bypass +---------------------- +Type + ``string`` +Default + ``"auto"`` +Example + * ``"JP"`` + * ``"105.48.0.0/12"`` +Description + Use fake IPs as + `X-Forwarded-For `__ + header to try bypassing geographic restrictions. + + | Can be either an + `ISO 3166-2 `__ + country code + | or an IP block in CIDR notation. + + extractor.*.headers ------------------- Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 4f1a4d4e..73782f31 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -21,6 +21,7 @@ "ciphers" : null, "tls12" : true, "browser" : null, + "geo-bypass" : null, "proxy" : null, "proxy-env" : true, "source-address": null, diff --git a/docs/options.md b/docs/options.md index 8f71d06d..861563f4 100644 --- a/docs/options.md +++ b/docs/options.md @@ -25,7 +25,6 @@ -d, --destination PATH Target location for file downloads -D, --directory PATH Exact location for file downloads -X, --extractors PATH Load external extractors from PATH - -a, --user-agent UA User-Agent request header --clear-cache MODULE Delete cached login sessions, cookies, etc. for MODULE (ALL to delete everything) --compat Restore legacy 'category' names @@ -90,8 +89,13 @@ -R, --retries N Maximum number of retries for failed HTTP requests or -1 for infinite retries (default: 4) + -a, --user-agent UA User-Agent request header --http-timeout SECONDS Timeout for HTTP connections (default: 30.0) --proxy URL Use the specified proxy + --xff VALUE Use a fake 'X-Forwarded-For' HTTP header to try + bypassing geographic restrictions. Can be an IP + block in CIDR notation or a two-letter ISO + 3166-2 country code --source-address IP Client-side IP address to bind to -4, --force-ipv4 Make all connections via IPv4 -6, --force-ipv6 Make all connections via IPv6 diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 6b80529f..89a56875 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -48,6 +48,7 @@ class Extractor(): tls12 = True browser = None useragent = util.USERAGENT_FIREFOX + geobypass = None request_interval = 0.0 request_interval_min = 0.0 request_interval_429 = 60.0 @@ -517,6 +518,17 @@ class Extractor(): custom_ua is not config.get(("extractor",), "user-agent"): headers["User-Agent"] = custom_ua + custom_xff = self.config("geo-bypass") + if custom_xff is None or custom_xff == "auto": + custom_xff = self.geobypass + if custom_xff is not None: + if ip := self.utils("/geo").random_ipv4(custom_xff): + headers["X-Forwarded-For"] = ip + self.log.debug("Using fake IP %s as 'X-Forwarded-For'", ip) + else: + self.log.warning("xff: Invalid ISO 3166 country code '%s'", + custom_xff) + if custom_headers := self.config("headers"): if isinstance(custom_headers, str): if custom_headers in HEADERS: diff --git a/gallery_dl/extractor/utils/geo.py b/gallery_dl/extractor/utils/geo.py new file mode 100644 index 00000000..be966fde --- /dev/null +++ b/gallery_dl/extractor/utils/geo.py @@ -0,0 +1,274 @@ +# -*- coding: utf-8 -*- + +# Copyright 2026 Mike Fährmann +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. + +# Adapted from yt-dlp. +# https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/utils/_utils.py + +import random +import socket +import struct + + +COUNTRY_IP_MAP = { + "AD": "46.172.224.0/19", + "AE": "94.200.0.0/13", + "AF": "149.54.0.0/17", + "AG": "209.59.64.0/18", + "AI": "204.14.248.0/21", + "AL": "46.99.0.0/16", + "AM": "46.70.0.0/15", + "AO": "105.168.0.0/13", + "AP": "182.50.184.0/21", + "AQ": "23.154.160.0/24", + "AR": "181.0.0.0/12", + "AS": "202.70.112.0/20", + "AT": "77.116.0.0/14", + "AU": "1.128.0.0/11", + "AW": "181.41.0.0/18", + "AX": "185.217.4.0/22", + "AZ": "5.197.0.0/16", + "BA": "31.176.128.0/17", + "BB": "65.48.128.0/17", + "BD": "114.130.0.0/16", + "BE": "57.0.0.0/8", + "BF": "102.178.0.0/15", + "BG": "95.42.0.0/15", + "BH": "37.131.0.0/17", + "BI": "154.117.192.0/18", + "BJ": "137.255.0.0/16", + "BL": "185.212.72.0/23", + "BM": "196.12.64.0/18", + "BN": "156.31.0.0/16", + "BO": "161.56.0.0/16", + "BQ": "161.0.80.0/20", + "BR": "191.128.0.0/12", + "BS": "24.51.64.0/18", + "BT": "119.2.96.0/19", + "BW": "168.167.0.0/16", + "BY": "178.120.0.0/13", + "BZ": "179.42.192.0/18", + "CA": "99.224.0.0/11", + "CD": "41.243.0.0/16", + "CF": "197.242.176.0/21", + "CG": "160.113.0.0/16", + "CH": "85.0.0.0/13", + "CI": "102.136.0.0/14", + "CK": "202.65.32.0/19", + "CL": "152.172.0.0/14", + "CM": "102.244.0.0/14", + "CN": "36.128.0.0/10", + "CO": "181.240.0.0/12", + "CR": "201.192.0.0/12", + "CU": "152.206.0.0/15", + "CV": "165.90.96.0/19", + "CW": "190.88.128.0/17", + "CY": "31.153.0.0/16", + "CZ": "88.100.0.0/14", + "DE": "53.0.0.0/8", + "DJ": "197.241.0.0/17", + "DK": "87.48.0.0/12", + "DM": "192.243.48.0/20", + "DO": "152.166.0.0/15", + "DZ": "41.96.0.0/12", + "EC": "186.68.0.0/15", + "EE": "90.190.0.0/15", + "EG": "156.160.0.0/11", + "ER": "196.200.96.0/20", + "ES": "88.0.0.0/11", + "ET": "196.188.0.0/14", + "EU": "2.16.0.0/13", + "FI": "91.152.0.0/13", + "FJ": "144.120.0.0/16", + "FK": "80.73.208.0/21", + "FM": "119.252.112.0/20", + "FO": "88.85.32.0/19", + "FR": "90.0.0.0/9", + "GA": "41.158.0.0/15", + "GB": "25.0.0.0/8", + "GD": "74.122.88.0/21", + "GE": "31.146.0.0/16", + "GF": "161.22.64.0/18", + "GG": "62.68.160.0/19", + "GH": "154.160.0.0/12", + "GI": "95.164.0.0/16", + "GL": "88.83.0.0/19", + "GM": "160.182.0.0/15", + "GN": "197.149.192.0/18", + "GP": "104.250.0.0/19", + "GQ": "105.235.224.0/20", + "GR": "94.64.0.0/13", + "GT": "168.234.0.0/16", + "GU": "168.123.0.0/16", + "GW": "197.214.80.0/20", + "GY": "181.41.64.0/18", + "HK": "113.252.0.0/14", + "HN": "181.210.0.0/16", + "HR": "93.136.0.0/13", + "HT": "148.102.128.0/17", + "HU": "84.0.0.0/14", + "ID": "39.192.0.0/10", + "IE": "87.32.0.0/12", + "IL": "79.176.0.0/13", + "IM": "5.62.80.0/20", + "IN": "117.192.0.0/10", + "IO": "203.83.48.0/21", + "IQ": "37.236.0.0/14", + "IR": "2.176.0.0/12", + "IS": "82.221.0.0/16", + "IT": "79.0.0.0/10", + "JE": "87.244.64.0/18", + "JM": "72.27.0.0/17", + "JO": "176.29.0.0/16", + "JP": "133.0.0.0/8", + "KE": "105.48.0.0/12", + "KG": "158.181.128.0/17", + "KH": "36.37.128.0/17", + "KI": "103.25.140.0/22", + "KM": "197.255.224.0/20", + "KN": "198.167.192.0/19", + "KP": "175.45.176.0/22", + "KR": "175.192.0.0/10", + "KW": "37.36.0.0/14", + "KY": "64.96.0.0/15", + "KZ": "2.72.0.0/13", + "LA": "115.84.64.0/18", + "LB": "178.135.0.0/16", + "LC": "24.92.144.0/20", + "LI": "82.117.0.0/19", + "LK": "112.134.0.0/15", + "LR": "102.183.0.0/16", + "LS": "129.232.0.0/17", + "LT": "78.56.0.0/13", + "LU": "188.42.0.0/16", + "LV": "46.109.0.0/16", + "LY": "41.252.0.0/14", + "MA": "105.128.0.0/11", + "MC": "88.209.64.0/18", + "MD": "37.246.0.0/16", + "ME": "178.175.0.0/17", + "MF": "74.112.232.0/21", + "MG": "154.126.0.0/17", + "MH": "117.103.88.0/21", + "MK": "77.28.0.0/15", + "ML": "154.118.128.0/18", + "MM": "37.111.0.0/17", + "MN": "49.0.128.0/17", + "MO": "60.246.0.0/16", + "MP": "202.88.64.0/20", + "MQ": "109.203.224.0/19", + "MR": "41.188.64.0/18", + "MS": "208.90.112.0/22", + "MT": "46.11.0.0/16", + "MU": "105.16.0.0/12", + "MV": "27.114.128.0/18", + "MW": "102.70.0.0/15", + "MX": "187.192.0.0/11", + "MY": "175.136.0.0/13", + "MZ": "197.218.0.0/15", + "NA": "41.182.0.0/16", + "NC": "101.101.0.0/18", + "NE": "197.214.0.0/18", + "NF": "203.17.240.0/22", + "NG": "105.112.0.0/12", + "NI": "186.76.0.0/15", + "NL": "145.96.0.0/11", + "NO": "84.208.0.0/13", + "NP": "36.252.0.0/15", + "NR": "203.98.224.0/19", + "NU": "49.156.48.0/22", + "NZ": "49.224.0.0/14", + "OM": "5.36.0.0/15", + "PA": "186.72.0.0/15", + "PE": "186.160.0.0/14", + "PF": "123.50.64.0/18", + "PG": "124.240.192.0/19", + "PH": "49.144.0.0/13", + "PK": "39.32.0.0/11", + "PL": "83.0.0.0/11", + "PM": "70.36.0.0/20", + "PR": "66.50.0.0/16", + "PS": "188.161.0.0/16", + "PT": "85.240.0.0/13", + "PW": "202.124.224.0/20", + "PY": "181.120.0.0/14", + "QA": "37.210.0.0/15", + "RE": "102.35.0.0/16", + "RO": "79.112.0.0/13", + "RS": "93.86.0.0/15", + "RU": "5.136.0.0/13", + "RW": "41.186.0.0/16", + "SA": "188.48.0.0/13", + "SB": "202.1.160.0/19", + "SC": "154.192.0.0/11", + "SD": "102.120.0.0/13", + "SE": "78.64.0.0/12", + "SG": "8.128.0.0/10", + "SI": "188.196.0.0/14", + "SK": "78.98.0.0/15", + "SL": "102.143.0.0/17", + "SM": "89.186.32.0/19", + "SN": "41.82.0.0/15", + "SO": "154.115.192.0/18", + "SR": "186.179.128.0/17", + "SS": "105.235.208.0/21", + "ST": "197.159.160.0/19", + "SV": "168.243.0.0/16", + "SX": "190.102.0.0/20", + "SY": "5.0.0.0/16", + "SZ": "41.84.224.0/19", + "TC": "65.255.48.0/20", + "TD": "154.68.128.0/19", + "TG": "196.168.0.0/14", + "TH": "171.96.0.0/13", + "TJ": "85.9.128.0/18", + "TK": "27.96.24.0/21", + "TL": "180.189.160.0/20", + "TM": "95.85.96.0/19", + "TN": "197.0.0.0/11", + "TO": "175.176.144.0/21", + "TR": "78.160.0.0/11", + "TT": "186.44.0.0/15", + "TV": "202.2.96.0/19", + "TW": "120.96.0.0/11", + "TZ": "156.156.0.0/14", + "UA": "37.52.0.0/14", + "UG": "102.80.0.0/13", + "US": "6.0.0.0/8", + "UY": "167.56.0.0/13", + "UZ": "84.54.64.0/18", + "VA": "212.77.0.0/19", + "VC": "207.191.240.0/21", + "VE": "186.88.0.0/13", + "VG": "66.81.192.0/20", + "VI": "146.226.0.0/16", + "VN": "14.160.0.0/11", + "VU": "202.80.32.0/20", + "WF": "117.20.32.0/21", + "WS": "202.4.32.0/19", + "YE": "134.35.0.0/16", + "YT": "41.242.116.0/22", + "ZA": "41.0.0.0/11", + "ZM": "102.144.0.0/13", + "ZW": "102.177.192.0/18", +} + + +def random_ipv4(block): + if len(block) == 2: + block = COUNTRY_IP_MAP.get(block.upper()) + if not block: + return None + + addr, _, preflen = block.partition("/") + if not preflen: + return addr + + addr_min = struct.unpack("!L", socket.inet_aton(addr))[0] + addr_max = addr_min | (0xffffffff >> int(preflen)) + return str(socket.inet_ntoa(struct.pack( + "!L", random.randint(addr_min, addr_max)))) diff --git a/gallery_dl/option.py b/gallery_dl/option.py index 81e063d2..ff3a49ee 100644 --- a/gallery_dl/option.py +++ b/gallery_dl/option.py @@ -273,11 +273,6 @@ def build_parser(): dest="extractor_sources", metavar="PATH", action="append", help="Load external extractors from PATH", ) - general.add_argument( - "-a", "--user-agent", - dest="user-agent", metavar="UA", action=ConfigAction, - help="User-Agent request header", - ) general.add_argument( "--clear-cache", dest="clear_cache", metavar="MODULE", @@ -480,6 +475,11 @@ def build_parser(): help=("Maximum number of retries for failed HTTP requests " "or -1 for infinite retries (default: 4)"), ) + networking.add_argument( + "-a", "--user-agent", + dest="user-agent", metavar="UA", action=ConfigAction, + help="User-Agent request header", + ) networking.add_argument( "--http-timeout", dest="timeout", metavar="SECONDS", type=float, action=ConfigAction, @@ -490,6 +490,13 @@ def build_parser(): dest="proxy", metavar="URL", action=ConfigAction, help="Use the specified proxy", ) + networking.add_argument( + "--xff", + dest="geo-bypass", metavar="VALUE", action=ConfigAction, + help=("Use a fake 'X-Forwarded-For' HTTP header to try bypassing " + "geographic restrictions. Can be an IP block in CIDR notation " + "or a two-letter ISO 3166-2 country code") + ) networking.add_argument( "--source-address", dest="source-address", metavar="IP", action=ConfigAction,