[common] implement '"user-agent": "@BROWSER"' (#7947)
support automatically fetching the 'User-Agent' header of a specific installed browser, e.g. "@firefox"
This commit is contained in:
@@ -696,12 +696,19 @@ Default
|
||||
* ``"Patreon/72.2.28 (Android; Android 14; Scale/2.10)"``: ``patreon``
|
||||
* ``"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/LATEST.0.0.0 Safari/537.36"``: ``instagram``
|
||||
* ``"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:LATEST) Gecko/20100101 Firefox/LATEST"``: otherwise
|
||||
Example
|
||||
* ``"curl/8.14.1"``
|
||||
* ``"browser"``
|
||||
* ``"@chrome"``
|
||||
Description
|
||||
User-Agent header value used for HTTP requests.
|
||||
|
||||
Setting this value to ``"browser"`` will try to automatically detect
|
||||
and use the ``User-Agent`` header of the system's default browser.
|
||||
|
||||
Setting this value to ``"@BROWSER"``, e.g. ``"@chrome"``, will try to automatically detect
|
||||
and use the ``User-Agent`` header of this installed browser.
|
||||
|
||||
|
||||
extractor.*.browser
|
||||
-------------------
|
||||
|
||||
@@ -464,7 +464,9 @@ class Extractor():
|
||||
if custom_ua is None or custom_ua == "auto":
|
||||
pass
|
||||
elif custom_ua == "browser":
|
||||
headers["User-Agent"] = _browser_useragent()
|
||||
headers["User-Agent"] = _browser_useragent(None)
|
||||
elif custom_ua[0] == "@":
|
||||
headers["User-Agent"] = _browser_useragent(custom_ua[1:])
|
||||
elif self.useragent is Extractor.useragent and not self.browser or \
|
||||
custom_ua is not config.get(("extractor",), "user-agent"):
|
||||
headers["User-Agent"] = custom_ua
|
||||
@@ -1042,19 +1044,20 @@ def _build_requests_adapter(
|
||||
return adapter
|
||||
|
||||
|
||||
@cache.cache(maxage=86400)
|
||||
def _browser_useragent():
|
||||
@cache.cache(maxage=86400, keyarg=0)
|
||||
def _browser_useragent(name):
|
||||
"""Get User-Agent header from default browser"""
|
||||
import webbrowser
|
||||
import socket
|
||||
browser = webbrowser.get(name)
|
||||
|
||||
import socket
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
server.bind(("127.0.0.1", 0))
|
||||
server.listen(1)
|
||||
|
||||
host, port = server.getsockname()
|
||||
webbrowser.open(f"http://{host}:{port}/user-agent")
|
||||
browser.open(f"http://{host}:{port}/user-agent")
|
||||
|
||||
client = server.accept()[0]
|
||||
server.close()
|
||||
|
||||
Reference in New Issue
Block a user