From b0d7de36032189f242e5bc607ea3ea9f74290f14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= Date: Fri, 20 Jun 2025 19:55:01 +0200 Subject: [PATCH] support using system certificates via 'truststore' (#6582) https://github.com/mikf/gallery-dl/issues/6582#issuecomment-2989290495 --- .github/workflows/executables.yml | 1 + README.rst | 6 +++-- docs/configuration.rst | 16 +++++++++++++ docs/gallery-dl.conf | 1 + gallery_dl/extractor/common.py | 38 +++++++++++++++++++++++-------- setup.py | 1 + 6 files changed, 52 insertions(+), 11 deletions(-) diff --git a/.github/workflows/executables.yml b/.github/workflows/executables.yml index 4b924a1c..d4e0f884 100644 --- a/.github/workflows/executables.yml +++ b/.github/workflows/executables.yml @@ -72,6 +72,7 @@ jobs: - name: Build executable run: | pip install requests requests[socks] yt-dlp[default] pyyaml ${{ matrix.python-packages }} pyinstaller + pip install truststore || true python ./scripts/pyinstaller.py --label '${{ env.LABEL }}' - uses: actions/upload-artifact@v4 diff --git a/README.rst b/README.rst index 1fbdce50..30df4ea9 100644 --- a/README.rst +++ b/README.rst @@ -35,6 +35,7 @@ Optional - toml_: TOML configuration file support for Python<3.11 - SecretStorage_: GNOME keyring passwords for ``--cookies-from-browser`` - Psycopg_: PostgreSQL archive support +- truststore_: Native system certificate stores Installation @@ -461,7 +462,7 @@ To authenticate with a ``mastodon`` instance, run *gallery-dl* with .. _Python: https://www.python.org/downloads/ .. _PyPI: https://pypi.org/ .. _pip: https://pip.pypa.io/en/stable/ -.. _Requests: https://requests.readthedocs.io/en/master/ +.. _Requests: https://requests.readthedocs.io/en/latest/ .. _FFmpeg: https://www.ffmpeg.org/ .. _mkvmerge: https://www.matroska.org/downloads/mkvtoolnix.html .. _yt-dlp: https://github.com/yt-dlp/yt-dlp @@ -474,10 +475,11 @@ To authenticate with a ``mastodon`` instance, run *gallery-dl* with .. _toml: https://pypi.org/project/toml/ .. _SecretStorage: https://pypi.org/project/SecretStorage/ .. _Psycopg: https://www.psycopg.org/ +.. _truststore: https://truststore.readthedocs.io/en/latest/ .. _Snapd: https://docs.snapcraft.io/installing-snapd .. _OAuth: https://en.wikipedia.org/wiki/OAuth .. _Chocolatey: https://chocolatey.org/install -.. _Scoop: https://scoop.sh +.. _Scoop: https://scoop.sh/ .. |pypi| image:: https://img.shields.io/pypi/v/gallery-dl.svg :target: https://pypi.org/project/gallery-dl/ diff --git a/docs/configuration.rst b/docs/configuration.rst index f8b9be28..4e4aae71 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1238,6 +1238,22 @@ Description |requests.request()|_ method. +extractor.*.truststore +---------------------- +Type + ``bool`` +Default + ``false`` +Description + | Use a + `truststore `__ + ``SSLContext`` for verifying SSL/TLS certificates + | to make use of your system's native certificate stores + instead of relying on + `certifi `__ + certificates. + + extractor.*.download -------------------- Type diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 4d6cecef..1bb50e3c 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -28,6 +28,7 @@ "retry-codes" : [], "timeout" : 30.0, "verify" : true, + "truststore" : false, "download" : true, "fallback" : true, diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 7e636e76..1046591e 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -485,8 +485,17 @@ class Extractor(): ssl_options |= ssl.OP_NO_TLSv1_2 self.log.debug("TLS 1.2 disabled.") + if self.config("truststore"): + try: + from truststore import SSLContext as ssl_ctx + except ImportError as exc: + self.log.error("%s: %s", exc.__class__.__name__, exc) + ssl_ctx = None + else: + ssl_ctx = None + adapter = _build_requests_adapter( - ssl_options, ssl_ciphers, source_address) + ssl_options, ssl_ciphers, ssl_ctx, source_address) session.mount("https://", adapter) session.mount("http://", adapter) @@ -979,19 +988,30 @@ class RequestsAdapter(HTTPAdapter): return HTTPAdapter.proxy_manager_for(self, *args, **kwargs) -def _build_requests_adapter(ssl_options, ssl_ciphers, source_address): - key = (ssl_options, ssl_ciphers, source_address) +def _build_requests_adapter( + ssl_options, ssl_ciphers, ssl_ctx, source_address): + + key = (ssl_options, ssl_ciphers, ssl_ctx, source_address) try: return _adapter_cache[key] except KeyError: pass - if ssl_options or ssl_ciphers: - ssl_context = urllib3.connection.create_urllib3_context( - options=ssl_options or None, ciphers=ssl_ciphers) - if not requests.__version__ < "2.32": - # https://github.com/psf/requests/pull/6731 - ssl_context.load_verify_locations(requests.certs.where()) + if ssl_options or ssl_ciphers or ssl_ctx: + if ssl_ctx is None: + ssl_context = urllib3.connection.create_urllib3_context( + options=ssl_options or None, ciphers=ssl_ciphers) + if not requests.__version__ < "2.32": + # https://github.com/psf/requests/pull/6731 + ssl_context.load_verify_locations(requests.certs.where()) + else: + ssl_ctx_orig = urllib3.util.ssl_.SSLContext + try: + urllib3.util.ssl_.SSLContext = ssl_ctx + ssl_context = urllib3.connection.create_urllib3_context( + options=ssl_options or None, ciphers=ssl_ciphers) + finally: + urllib3.util.ssl_.SSLContext = ssl_ctx_orig ssl_context.check_hostname = False else: ssl_context = None diff --git a/setup.py b/setup.py index 83e61b9c..c52d1d74 100644 --- a/setup.py +++ b/setup.py @@ -113,6 +113,7 @@ def build_setuptools(): "yt-dlp[default]", "pyyaml", "toml; python_version < '3.11'", + "truststore; python_version >= '3.10'", "secretstorage; sys_platform == 'linux'", ], },