support using system certificates via 'truststore' (#6582)

https://github.com/mikf/gallery-dl/issues/6582#issuecomment-2989290495

.github/workflows/executables.yml

@@ -72,6 +72,7 @@ jobs:
     - name: Build executable
       run: |
         pip install requests requests[socks] yt-dlp[default] pyyaml ${{ matrix.python-packages }} pyinstaller
+        pip install truststore || true
         python ./scripts/pyinstaller.py --label '${{ env.LABEL }}'
 
     - uses: actions/upload-artifact@v4

README.rst

@@ -35,6 +35,7 @@ Optional
 - toml_: TOML configuration file support for Python<3.11
 - SecretStorage_: GNOME keyring passwords for ``--cookies-from-browser``
 - Psycopg_: PostgreSQL archive support
+- truststore_: Native system certificate stores
 
 
 Installation
@@ -461,7 +462,7 @@ To authenticate with a ``mastodon`` instance, run *gallery-dl* with
 .. _Python:     https://www.python.org/downloads/
 .. _PyPI:       https://pypi.org/
 .. _pip:        https://pip.pypa.io/en/stable/
-.. _Requests:   https://requests.readthedocs.io/en/master/
+.. _Requests:   https://requests.readthedocs.io/en/latest/
 .. _FFmpeg:     https://www.ffmpeg.org/
 .. _mkvmerge:   https://www.matroska.org/downloads/mkvtoolnix.html
 .. _yt-dlp:     https://github.com/yt-dlp/yt-dlp
@@ -474,10 +475,11 @@ To authenticate with a ``mastodon`` instance, run *gallery-dl* with
 .. _toml:       https://pypi.org/project/toml/
 .. _SecretStorage: https://pypi.org/project/SecretStorage/
 .. _Psycopg:    https://www.psycopg.org/
+.. _truststore: https://truststore.readthedocs.io/en/latest/
 .. _Snapd:      https://docs.snapcraft.io/installing-snapd
 .. _OAuth:      https://en.wikipedia.org/wiki/OAuth
 .. _Chocolatey: https://chocolatey.org/install
-.. _Scoop:      https://scoop.sh
+.. _Scoop:      https://scoop.sh/
 
 .. |pypi| image:: https://img.shields.io/pypi/v/gallery-dl.svg
     :target: https://pypi.org/project/gallery-dl/

docs/configuration.rst

@@ -1238,6 +1238,22 @@ Description
     |requests.request()|_ method.
 
 
+extractor.*.truststore
+----------------------
+Type
+    ``bool``
+Default
+    ``false``
+Description
+    | Use a
+      `truststore <https://truststore.readthedocs.io/en/latest/>`__
+      ``SSLContext`` for verifying SSL/TLS certificates
+    | to make use of your system's native certificate stores
+      instead of relying on
+      `certifi <https://pypi.org/project/certifi/>`__
+      certificates.
+
+
 extractor.*.download
 --------------------
 Type

docs/gallery-dl.conf

@@ -28,6 +28,7 @@
         "retry-codes"   : [],
         "timeout"       : 30.0,
         "verify"        : true,
+        "truststore"    : false,
         "download"      : true,
         "fallback"      : true,
 

gallery_dl/extractor/common.py

@@ -485,8 +485,17 @@ class Extractor():
             ssl_options |= ssl.OP_NO_TLSv1_2
             self.log.debug("TLS 1.2 disabled.")
 
+        if self.config("truststore"):
+            try:
+                from truststore import SSLContext as ssl_ctx
+            except ImportError as exc:
+                self.log.error("%s: %s", exc.__class__.__name__, exc)
+                ssl_ctx = None
+        else:
+            ssl_ctx = None
+
         adapter = _build_requests_adapter(
-            ssl_options, ssl_ciphers, source_address)
+            ssl_options, ssl_ciphers, ssl_ctx, source_address)
         session.mount("https://", adapter)
         session.mount("http://", adapter)
 
@@ -979,19 +988,30 @@ class RequestsAdapter(HTTPAdapter):
         return HTTPAdapter.proxy_manager_for(self, *args, **kwargs)
 
 
-def _build_requests_adapter(ssl_options, ssl_ciphers, source_address):
-    key = (ssl_options, ssl_ciphers, source_address)
+def _build_requests_adapter(
+        ssl_options, ssl_ciphers, ssl_ctx, source_address):
+
+    key = (ssl_options, ssl_ciphers, ssl_ctx, source_address)
     try:
         return _adapter_cache[key]
     except KeyError:
         pass
 
-    if ssl_options or ssl_ciphers:
-        ssl_context = urllib3.connection.create_urllib3_context(
-            options=ssl_options or None, ciphers=ssl_ciphers)
-        if not requests.__version__ < "2.32":
-            # https://github.com/psf/requests/pull/6731
-            ssl_context.load_verify_locations(requests.certs.where())
+    if ssl_options or ssl_ciphers or ssl_ctx:
+        if ssl_ctx is None:
+            ssl_context = urllib3.connection.create_urllib3_context(
+                options=ssl_options or None, ciphers=ssl_ciphers)
+            if not requests.__version__ < "2.32":
+                # https://github.com/psf/requests/pull/6731
+                ssl_context.load_verify_locations(requests.certs.where())
+        else:
+            ssl_ctx_orig = urllib3.util.ssl_.SSLContext
+            try:
+                urllib3.util.ssl_.SSLContext = ssl_ctx
+                ssl_context = urllib3.connection.create_urllib3_context(
+                    options=ssl_options or None, ciphers=ssl_ciphers)
+            finally:
+                urllib3.util.ssl_.SSLContext = ssl_ctx_orig
         ssl_context.check_hostname = False
     else:
         ssl_context = None

setup.py

@@ -113,6 +113,7 @@ def build_setuptools():
                 "yt-dlp[default]",
                 "pyyaml",
                 "toml; python_version < '3.11'",
+                "truststore; python_version >= '3.10'",
                 "secretstorage; sys_platform == 'linux'",
             ],
         },