From de20cadc68aab54327e07a8d1884d48eff3ba51b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mike=20F=C3=A4hrmann?= <mike_faehrmann@web.de>
Date: Wed, 29 Jun 2022 13:22:59 +0200
Subject: [PATCH] add 'brotli' as optional dependency (#2716)

only send 'Accept-Encoding: br' if supported
---
 README.rst                     |  3 +++
 gallery_dl/extractor/common.py | 15 +++++++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index cd6e9ae1..5ba5bf88 100644
--- a/README.rst
+++ b/README.rst
@@ -25,6 +25,7 @@ Optional
 - FFmpeg_: Pixiv Ugoira to WebM conversion
 - yt-dlp_ or youtube-dl_: Video downloads
 - PySocks_: SOCKS proxy support
+- brotli_ or brotlicffi_: Brotli compression support
 
 
 Installation
@@ -332,6 +333,8 @@ To authenticate with a ``mastodon`` instance, run *gallery-dl* with
 .. _yt-dlp:     https://github.com/yt-dlp/yt-dlp
 .. _youtube-dl: https://ytdl-org.github.io/youtube-dl/
 .. _PySocks:    https://pypi.org/project/PySocks/
+.. _brotli:     https://github.com/google/brotli
+.. _brotlicffi: https://github.com/python-hyper/brotlicffi
 .. _pyOpenSSL:  https://pyopenssl.org/
 .. _Snapd:      https://docs.snapcraft.io/installing-snapd
 .. _OAuth:      https://en.wikipedia.org/wiki/OAuth
diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py
index 5c5e29ec..eab2900d 100644
--- a/gallery_dl/extractor/common.py
+++ b/gallery_dl/extractor/common.py
@@ -259,6 +259,10 @@ class Extractor():
                 "rv:102.0) Gecko/20100101 Firefox/102.0"))
             headers["Accept"] = "*/*"
             headers["Accept-Language"] = "en-US,en;q=0.5"
+
+        if BROTLI:
+            headers["Accept-Encoding"] = "gzip, deflate, br"
+        else:
             headers["Accept-Encoding"] = "gzip, deflate"
 
         custom_headers = self.config("headers")
@@ -718,7 +722,7 @@ HTTP_HEADERS = {
         ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,"
                    "image/avif,image/webp,*/*;q=0.8"),
         ("Accept-Language", "en-US,en;q=0.5"),
-        ("Accept-Encoding", "gzip, deflate, br"),
+        ("Accept-Encoding", None),
         ("Referer", None),
         ("DNT", "1"),
         ("Connection", "keep-alive"),
@@ -736,7 +740,7 @@ HTTP_HEADERS = {
         ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,"
                    "image/webp,image/apng,*/*;q=0.8"),
         ("Referer", None),
-        ("Accept-Encoding", "gzip, deflate"),
+        ("Accept-Encoding", None),
         ("Accept-Language", "en-US,en;q=0.9"),
         ("Cookie", None),
     ),
@@ -783,6 +787,13 @@ SSL_CIPHERS = {
 }
 
 
+# detect brotli support
+try:
+    BROTLI = requests.packages.urllib3.response.brotli is not None
+except AttributeError:
+    BROTLI = False
+
+
 # Undo automatic pyOpenSSL injection by requests
 pyopenssl = config.get((), "pyopenssl", False)
 if not pyopenssl: