refactor proxy handling code (#2357)

- allow gallery-dl proxy settings to overwrite environment proxies
- allow specifying different proxies for data extraction and download
  - add 'downloader.proxy' option
  - '-o extractor.proxy=–PROXY_URL -o downloader.proxy=null'
    now has the same effect as youtube-dl's '--geo-verification-proxy'

gallery_dl/downloader/common.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2014-2020 Mike Fährmann
+# Copyright 2014-2022 Mike Fährmann
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -27,6 +27,12 @@ class DownloaderBase():
             self.partdir = util.expand_path(self.partdir)
             os.makedirs(self.partdir, exist_ok=True)
 
+        proxies = self.config("proxy", util.SENTINEL)
+        if proxies is util.SENTINEL:
+            self.proxies = job.extractor._proxies
+        else:
+            self.proxies = util.build_proxy_map(proxies, self.log)
+
     def config(self, key, default=None):
         """Interpolate downloader config value for 'key'"""
         return config.interpolate(("downloader", self.scheme), key, default)

gallery_dl/downloader/http.py

@@ -121,7 +121,8 @@ class HttpDownloader(DownloaderBase):
             try:
                 response = self.session.request(
                     "GET", url, stream=True, headers=headers,
-                    timeout=self.timeout, verify=self.verify)
+                    timeout=self.timeout, verify=self.verify,
+                    proxies=self.proxies)
             except (ConnectionError, Timeout) as exc:
                 msg = str(exc)
                 continue

gallery_dl/downloader/ytdl.py

@@ -25,6 +25,7 @@ class YoutubeDLDownloader(DownloaderBase):
             "retries": retries+1 if retries >= 0 else float("inf"),
             "socket_timeout": self.config("timeout", extractor._timeout),
             "nocheckcertificate": not self.config("verify", extractor._verify),
+            "proxy": self.proxies.get("http") if self.proxies else None,
         }
 
         self.ytdl_instance = None