move dump_response() into a separate function (#737)

gallery_dl/extractor/common.py

@@ -343,43 +343,14 @@ class Extractor():
             Extractor._dump_index = 1
             Extractor._dump_sanitize = re.compile(r"[\\\\|/<>:\"?*&=#]+").sub
 
-        outfmt = """\
-{request.method} {request.url}
-Status: {response.status_code} {response.reason}
-
-Request Headers
----------------
-{request_headers}
-
-Response Headers
-----------------
-{response_headers}
-
-Content
--------
-"""
         fname = "{:>02}_{}".format(
             Extractor._dump_index,
             Extractor._dump_sanitize('_', response.url)
         )[:250]
 
-        headers = outfmt.format(
-            request=response.request,
-            response=response,
-            request_headers=("\n".join(
-                name + ": " + value
-                for name, value in response.request.headers.items()
-            )),
-            response_headers=("\n".join(
-                name + ": " + value
-                for name, value in response.headers.items()
-            )),
-        )
-
         try:
             with open(fname + ".dump", 'wb') as fp:
-                fp.write(headers.encode())
+                util.dump_response(response, fp)
-                fp.write(response.content)
         except Exception as e:
             self.log.warning("Failed to dump HTTP request (%s: %s)",
                              e.__class__.__name__, e)

gallery_dl/util.py

@@ -113,6 +113,57 @@ def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4):
     fp.write("\n")
 
 
+def dump_response(response, fp=sys.stdout,
+                  headers=True, content=True, hide_auth=True):
+    """Write the contents of 'response' into a file-like object"""
+
+    if headers:
+        request = response.request
+        req_headers = request.headers.copy()
+        outfmt = """\
+{request.method} {request.url}
+Status: {response.status_code} {response.reason}
+
+Request Headers
+---------------
+{request_headers}
+
+Response Headers
+----------------
+{response_headers}
+"""
+        if hide_auth:
+            authorization = req_headers.get("Authorization")
+            if authorization:
+                atype, sep, _ = authorization.partition(" ")
+                req_headers["Authorization"] = atype + " ***" if sep else "***"
+
+            cookies = req_headers.get("Cookie")
+            if cookies:
+                req_headers["Cookie"] = ";".join(
+                    cookie.partition("=")[0] + "=***"
+                    for cookie in cookies.split(";")
+                )
+
+        fp.write(outfmt.format(
+            request=request,
+            response=response,
+            request_headers="\n".join(
+                name + ": " + value
+                for name, value in req_headers.items()
+            ),
+            response_headers="\n".join(
+                name + ": " + value
+                for name, value in response.headers.items()
+            ),
+        ).encode())
+
+    if content:
+        if headers:
+            fp.write(b"\nContent\n-------\n")
+        fp.write(response.content)
+
+
 def expand_path(path):
     """Expand environment variables and tildes (~)"""
     if not path: