Redact credentials in urllib3's DEBUG request logs

morpc/req.py

@@ -25,6 +25,20 @@ def redact(value):
     return _SENSITIVE_PATTERN.sub(r'\1REDACTED', str(value))
 
 
+class _RedactFilter(logging.Filter):
+    """Redact credentials in log records. urllib3 logs every request line, query string included, at DEBUG."""
+
+    def filter(self, record):
+        if isinstance(record.msg, str):
+            record.msg = redact(record.msg)
+        if isinstance(record.args, tuple):
+            record.args = tuple(redact(a) if isinstance(a, str) else a for a in record.args)
+        return True
+
+
+logging.getLogger("urllib3.connectionpool").addFilter(_RedactFilter())
+
+
 default_headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"}
 
 

tests/test_req.py

@@ -127,3 +127,16 @@ def test_redact_hides_credentials_in_urls_and_params():
     assert redact(f"http://x/y?a=1&key={SECRET}") == "http://x/y?a=1&key=REDACTED"
     assert redact({"get": "NAME", "key": SECRET}) == {"get": "NAME", "key": "REDACTED"}
     assert redact(None) is None
+
+
+def test_urllib3_debug_request_log_is_redacted(caplog):
+    # urllib3 logs every request line, including the query string, at DEBUG. Mirror its call.
+    import logging
+    import morpc.req  # noqa: F401  (installs the filter)
+    caplog.set_level("DEBUG")
+    logging.getLogger("urllib3.connectionpool").debug(
+        '%s://%s:%s "%s %s %s" %s %s', "https", "api.census.gov", 443, "GET", f"/data/2024/acs/acs5?get=NAME&key={SECRET}", "HTTP/1.1", 200, None
+    )
+    assert "api.census.gov" in caplog.text
+    assert "key=REDACTED" in caplog.text
+    assert SECRET not in caplog.text