Inspect HTTPS request and response bodies with a trusted CA

README.md

@@ -297,3 +297,64 @@ dashboard's destination-header API follow its separate HTTP forwarding path.
 
 See [RUNBOOK.md](RUNBOOK.md) for release validation, local troubleshooting,
 CLI/container rollback, and the applicability of cost checks to this distributed tool.
+
+### Inspect HTTPS request and response bodies
+
+Netkit can decrypt HTTPS for **every destination** when clients explicitly trust
+its inspection CA. There is no domain allowlist. Without `--inspect-https`, CONNECT
+continues to work as an encrypted tunnel.
+
+Generate a CA once and keep it across proxy restarts:
+
+```sh
+mkdir -p "$HOME/.config/netkit"
+netkit ca --cert "$HOME/.config/netkit/ca.pem" --key "$HOME/.config/netkit/ca-key.pem"
+netkit serve --inspect-https \
+  --ca-cert "$HOME/.config/netkit/ca.pem" \
+  --ca-key "$HOME/.config/netkit/ca-key.pem"
+```
+
+The `ca` command refuses to overwrite existing files and creates the private key
+with mode `0600`. Only the public `ca.pem` belongs in client trust stores. The
+private key stays with Netkit. Existing signing CA PEM files can also be loaded;
+an invalid, expired, or mismatched CA fails startup.
+
+For Node.js, make the public certificate available to the application and restart
+its process with:
+
+```sh
+NODE_EXTRA_CA_CERTS=/path/to/ca.pem \
+NODE_OPTIONS=--use-env-proxy \
+HTTP_PROXY=http://127.0.0.1:8080 \
+HTTPS_PROXY=http://127.0.0.1:8080 \
+node your-app.js
+```
+
+For curl:
+
+```sh
+curl --proxy http://127.0.0.1:8080 --cacert /path/to/ca.pem \
+  https://example.com/
+```
+
+Python requests uses `REQUESTS_CA_BUNDLE=/path/to/ca.pem`; Chromium requires the
+CA in its own applicable trust store and an explicit proxy configuration.
+Certificate-pinned applications need their pinning configuration adjusted.
+Netkit still verifies upstream certificates using its system trust roots; it
+does not disable TLS verification. Private upstream CAs must also be trusted by
+Netkit itself.
+
+The existing request dashboard and `/requests` API show each decrypted HTTP
+exchange, including its full URL, headers, and both bodies. Headers and bodies
+are captured as received, including credentials. Keep access within your trusted
+debugging environment. History remains in memory and is cleared on restart.
+
+Responses are streamed to clients immediately, including SSE; their history
+record is saved when the exchange finishes or disconnects. Long-lived streams
+therefore appear after closing. Capture stores complete bodies in memory, so
+choose `--history-size` to suit payload sizes. Binary or compressed payloads are
+not decoded for display. Inspected connections negotiate HTTP/1.1 (including
+keep-alive); HTTP/2-only clients are not supported. WebSocket and other protocol
+upgrades return 501 in inspection mode; frame inspection is not implemented.
+Plain HTTP carried inside CONNECT (as used by Node fetch) is also captured.
+Only traffic configured to pass through Netkit can be inspected.

cmd/netkit/ca.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/biancarosa/netkit/internal/proxy"
+)
+
+func runCA(args []string) error {
+	flags := flag.NewFlagSet("ca", flag.ContinueOnError)
+	certPath := flags.String("cert", "netkit-ca.pem", "CA certificate output (share with clients)")
+	keyPath := flags.String("key", "netkit-ca-key.pem", "Private key output (keep on proxy)")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+	cert, key, err := proxy.GenerateCA()
+	if err != nil {
+		return err
+	}
+	// Exclusive creation prevents accidental replacement of an already trusted CA.
+	keyFile, err := os.OpenFile(*keyPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
+	if err != nil {
+		return err
+	}
+	if _, err = keyFile.Write(key); err != nil {
+		_ = keyFile.Close()
+		return err
+	}
+	if err = keyFile.Close(); err != nil {
+		return err
+	}
+	certFile, err := os.OpenFile(*certPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
+	if err != nil {
+		_ = os.Remove(*keyPath)
+		return err
+	}
+	if _, err = certFile.Write(cert); err != nil {
+		_ = certFile.Close()
+		return err
+	}
+	if err = certFile.Close(); err != nil {
+		return err
+	}
+	fmt.Printf("CA certificate: %s\nPrivate key: %s\nTrust only the certificate in proxy clients.\n", *certPath, *keyPath)
+	return nil
+}

cmd/netkit/ca_test.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/biancarosa/netkit/internal/proxy"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCAGenerationDoesNotOverwrite(t *testing.T) {
+	dir := t.TempDir()
+	cert, key := filepath.Join(dir, "ca.pem"), filepath.Join(dir, "key.pem")
+	args := []string{"--cert", cert, "--key", key}
+	require.NoError(t, runCA(args))
+	_, err := proxy.LoadCA(cert, key)
+	require.NoError(t, err)
+	original, err := os.ReadFile(key)
+	require.NoError(t, err)
+	st, err := os.Stat(key)
+	require.NoError(t, err)
+	require.Equal(t, os.FileMode(0600), st.Mode().Perm())
+	require.Error(t, runCA(args))
+	unchanged, err := os.ReadFile(key)
+	require.NoError(t, err)
+	require.Equal(t, original, unchanged)
+	otherKey := filepath.Join(dir, "other-key.pem")
+	require.Error(t, runCA([]string{"--cert", cert, "--key", otherKey}))
+	_, err = os.Stat(otherKey)
+	require.True(t, os.IsNotExist(err))
+}

cmd/netkit/main.go

@@ -13,21 +13,25 @@ import (
 func main() {
 	// Parse command
 	if len(os.Args) < 2 {
-		log.Fatal("Please specify a command: serve or request")
+		log.Fatal("Please specify a command: serve, request, or ca")
 	}
 
 	command := os.Args[1]
 	os.Args = os.Args[1:] // Remove command from args for flag parsing
 
 	switch command {
+	case "ca":
+		if err := runCA(os.Args[1:]); err != nil {
+			log.Fatal(err)
+		}
 	case "serve":
 		runServe()
 	case "request":
 		if err := runRequest(); err != nil {
 			log.Fatal(err)
 		}
 	default:
-		log.Fatalf("Unknown command: %s. Use 'serve' or 'request'", command)
+		log.Fatalf("Unknown command: %s. Use 'serve', 'request', or 'ca'", command)
 	}
 }
 
@@ -41,10 +45,24 @@ func runServe() {
 	dashboardDir := flag.String("dashboard-dir", "", "Directory containing dashboard build files (optional if embedded)")
 	dashboardBasePath := flag.String("dashboard-base-path", envOrDefault("NETKIT_DASHBOARD_BASE_PATH", ""), "Public dashboard URL prefix for path-based reverse proxies")
 	logLevel := flag.String("log-level", "info", "Logging level (debug, info, warn, error)")
+	inspectHTTPS := flag.Bool("inspect-https", false, "Decrypt and capture HTTPS for all destinations (clients must trust the CA)")
+	caCert := flag.String("ca-cert", "", "Inspection CA certificate PEM file")
+	caKey := flag.String("ca-key", "", "Inspection CA private key PEM file")
 	flag.Parse()
+	var ca *proxy.CertificateAuthority
+	if *inspectHTTPS {
+		var err error
+		ca, err = proxy.LoadCA(*caCert, *caKey)
+		if err != nil {
+			log.Fatal(err)
+		}
+	} else if *caCert != "" || *caKey != "" {
+		log.Fatal("--ca-cert and --ca-key require --inspect-https")
+	}
 
 	// Create proxy configuration
 	config := &proxy.Config{
+		InspectionCA:      ca,
 		Port:              *port,
 		AdminPort:         *adminPort,
 		HistorySize:       *historySize,

internal/proxy/ca.go

@@ -0,0 +1,98 @@
+package proxy
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"net"
+	"time"
+)
+
+// CertificateAuthority signs the certificates presented to consenting proxy clients.
+// Its private key must stay on the proxy; clients need only the public certificate.
+type CertificateAuthority struct {
+	pair tls.Certificate
+	cert *x509.Certificate
+}
+
+func serialNumber() (*big.Int, error) {
+	return rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+}
+
+// GenerateCA returns a new CA certificate and PKCS#8 private key in PEM format.
+func GenerateCA() ([]byte, []byte, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, err
+	}
+	serial, err := serialNumber()
+	if err != nil {
+		return nil, nil, err
+	}
+	now := time.Now()
+	template := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: "Netkit inspection CA"}, NotBefore: now.Add(-time.Minute), NotAfter: now.AddDate(5, 0, 0), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, MaxPathLenZero: true}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	if err != nil {
+		return nil, nil, err
+	}
+	private, err := x509.MarshalPKCS8PrivateKey(key)
+	if err != nil {
+		return nil, nil, err
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: private}), nil
+}
+
+// LoadCA validates an existing CA and matching private key without replacing them.
+func LoadCA(certPath, keyPath string) (*CertificateAuthority, error) {
+	pair, err := tls.LoadX509KeyPair(certPath, keyPath)
+	if err != nil {
+		return nil, fmt.Errorf("load inspection CA: %w", err)
+	}
+	cert, err := x509.ParseCertificate(pair.Certificate[0])
+	if err != nil {
+		return nil, err
+	}
+	if !cert.IsCA || !cert.BasicConstraintsValid || cert.KeyUsage&x509.KeyUsageCertSign == 0 {
+		return nil, fmt.Errorf("inspection certificate must be a signing CA")
+	}
+	if time.Now().Before(cert.NotBefore) || !time.Now().Before(cert.NotAfter) {
+		return nil, fmt.Errorf("inspection CA is not currently valid")
+	}
+	return &CertificateAuthority{pair: pair, cert: cert}, nil
+}
+
+func (ca *CertificateAuthority) certificate(host string) (*tls.Certificate, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+	serial, err := serialNumber()
+	if err != nil {
+		return nil, err
+	}
+	now := time.Now()
+	expiry := now.Add(24 * time.Hour)
+	if ca.cert.NotAfter.Before(expiry) {
+		expiry = ca.cert.NotAfter
+	}
+	if !now.Before(expiry) {
+		return nil, fmt.Errorf("inspection CA has expired")
+	}
+	cert := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: host}, NotBefore: now.Add(-time.Minute), NotAfter: expiry, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true}
+	if ip := net.ParseIP(host); ip != nil {
+		cert.IPAddresses = []net.IP{ip}
+	} else {
+		cert.DNSNames = []string{host}
+	}
+	der, err := x509.CreateCertificate(rand.Reader, cert, ca.cert, &key.PublicKey, ca.pair.PrivateKey)
+	if err != nil {
+		return nil, err
+	}
+	return &tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, nil
+}