From 710e794cc38b29995574d21a2d24aae7be36dc23 Mon Sep 17 00:00:00 2001 From: Bianca Rosa Date: Fri, 18 Sep 2026 17:22:20 -0300 Subject: [PATCH 1/2] feat: inspect HTTPS bodies with a trusted local CA --- README.md | 61 ++++++++ cmd/netkit/ca.go | 48 +++++++ cmd/netkit/ca_test.go | 32 +++++ cmd/netkit/main.go | 22 ++- internal/proxy/ca.go | 98 +++++++++++++ internal/proxy/inspection.go | 207 +++++++++++++++++++++++++++ internal/proxy/inspection_test.go | 227 ++++++++++++++++++++++++++++++ internal/proxy/proxy.go | 37 ++++- 8 files changed, 723 insertions(+), 9 deletions(-) create mode 100644 cmd/netkit/ca.go create mode 100644 cmd/netkit/ca_test.go create mode 100644 internal/proxy/ca.go create mode 100644 internal/proxy/inspection.go create mode 100644 internal/proxy/inspection_test.go diff --git a/README.md b/README.md index 27a8941..3d615a5 100644 --- a/README.md +++ b/README.md @@ -297,3 +297,64 @@ dashboard's destination-header API follow its separate HTTP forwarding path. See [RUNBOOK.md](RUNBOOK.md) for release validation, local troubleshooting, CLI/container rollback, and the applicability of cost checks to this distributed tool. + +### Inspect HTTPS request and response bodies + +Netkit can decrypt HTTPS for **every destination** when clients explicitly trust +its inspection CA. There is no domain allowlist. Without `--inspect-https`, CONNECT +continues to work as an encrypted tunnel. + +Generate a CA once and keep it across proxy restarts: + +```sh +mkdir -p "$HOME/.config/netkit" +netkit ca --cert "$HOME/.config/netkit/ca.pem" --key "$HOME/.config/netkit/ca-key.pem" +netkit serve --inspect-https \ + --ca-cert "$HOME/.config/netkit/ca.pem" \ + --ca-key "$HOME/.config/netkit/ca-key.pem" +``` + +The `ca` command refuses to overwrite existing files and creates the private key +with mode `0600`. Only the public `ca.pem` belongs in client trust stores. The +private key stays with Netkit. Existing signing CA PEM files can also be loaded; +an invalid, expired, or mismatched CA fails startup. + +For Node.js / OpenClaw, mount the public certificate in the gateway and restart +its process with: + +```sh +NODE_EXTRA_CA_CERTS=/path/to/ca.pem \ +NODE_OPTIONS=--use-env-proxy \ +HTTP_PROXY=http://127.0.0.1:8080 \ +HTTPS_PROXY=http://127.0.0.1:8080 \ +node your-app.js +``` + +For curl: + +```sh +curl --proxy http://127.0.0.1:8080 --cacert /path/to/ca.pem \ + https://example.com/ +``` + +Python requests uses `REQUESTS_CA_BUNDLE=/path/to/ca.pem`; Chromium requires the +CA in its own applicable trust store and an explicit proxy configuration. +Certificate-pinned applications need their pinning configuration adjusted. +Netkit still verifies upstream certificates using its system trust roots; it +does not disable TLS verification. Private upstream CAs must also be trusted by +Netkit itself. + +The existing request dashboard and `/requests` API show each decrypted HTTP +exchange, including its full URL, headers, and both bodies. Headers and bodies +are captured as received, including credentials. Keep access within your trusted +debugging environment. History remains in memory and is cleared on restart. + +Responses are streamed to clients immediately, including SSE; their history +record is saved when the exchange finishes or disconnects. Long-lived streams +therefore appear after closing. Capture stores complete bodies in memory, so +choose `--history-size` to suit payload sizes. Binary or compressed payloads are +not decoded for display. Inspected connections negotiate HTTP/1.1 (including +keep-alive); HTTP/2-only clients are not supported. WebSocket and other protocol +upgrades return 501 in inspection mode; frame inspection is not implemented. +Plain HTTP carried inside CONNECT (as used by Node fetch) is also captured. +Only traffic configured to pass through Netkit can be inspected. diff --git a/cmd/netkit/ca.go b/cmd/netkit/ca.go new file mode 100644 index 0000000..1e21796 --- /dev/null +++ b/cmd/netkit/ca.go @@ -0,0 +1,48 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/biancarosa/netkit/internal/proxy" +) + +func runCA(args []string) error { + flags := flag.NewFlagSet("ca", flag.ContinueOnError) + certPath := flags.String("cert", "netkit-ca.pem", "CA certificate output (share with clients)") + keyPath := flags.String("key", "netkit-ca-key.pem", "Private key output (keep on proxy)") + if err := flags.Parse(args); err != nil { + return err + } + cert, key, err := proxy.GenerateCA() + if err != nil { + return err + } + // Exclusive creation prevents accidental replacement of an already trusted CA. + keyFile, err := os.OpenFile(*keyPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + if _, err = keyFile.Write(key); err != nil { + _ = keyFile.Close() + return err + } + if err = keyFile.Close(); err != nil { + return err + } + certFile, err := os.OpenFile(*certPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + if err != nil { + _ = os.Remove(*keyPath) + return err + } + if _, err = certFile.Write(cert); err != nil { + _ = certFile.Close() + return err + } + if err = certFile.Close(); err != nil { + return err + } + fmt.Printf("CA certificate: %s\nPrivate key: %s\nTrust only the certificate in proxy clients.\n", *certPath, *keyPath) + return nil +} diff --git a/cmd/netkit/ca_test.go b/cmd/netkit/ca_test.go new file mode 100644 index 0000000..9efa5af --- /dev/null +++ b/cmd/netkit/ca_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/biancarosa/netkit/internal/proxy" + "github.com/stretchr/testify/require" +) + +func TestCAGenerationDoesNotOverwrite(t *testing.T) { + dir := t.TempDir() + cert, key := filepath.Join(dir, "ca.pem"), filepath.Join(dir, "key.pem") + args := []string{"--cert", cert, "--key", key} + require.NoError(t, runCA(args)) + _, err := proxy.LoadCA(cert, key) + require.NoError(t, err) + original, err := os.ReadFile(key) + require.NoError(t, err) + st, err := os.Stat(key) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), st.Mode().Perm()) + require.Error(t, runCA(args)) + unchanged, err := os.ReadFile(key) + require.NoError(t, err) + require.Equal(t, original, unchanged) + otherKey := filepath.Join(dir, "other-key.pem") + require.Error(t, runCA([]string{"--cert", cert, "--key", otherKey})) + _, err = os.Stat(otherKey) + require.True(t, os.IsNotExist(err)) +} diff --git a/cmd/netkit/main.go b/cmd/netkit/main.go index 1d6ad45..d679916 100644 --- a/cmd/netkit/main.go +++ b/cmd/netkit/main.go @@ -13,13 +13,17 @@ import ( func main() { // Parse command if len(os.Args) < 2 { - log.Fatal("Please specify a command: serve or request") + log.Fatal("Please specify a command: serve, request, or ca") } command := os.Args[1] os.Args = os.Args[1:] // Remove command from args for flag parsing switch command { + case "ca": + if err := runCA(os.Args[1:]); err != nil { + log.Fatal(err) + } case "serve": runServe() case "request": @@ -27,7 +31,7 @@ func main() { log.Fatal(err) } default: - log.Fatalf("Unknown command: %s. Use 'serve' or 'request'", command) + log.Fatalf("Unknown command: %s. Use 'serve', 'request', or 'ca'", command) } } @@ -41,10 +45,24 @@ func runServe() { dashboardDir := flag.String("dashboard-dir", "", "Directory containing dashboard build files (optional if embedded)") dashboardBasePath := flag.String("dashboard-base-path", envOrDefault("NETKIT_DASHBOARD_BASE_PATH", ""), "Public dashboard URL prefix for path-based reverse proxies") logLevel := flag.String("log-level", "info", "Logging level (debug, info, warn, error)") + inspectHTTPS := flag.Bool("inspect-https", false, "Decrypt and capture HTTPS for all destinations (clients must trust the CA)") + caCert := flag.String("ca-cert", "", "Inspection CA certificate PEM file") + caKey := flag.String("ca-key", "", "Inspection CA private key PEM file") flag.Parse() + var ca *proxy.CertificateAuthority + if *inspectHTTPS { + var err error + ca, err = proxy.LoadCA(*caCert, *caKey) + if err != nil { + log.Fatal(err) + } + } else if *caCert != "" || *caKey != "" { + log.Fatal("--ca-cert and --ca-key require --inspect-https") + } // Create proxy configuration config := &proxy.Config{ + InspectionCA: ca, Port: *port, AdminPort: *adminPort, HistorySize: *historySize, diff --git a/internal/proxy/ca.go b/internal/proxy/ca.go new file mode 100644 index 0000000..b67cf7e --- /dev/null +++ b/internal/proxy/ca.go @@ -0,0 +1,98 @@ +package proxy + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "time" +) + +// CertificateAuthority signs the certificates presented to consenting proxy clients. +// Its private key must stay on the proxy; clients need only the public certificate. +type CertificateAuthority struct { + pair tls.Certificate + cert *x509.Certificate +} + +func serialNumber() (*big.Int, error) { + return rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) +} + +// GenerateCA returns a new CA certificate and PKCS#8 private key in PEM format. +func GenerateCA() ([]byte, []byte, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, err + } + serial, err := serialNumber() + if err != nil { + return nil, nil, err + } + now := time.Now() + template := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: "Netkit inspection CA"}, NotBefore: now.Add(-time.Minute), NotAfter: now.AddDate(5, 0, 0), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, MaxPathLenZero: true} + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return nil, nil, err + } + private, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, nil, err + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: private}), nil +} + +// LoadCA validates an existing CA and matching private key without replacing them. +func LoadCA(certPath, keyPath string) (*CertificateAuthority, error) { + pair, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, fmt.Errorf("load inspection CA: %w", err) + } + cert, err := x509.ParseCertificate(pair.Certificate[0]) + if err != nil { + return nil, err + } + if !cert.IsCA || !cert.BasicConstraintsValid || cert.KeyUsage&x509.KeyUsageCertSign == 0 { + return nil, fmt.Errorf("inspection certificate must be a signing CA") + } + if time.Now().Before(cert.NotBefore) || !time.Now().Before(cert.NotAfter) { + return nil, fmt.Errorf("inspection CA is not currently valid") + } + return &CertificateAuthority{pair: pair, cert: cert}, nil +} + +func (ca *CertificateAuthority) certificate(host string) (*tls.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + serial, err := serialNumber() + if err != nil { + return nil, err + } + now := time.Now() + expiry := now.Add(24 * time.Hour) + if ca.cert.NotAfter.Before(expiry) { + expiry = ca.cert.NotAfter + } + if !now.Before(expiry) { + return nil, fmt.Errorf("inspection CA has expired") + } + cert := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: host}, NotBefore: now.Add(-time.Minute), NotAfter: expiry, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true} + if ip := net.ParseIP(host); ip != nil { + cert.IPAddresses = []net.IP{ip} + } else { + cert.DNSNames = []string{host} + } + der, err := x509.CreateCertificate(rand.Reader, cert, ca.cert, &key.PublicKey, ca.pair.PrivateKey) + if err != nil { + return nil, err + } + return &tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, nil +} diff --git a/internal/proxy/inspection.go b/internal/proxy/inspection.go new file mode 100644 index 0000000..2a49800 --- /dev/null +++ b/internal/proxy/inspection.go @@ -0,0 +1,207 @@ +package proxy + +import ( + "bufio" + "bytes" + "crypto/tls" + "errors" + "io" + "log" + "net" + "net/http" + "net/http/httputil" + "sync" + "time" +) + +// bufferedConn preserves bytes already read by the outer CONNECT HTTP server. +type bufferedConn struct { + net.Conn + reader *bufio.Reader +} + +func (c *bufferedConn) Read(b []byte) (int, error) { return c.reader.Read(b) } + +type inspectionConn struct { + net.Conn + done chan struct{} + once sync.Once +} + +func (c *inspectionConn) Close() error { + err := c.Conn.Close() + c.once.Do(func() { close(c.done) }) + return err +} + +type singleConnListener struct { + conn net.Conn + done <-chan struct{} + accepted bool +} + +func (l *singleConnListener) Accept() (net.Conn, error) { + if !l.accepted { + l.accepted = true + return l.conn, nil + } + <-l.done + return nil, net.ErrClosed +} +func (l *singleConnListener) Close() error { return l.conn.Close() } +func (l *singleConnListener) Addr() net.Addr { return l.conn.LocalAddr() } + +func (p *Proxy) handleInspectedConnect(w http.ResponseWriter, r *http.Request) { + host, _, err := net.SplitHostPort(r.Host) + if err != nil || host == "" { + http.Error(w, "CONNECT requires host:port", http.StatusBadRequest) + return + } + cert, err := p.config.InspectionCA.certificate(host) + if err != nil { + http.Error(w, "Cannot create inspection certificate", http.StatusBadGateway) + return + } + h, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "Hijacking unavailable", http.StatusInternalServerError) + return + } + conn, rw, err := h.Hijack() + if err != nil { + return + } + defer func() { + if err := conn.Close(); err != nil && !errors.Is(err, net.ErrClosed) { + log.Printf("Inspection connection close: %v", err) + } + }() + p.inspectionMu.Lock() + if p.inspectionStopped { + p.inspectionMu.Unlock() + return + } + p.inspectionConns[conn] = struct{}{} + p.inspectionMu.Unlock() + defer func() { p.inspectionMu.Lock(); delete(p.inspectionConns, conn); p.inspectionMu.Unlock() }() + if _, err = rw.WriteString("HTTP/1.1 200 Connection Established\r\n\r\n"); err != nil { + return + } + if err = rw.Flush(); err != nil { + return + } + // Node fetch also uses CONNECT for plain HTTP. Preserve that behavior. + if err = conn.SetReadDeadline(time.Now().Add(15 * time.Second)); err != nil { + return + } + first, err := rw.Reader.Peek(1) + if err != nil { + return + } + if err = conn.SetReadDeadline(time.Time{}); err != nil { + return + } + tracked := &inspectionConn{Conn: &bufferedConn{Conn: conn, reader: rw.Reader}, done: make(chan struct{})} + var inner net.Conn = tracked + scheme := "http" + if first[0] == 22 { + scheme = "https" + inner = tls.Server(tracked, &tls.Config{Certificates: []tls.Certificate{*cert}, MinVersion: tls.VersionTLS12, NextProtos: []string{"http/1.1"}}) + } + server := &http.Server{ReadHeaderTimeout: 15 * time.Second, IdleTimeout: 60 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, innerReq *http.Request) { + if innerReq.Method == http.MethodConnect { + http.Error(w, "Nested CONNECT unsupported", http.StatusBadRequest) + return + } + p.inspectHTTP(w, innerReq, scheme, r.Host) + })} + listener := &singleConnListener{conn: inner, done: tracked.done} + if err := server.Serve(listener); err != nil && err != net.ErrClosed && err != http.ErrServerClosed { + log.Printf("Inspection server: %v", err) + } +} + +// synchronizedCapture records bytes as the transport consumes them. Response data +// is forwarded immediately (including SSE), and the completed exchange is saved +// when the handler returns. Upload reads may run in a transport goroutine. +type synchronizedCapture struct { + mu sync.Mutex + data bytes.Buffer +} + +func (c *synchronizedCapture) Write(b []byte) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.data.Write(b) +} +func (c *synchronizedCapture) snapshot() (string, int64) { + c.mu.Lock() + defer c.mu.Unlock() + return c.data.String(), int64(c.data.Len()) +} + +type captureReadCloser struct { + io.Reader + io.Closer +} + +func (p *Proxy) inspectHTTP(w http.ResponseWriter, r *http.Request, scheme, authority string) { + start := time.Now() + record := RequestRecord{ID: generateID(), Timestamp: start, ProxyStartTime: start, Method: r.Method, RequestHeaders: convertHeaders(r.Header)} + target := *r.URL + target.Scheme = scheme + target.Host = authority + record.URL = target.String() + var requestBody, responseBody synchronizedCapture + defer func() { + record.RequestBody, record.RequestSize = requestBody.snapshot() + record.ResponseBody, record.ResponseSize = responseBody.snapshot() + record.ProxyEndTime = time.Now() + p.history.AddRecord(record) + }() + if r.Header.Get("Upgrade") != "" { + record.Error = "Protocol upgrades are not supported in inspection mode" + record.ResponseStatus = http.StatusNotImplemented + http.Error(w, record.Error, record.ResponseStatus) + return + } + if r.Body != nil { + r.Body = &captureReadCloser{Reader: io.TeeReader(r.Body, &requestBody), Closer: r.Body} + } + forward := &httputil.ReverseProxy{ + Rewrite: func(pr *httputil.ProxyRequest) { + pr.Out.URL.Scheme = scheme + pr.Out.URL.Host = authority + pr.Out.Host = authority + pr.Out.Header.Del("Proxy-Authorization") + pr.Out.Header.Del("Proxy-Connection") + record.UpstreamStartTime = time.Now() + }, + Transport: p.inspectionTransport, + FlushInterval: -1, + ModifyResponse: func(resp *http.Response) error { + record.UpstreamEndTime = time.Now() + record.ResponseStatus = resp.StatusCode + record.ResponseHeaders = convertHeaders(resp.Header) + record.Success = true + resp.Body = &captureReadCloser{Reader: io.TeeReader(resp.Body, &responseBody), Closer: resp.Body} + return nil + }, + ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { + record.UpstreamEndTime = time.Now() + record.Error = err.Error() + record.ResponseStatus = http.StatusBadGateway + http.Error(w, "Upstream request failed", http.StatusBadGateway) + }, + } + // ReverseProxy aborts a partially written response on stream failure. Preserve + // that behavior while recording the failed exchange instead of a false success. + defer func() { + if v := recover(); v != nil { + record.Success = false + record.Error = "Response stream interrupted" + panic(v) + } + }() + forward.ServeHTTP(w, r) +} diff --git a/internal/proxy/inspection_test.go b/internal/proxy/inspection_test.go new file mode 100644 index 0000000..9099c4e --- /dev/null +++ b/internal/proxy/inspection_test.go @@ -0,0 +1,227 @@ +package proxy + +import ( + "bufio" + "crypto/tls" + "crypto/x509" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func inspectionCA(t *testing.T) (*CertificateAuthority, *x509.CertPool) { + t.Helper() + cert, key, err := GenerateCA() + require.NoError(t, err) + dir := t.TempDir() + cp, kp := filepath.Join(dir, "ca.pem"), filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(cp, cert, 0600)) + require.NoError(t, os.WriteFile(kp, key, 0600)) + ca, err := LoadCA(cp, kp) + require.NoError(t, err) + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(cert)) + return ca, roots +} + +func inspectionClient(t *testing.T, upstream *httptest.Server) (*Proxy, *http.Client, string) { + t.Helper() + ca, roots := inspectionCA(t) + p := New(&Config{InspectionCA: ca}) + // Trust this test origin explicitly; real origins use the system roots. + p.inspectionTransport.TLSClientConfig = &tls.Config{RootCAs: x509.NewCertPool(), MinVersion: tls.VersionTLS12} + p.inspectionTransport.TLSClientConfig.RootCAs.AddCert(upstream.Certificate()) + server := httptest.NewServer(p) + t.Cleanup(func() { require.NoError(t, p.Stop()); server.Close() }) + u, err := url.Parse(server.URL) + require.NoError(t, err) + tr := &http.Transport{Proxy: http.ProxyURL(u), TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}} + t.Cleanup(tr.CloseIdleConnections) + return p, &http.Client{Transport: tr, Timeout: 3 * time.Second}, server.URL +} + +func waitRecords(t *testing.T, p *Proxy, count int) []RequestRecord { + t.Helper() + require.Eventually(t, func() bool { return len(p.history.GetRecords()) >= count }, time.Second, time.Millisecond) + return p.history.GetRecords() +} + +func TestInspectionCapturesHTTPSBodiesAndReusesConnection(t *testing.T) { + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.Equal(t, "secret-for-test", r.Header.Get("Authorization")) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(201) + _, err = w.Write(append([]byte("response:"), b...)) + require.NoError(t, err) + })) + defer upstream.Close() + p, c, _ := inspectionClient(t, upstream) + for i := 0; i < 2; i++ { + req, err := http.NewRequest(http.MethodPost, upstream.URL+"/body?q=test", strings.NewReader(`{"hello":"world"}`)) + require.NoError(t, err) + req.Header.Set("Authorization", "secret-for-test") + resp, err := c.Do(req) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, 201, resp.StatusCode) + require.Equal(t, `response:{"hello":"world"}`, string(body)) + } + records := waitRecords(t, p, 2) + require.Len(t, records, 2) + for _, r := range records { + require.Equal(t, "POST", r.Method) + require.Equal(t, upstream.URL+"/body?q=test", r.URL) + require.Equal(t, `{"hello":"world"}`, r.RequestBody) + require.Equal(t, `response:{"hello":"world"}`, r.ResponseBody) + require.Equal(t, 201, r.ResponseStatus) + require.True(t, r.Success) + } +} + +func TestInspectionStreamsBeforeResponseCompletes(t *testing.T) { + release := make(chan struct{}) + defer close(release) + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, err := io.WriteString(w, "data: first\n\n") + require.NoError(t, err) + w.(http.Flusher).Flush() + select { + case <-release: + case <-r.Context().Done(): + } + })) + defer upstream.Close() + p, c, _ := inspectionClient(t, upstream) + resp, err := c.Get(upstream.URL + "/stream") + require.NoError(t, err) + line, err := bufio.NewReader(resp.Body).ReadString('\n') + require.NoError(t, err) + require.Equal(t, "data: first\n", line) + // Closing the downstream stream must cancel the upstream and record capture. + require.NoError(t, resp.Body.Close()) + r := waitRecords(t, p, 1)[0] + require.Contains(t, r.ResponseBody, "data: first") +} + +func TestInspectionVerifiesOriginCertificate(t *testing.T) { + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Error("untrusted origin must not receive a request") })) + defer upstream.Close() + p, c, _ := inspectionClient(t, upstream) + p.inspectionTransport.TLSClientConfig.RootCAs = x509.NewCertPool() + resp, err := c.Get(upstream.URL) + require.NoError(t, err) + require.Equal(t, 502, resp.StatusCode) + require.NoError(t, resp.Body.Close()) + r := waitRecords(t, p, 1)[0] + require.False(t, r.Success) + require.Contains(t, r.Error, "certificate") +} + +func TestInspectionRequiresClientTrust(t *testing.T) { + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Error("untrusted client must not reach upstream") })) + defer upstream.Close() + _, c, _ := inspectionClient(t, upstream) + c.Transport.(*http.Transport).TLSClientConfig.RootCAs = x509.NewCertPool() + _, err := c.Get(upstream.URL) + require.Error(t, err) + require.Contains(t, err.Error(), "certificate") +} + +func TestInspectionPlaintextCONNECT(t *testing.T) { + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer upstream.Close() + p, _, proxyURL := inspectionClient(t, upstream) + plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := io.WriteString(w, "plain-body") + require.NoError(t, err) + })) + defer plain.Close() + origin, err := url.Parse(plain.URL) + require.NoError(t, err) + proxyAddr := strings.TrimPrefix(proxyURL, "http://") + conn, err := net.Dial("tcp", proxyAddr) + require.NoError(t, err) + defer func() { require.NoError(t, conn.Close()) }() + require.NoError(t, conn.SetDeadline(time.Now().Add(3*time.Second))) + // Pipeline the request with CONNECT to verify buffered bytes are preserved. + _, err = io.WriteString(conn, "CONNECT "+origin.Host+" HTTP/1.1\r\nHost: "+origin.Host+"\r\n\r\nGET /plain HTTP/1.1\r\nHost: "+origin.Host+"\r\nConnection: close\r\n\r\n") + require.NoError(t, err) + reader := bufio.NewReader(conn) + connect, err := http.ReadResponse(reader, &http.Request{Method: "CONNECT"}) + require.NoError(t, err) + require.Equal(t, 200, connect.StatusCode) + resp, err := http.ReadResponse(reader, &http.Request{Method: "GET"}) + require.NoError(t, err) + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.Equal(t, "plain-body", string(b)) + r := waitRecords(t, p, 1)[0] + require.Equal(t, plain.URL+"/plain", r.URL) + require.Equal(t, "plain-body", r.ResponseBody) +} + +func TestInspectionPreservesRedirectsAndOptions(t *testing.T) { + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "OPTIONS" { + w.WriteHeader(204) + return + } + w.Header().Set("Location", "/next") + w.WriteHeader(302) + })) + defer upstream.Close() + p, c, _ := inspectionClient(t, upstream) + c.CheckRedirect = func(r *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } + resp, err := c.Get(upstream.URL) + require.NoError(t, err) + require.Equal(t, 302, resp.StatusCode) + require.NoError(t, resp.Body.Close()) + req, err := http.NewRequest("OPTIONS", upstream.URL, nil) + require.NoError(t, err) + resp, err = c.Do(req) + require.NoError(t, err) + require.Equal(t, 204, resp.StatusCode) + require.NoError(t, resp.Body.Close()) + require.Len(t, waitRecords(t, p, 2), 2) +} + +func TestInspectionCertificateDNSAndIP(t *testing.T) { + ca, roots := inspectionCA(t) + for _, host := range []string{"api.example.com", "127.0.0.1", "::1"} { + cert, err := ca.certificate(host) + require.NoError(t, err) + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + require.NoError(t, err) + _, err = leaf.Verify(x509.VerifyOptions{Roots: roots, DNSName: host}) + require.NoError(t, err) + } + ca.cert.NotAfter = time.Now().Add(-time.Second) + _, err := ca.certificate("example.com") + require.Error(t, err) +} + +func TestInspectionStopClosesIdleTunnels(t *testing.T) { + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) })) + defer upstream.Close() + p, c, _ := inspectionClient(t, upstream) + resp, err := c.Get(upstream.URL) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.NoError(t, p.Stop()) + require.Eventually(t, func() bool { p.inspectionMu.Lock(); defer p.inspectionMu.Unlock(); return len(p.inspectionConns) == 0 }, time.Second, time.Millisecond) +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 17440fa..4a81eba 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -13,6 +13,7 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "github.com/biancarosa/netkit/internal/dashboard" @@ -20,6 +21,7 @@ import ( // Config holds the proxy configuration type Config struct { + InspectionCA *CertificateAuthority // Non-nil enables inspection for every CONNECT destination Port int AdminPort int LogLevel string @@ -32,12 +34,16 @@ type Config struct { // Proxy represents the HTTP proxy server type Proxy struct { - config *Config - server *http.Server - adminServer *http.Server - dashboardServer *http.Server - httpClient *http.Client - history *RequestHistory + inspectionMu sync.Mutex + inspectionConns map[net.Conn]struct{} + inspectionStopped bool + inspectionTransport *http.Transport + config *Config + server *http.Server + adminServer *http.Server + dashboardServer *http.Server + httpClient *http.Client + history *RequestHistory } // New creates a new Proxy instance @@ -56,6 +62,11 @@ func New(config *Config) *Proxy { history: NewRequestHistory(historySize), } + proxy.inspectionConns = make(map[net.Conn]struct{}) + proxy.inspectionTransport = http.DefaultTransport.(*http.Transport).Clone() + proxy.inspectionTransport.ResponseHeaderTimeout = 30 * time.Second + proxy.inspectionTransport.Proxy = nil // Never route upstream connections back into this proxy. + // Initialize the main HTTP proxy server proxy.server = &http.Server{ Addr: fmt.Sprintf(":%d", config.Port), @@ -201,7 +212,11 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { // For CONNECT method (HTTPS tunneling) if r.Method == http.MethodConnect { - p.handleConnect(w, r) + if p.config.InspectionCA != nil { + p.handleInspectedConnect(w, r) + } else { + p.handleConnect(w, r) + } return } @@ -621,6 +636,14 @@ func (p *Proxy) Stop() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + p.inspectionMu.Lock() + p.inspectionStopped = true + for conn := range p.inspectionConns { + _ = conn.Close() + } + p.inspectionMu.Unlock() + p.inspectionTransport.CloseIdleConnections() + var proxyErr, adminErr, dashboardErr error if p.server != nil { From 0fe844ed772278ad9870d09a3c66edb29596f0ec Mon Sep 17 00:00:00 2001 From: Bianca Rosa Date: Fri, 18 Sep 2026 17:40:29 -0300 Subject: [PATCH 2/2] fix: satisfy inspection lint and keep docs application agnostic --- README.md | 2 +- internal/proxy/inspection.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3d615a5..d0bed16 100644 --- a/README.md +++ b/README.md @@ -319,7 +319,7 @@ with mode `0600`. Only the public `ca.pem` belongs in client trust stores. The private key stays with Netkit. Existing signing CA PEM files can also be loaded; an invalid, expired, or mismatched CA fails startup. -For Node.js / OpenClaw, mount the public certificate in the gateway and restart +For Node.js, make the public certificate available to the application and restart its process with: ```sh diff --git a/internal/proxy/inspection.go b/internal/proxy/inspection.go index 2a49800..496bf7c 100644 --- a/internal/proxy/inspection.go +++ b/internal/proxy/inspection.go @@ -94,7 +94,7 @@ func (p *Proxy) handleInspectedConnect(w http.ResponseWriter, r *http.Request) { if err = conn.SetReadDeadline(time.Now().Add(15 * time.Second)); err != nil { return } - first, err := rw.Reader.Peek(1) + first, err := rw.Peek(1) if err != nil { return }