network: chain proxy requests via upstream proxy

docs/proxy.md

@@ -76,8 +76,10 @@ process:
 ```text
 HTTP_PROXY
 HTTPS_PROXY
+ALL_PROXY
 http_proxy
 https_proxy
+all_proxy
 NO_PROXY
 no_proxy
 ```
@@ -91,6 +93,49 @@ BuildKit also injects a generated CA certificate into common Linux trust bundle
 locations for the duration of the exec. This lets HTTPS requests using the
 system trust store pass through the BuildKit proxy.
 
+## Upstream proxies
+
+To use an upstream proxy, set `HTTP_PROXY` and/or `HTTPS_PROXY` in the
+`buildkitd` environment and enable proxy networking:
+
+```bash
+HTTP_PROXY=http://proxy.example:3128 \
+HTTPS_PROXY=http://proxy.example:3128 \
+NO_PROXY=localhost,127.0.0.1,.example.internal \
+buildkitd --proxy-network
+```
+
+BuildKit uses Go's standard proxy environment handling:
+
+| Variable pair | Purpose |
+| --- | --- |
+| `HTTP_PROXY`, `http_proxy` | Selects the upstream proxy for HTTP destinations. |
+| `HTTPS_PROXY`, `https_proxy` | Selects the upstream proxy for HTTPS destinations. |
+| `NO_PROXY`, `no_proxy` | Lists destinations that bypass the upstream proxy. |
+
+For each pair, BuildKit uses the uppercase variable when it is non-empty and
+falls back to the lowercase variable. The HTTP and HTTPS settings are
+independent: `HTTP_PROXY` and `http_proxy` do not apply to HTTPS destinations.
+
+BuildKit injects `ALL_PROXY` and `all_proxy` into proxy-network execs. It does
+not read these variables from the `buildkitd` environment when it configures
+upstream routing.
+
+Proxy values can be complete `http://`, `https://`, `socks5://`, or
+`socks5h://` URLs. A bare `host[:port]` uses HTTP.
+
+`NO_PROXY` is a comma-separated list of domain names, IP addresses, and CIDR
+prefixes. Domain names and IP addresses can include a port. When a destination
+matches the list, BuildKit's proxy connects to it directly. A value of `*`
+makes direct connections to all destinations. `NO_PROXY` controls how the
+BuildKit proxy reaches the destination. It does not change the proxy variables
+in the exec.
+
+These settings apply to all proxy-network execs. Proxy settings passed to an
+individual exec do not change upstream routing because its proxy variables
+point to BuildKit's internal proxy. If an upstream proxy URL is invalid, the
+proxy-network exec fails to start instead of connecting directly.
+
 ## Request capture and logs
 
 The proxy records network requests made by exec steps. Build output includes a
@@ -126,6 +171,6 @@ The current implementation is Linux-focused. Rootless workers also have the
 usual rootless networking limitations, where worker networking may behave like
 host networking.
 
-Applications that ignore `HTTP_PROXY` and `HTTPS_PROXY`, use custom trust
-stores, or open raw TCP connections cannot bypass the proxy. That traffic is
-blocked instead of being captured.
+Applications that ignore the injected proxy environment variables, use custom
+trust stores, or open raw TCP connections cannot bypass the proxy. That traffic
+is blocked instead of being captured.

executor/containerdexecutor/executor.go

@@ -192,7 +192,7 @@ func (w *containerdExecutor) Run(ctx context.Context, id string, root executor.M
 	}
 	defer namespace.Close()
 	if proxyNS, ok := namespace.(network.ProxyNamespace); ok {
-		meta.Env = append(meta.Env, proxyNS.ProxyEnv()...)
+		meta.Env = executor.ReplaceEnv(meta.Env, proxyNS.ProxyEnv())
 		cleanProxyCA, err := executor.InjectProxyCA(details.rootfsPath, proxyNS.ProxyCACert())
 		if err != nil {
 			return nil, err

executor/env_test.go

@@ -0,0 +1,30 @@
+package executor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestReplaceEnv(t *testing.T) {
+	env := []string{
+		"FOO=one",
+		"HTTP_PROXY=http://upstream.example",
+		"http_proxy=http://upstream.example",
+		"NO_PROXY=example.com",
+		"BAR=two",
+	}
+	replacement := []string{
+		"HTTP_PROXY=http://buildkit-proxy",
+		"http_proxy=http://buildkit-proxy",
+		"NO_PROXY=localhost",
+	}
+
+	require.Equal(t, []string{
+		"FOO=one",
+		"BAR=two",
+		"HTTP_PROXY=http://buildkit-proxy",
+		"http_proxy=http://buildkit-proxy",
+		"NO_PROXY=localhost",
+	}, ReplaceEnv(env, replacement))
+}

executor/executor.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 	"net"
+	"strings"
 	"syscall"
 
 	"github.com/containerd/containerd/v2/core/mount"
@@ -34,6 +35,24 @@ type Meta struct {
 	RemoveMountStubsRecursive bool
 }
 
+// ReplaceEnv removes entries whose names are present in replacement, then
+// appends replacement in order.
+func ReplaceEnv(env, replacement []string) []string {
+	names := make(map[string]struct{}, len(replacement))
+	for _, entry := range replacement {
+		name, _, _ := strings.Cut(entry, "=")
+		names[name] = struct{}{}
+	}
+	out := make([]string, 0, len(env)+len(replacement))
+	for _, entry := range env {
+		name, _, _ := strings.Cut(entry, "=")
+		if _, ok := names[name]; !ok {
+			out = append(out, entry)
+		}
+	}
+	return append(out, replacement...)
+}
+
 type MountableRef interface {
 	Mount() ([]mount.Mount, func() error, error)
 	IdentityMapping() *user.IdentityMapping

executor/runcexecutor/executor.go

@@ -213,7 +213,7 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
 		return nil, err
 	}
 	if proxyNS, ok := namespace.(network.ProxyNamespace); ok {
-		meta.Env = append(meta.Env, proxyNS.ProxyEnv()...)
+		meta.Env = executor.ReplaceEnv(meta.Env, proxyNS.ProxyEnv())
 	}
 	doReleaseNetwork := true
 	defer func() {

util/network/proxyprovider/provider_linux.go

@@ -281,8 +281,10 @@ func (n *proxyNS) ProxyEnv() []string {
 	return []string{
 		"HTTP_PROXY=" + proxy,
 		"HTTPS_PROXY=" + proxy,
+		"ALL_PROXY=" + proxy,
 		"http_proxy=" + proxy,
 		"https_proxy=" + proxy,
+		"all_proxy=" + proxy,
 		"NO_PROXY=" + noProxy,
 		"no_proxy=" + noProxy,
 	}
@@ -381,6 +383,10 @@ func (n *proxyNS) startProxy(ctx context.Context, proxy *network.ProxyConfig) er
 	if proxy == nil {
 		return errors.New("proxy network config is required")
 	}
+	hasHTTPSProxy, err := upstreamProxyEnvironment()
+	if err != nil {
+		return err
+	}
 	ln, err := listenInNetNS(ctx, n.proxyNSPath, "tcp4", net.JoinHostPort(n.hostIP.String(), "0"))
 	if err != nil {
 		return errors.WithStack(err)
@@ -402,12 +408,17 @@ func (n *proxyNS) startProxy(ctx context.Context, proxy *network.ProxyConfig) er
 	n.egressNS = egressNS
 	transport := n.provider.transport.Clone()
 	transport.DialContext = dialer.DialContext
+	transport.Proxy = http.ProxyFromEnvironment
+	var roundTripper http.RoundTripper = transport
+	if hasHTTPSProxy {
+		roundTripper = configureTransportForUpstream(transport)
+	}
 	n.transport = transport
 	handler := &proxyHandler{
 		provider:  n.provider,
 		policy:    proxy.Policy,
 		capture:   proxy.Capture,
-		transport: transport,
+		transport: roundTripper,
 	}
 	n.server = &http.Server{
 		Handler:           handler,
@@ -503,7 +514,7 @@ type proxyHandler struct {
 	provider  *provider
 	policy    network.ProxyPolicy
 	capture   *network.ProxyCapture
-	transport *http.Transport
+	transport http.RoundTripper
 }
 
 func (h *proxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

util/network/proxyprovider/provider_linux_test.go

@@ -7,9 +7,12 @@ import (
 	"container/list"
 	"context"
 	"crypto/x509"
+	"encoding/base64"
+	"io"
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"strings"
 	"testing"
 	"time"
@@ -22,6 +25,120 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestProxyHandlerUsesUpstreamHTTPProxy(t *testing.T) {
+	requestCh := make(chan *http.Request, 1)
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestCh <- r.Clone(r.Context())
+		_, _ = w.Write([]byte("from upstream proxy"))
+	}))
+	t.Cleanup(upstream.Close)
+
+	handler := newTestProxyHandler(t, nil)
+	transport := handler.transport.(*http.Transport)
+	proxyURL, err := url.Parse(upstream.URL)
+	require.NoError(t, err)
+	transport.Proxy = http.ProxyURL(proxyURL)
+	resp := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "http://destination.invalid/file", nil)
+
+	handler.ServeHTTP(resp, req)
+
+	require.Equal(t, http.StatusOK, resp.Code)
+	require.Equal(t, "from upstream proxy", resp.Body.String())
+	proxied := <-requestCh
+	require.Equal(t, "destination.invalid", proxied.Host)
+	require.Equal(t, "http://destination.invalid/file", proxied.URL.String())
+}
+
+func TestProxyHandlerUsesUpstreamProxyForHTTPS(t *testing.T) {
+	type connectRequest struct {
+		host string
+		auth string
+	}
+
+	originRequestCh := make(chan *http.Request, 1)
+	origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originRequestCh <- r.Clone(r.Context())
+		w.Header().Set("Connection", "close")
+		_, _ = w.Write([]byte("from TLS origin"))
+	}))
+	t.Cleanup(origin.Close)
+
+	connectCh := make(chan connectRequest, 1)
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodConnect {
+			http.Error(w, "CONNECT required", http.StatusMethodNotAllowed)
+			return
+		}
+		connectCh <- connectRequest{
+			host: r.Host,
+			auth: r.Header.Get("Proxy-Authorization"),
+		}
+		originConn, err := net.Dial("tcp", origin.Listener.Addr().String())
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		}
+		clientConn, _, err := w.(http.Hijacker).Hijack()
+		if err != nil {
+			_ = originConn.Close()
+			return
+		}
+		_, _ = io.WriteString(clientConn, "HTTP/1.1 200 Connection Established\r\n\r\n")
+		go func() {
+			_, _ = io.Copy(originConn, clientConn)
+			_ = originConn.Close()
+		}()
+		_, _ = io.Copy(clientConn, originConn)
+		_ = clientConn.Close()
+	}))
+	t.Cleanup(upstream.Close)
+
+	_, originPort, err := net.SplitHostPort(origin.Listener.Addr().String())
+	require.NoError(t, err)
+	targetURL := "https://example.com:" + originPort + "/file"
+	proxyURL := strings.Replace(upstream.URL, "http://", "http://proxy-user:proxy-pass@", 1)
+	handler := newTestProxyHandler(t, nil)
+	transport := handler.transport.(*http.Transport)
+	parsedProxyURL, err := url.Parse(proxyURL)
+	require.NoError(t, err)
+	transport.Proxy = http.ProxyURL(parsedProxyURL)
+	pool := x509.NewCertPool()
+	pool.AddCert(origin.Certificate())
+	transport.TLSClientConfig.RootCAs = pool
+	req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, targetURL, nil)
+	require.NoError(t, err)
+	req.Close = true
+	req.Header.Set("Proxy-Authorization", "Basic client-supplied")
+
+	resp, err := handler.roundTrip(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, "from TLS origin", string(body))
+
+	connect := <-connectCh
+	require.Equal(t, "example.com:"+originPort, connect.host)
+	expectedAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte("proxy-user:proxy-pass"))
+	require.Equal(t, expectedAuth, connect.auth)
+	originRequest := <-originRequestCh
+	require.Equal(t, http.MethodGet, originRequest.Method)
+	require.Equal(t, "/file", originRequest.URL.Path)
+	require.Empty(t, originRequest.Header.Get("Proxy-Authorization"))
+}
+
+func TestProxyNamespaceEnvIncludesAllProxy(t *testing.T) {
+	ln, err := net.Listen("tcp4", "127.0.0.1:0")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = ln.Close() })
+
+	ns := proxyNS{ln: ln}
+	proxy := "http://" + ln.Addr().String()
+	require.Contains(t, ns.ProxyEnv(), "ALL_PROXY="+proxy)
+	require.Contains(t, ns.ProxyEnv(), "all_proxy="+proxy)
+}
+
 func TestProxyHandlerCapturesGetMaterial(t *testing.T) {
 	methodCh := make(chan string, 1)
 	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -86,8 +203,9 @@ func TestProxyHandlerRoundTripIgnoresClientContextCancel(t *testing.T) {
 	pool := x509.NewCertPool()
 	pool.AddCert(upstream.Certificate())
 	handler := newTestProxyHandler(t, nil)
-	handler.transport.TLSClientConfig = upstream.Client().Transport.(*http.Transport).TLSClientConfig.Clone()
-	handler.transport.TLSClientConfig.RootCAs = pool
+	transport := handler.transport.(*http.Transport)
+	transport.TLSClientConfig = upstream.Client().Transport.(*http.Transport).TLSClientConfig.Clone()
+	transport.TLSClientConfig.RootCAs = pool
 
 	ctx, cancel := context.WithCancelCause(t.Context())
 	cancel(context.Canceled)