atenet: h2 on the HTTPS ingress and mirror the protocol to actors

cmd/atenet/internal/router/xds.go

@@ -129,6 +129,11 @@ const defaultExtProcMaxRequests = 2048
 // does not silently stretch every shutdown past terminationGracePeriodSeconds.
 // Operators who raise --route-timeout and want such turns to survive a drain
 // must raise --drain-timeout (and the grace period) explicitly.
+//
+// TODO(liorlieberman): this ceiling also cuts off gRPC server-streaming and
+// bidi RPCs longer than 10s, so streaming gRPC effectively requires raising
+// --route-timeout today. We need to fix it so streaming works without
+// inflating the timeout for all workloads.
 const defaultRouteTimeout = 10 * time.Second
 
 // envoyDefaultStreamIdleTimeout is the stream idle timeout Envoy applies when
@@ -759,19 +764,21 @@ func (x *XdsServer) buildOriginalDstCluster() *clusterv3.Cluster {
 
 	if ts := x.buildUpstreamTransportSocket(); ts != nil {
 		cluster.TransportSocket = ts
-		// The atunnel ingress server terminates TLS and reverse-proxies to the
-		// actor over HTTP/1.1.
-		httpOpts := newAny(&httpv3.HttpProtocolOptions{
-			UpstreamProtocolOptions: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{
-				ExplicitHttpConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig{
-					ProtocolConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{
-						HttpProtocolOptions: &corev3.Http1ProtocolOptions{},
+		// Mirror the downstream protocol to atunnel so HTTP/2 requests stay
+		// HTTP/2 and gRPC keeps trailers and streaming end to end. atunnel
+		// gates its own actor leg: it forwards HTTP/2 only for gRPC and
+		// downgrades everything else, so an HTTP/2 client cannot break an
+		// HTTP/1.1-only actor. Without upstream mTLS there is no atunnel leg
+		// to carry HTTP/2, so that mode keeps Envoy's implicit HTTP/1.1.
+		cluster.TypedExtensionProtocolOptions = map[string]*anypb.Any{
+			httpProtocolOptionsName: newAny(&httpv3.HttpProtocolOptions{
+				UpstreamProtocolOptions: &httpv3.HttpProtocolOptions_UseDownstreamProtocolConfig{
+					UseDownstreamProtocolConfig: &httpv3.HttpProtocolOptions_UseDownstreamHttpConfig{
+						HttpProtocolOptions:  &corev3.Http1ProtocolOptions{},
+						Http2ProtocolOptions: &corev3.Http2ProtocolOptions{},
 					},
 				},
-			},
-		})
-		cluster.TypedExtensionProtocolOptions = map[string]*anypb.Any{
-			httpProtocolOptionsName: httpOpts,
+			}),
 		}
 	}
 
@@ -1170,9 +1177,12 @@ func (x *XdsServer) buildListener() *listenerv3.Listener {
 // socket shared by every TLS-terminating listener: it serves the SDS-fetched
 // certificate at HTTPSCertSecretName (see buildTlsSecret), which UpdateSnapshot
 // includes whenever any TLS listener (HTTPS or CONNECT-TLS) is configured.
-func buildDownstreamTlsTransportSocket() *corev3.TransportSocket {
+// alpnProtocols, when non-empty, is offered during the handshake; empty
+// advertises nothing, leaving clients on HTTP/1.1.
+func buildDownstreamTlsTransportSocket(alpnProtocols []string) *corev3.TransportSocket {
 	tlsConfig := &tlsv3.DownstreamTlsContext{
 		CommonTlsContext: &tlsv3.CommonTlsContext{
+			AlpnProtocols: alpnProtocols,
 			TlsCertificateSdsSecretConfigs: []*tlsv3.SdsSecretConfig{
 				{
 					Name: HTTPSCertSecretName,
@@ -1198,6 +1208,13 @@ func buildDownstreamTlsTransportSocket() *corev3.TransportSocket {
 func (x *XdsServer) buildHttpsListener() *listenerv3.Listener {
 	hcm := x.buildHcm("ingress_https", true)
 
+	// gRPC requires a negotiated "h2"; http/1.1 keeps plain HTTPS clients
+	// working alongside it. HTTP/1.1-only actors are safe either way: atunnel
+	// downgrades every non-gRPC request to HTTP/1.1 on the actor leg (see
+	// atunnel.protocolMirrorTransport), so offering h2 at the edge cannot
+	// change what an actor receives.
+	alpn := []string{"h2", "http/1.1"}
+
 	return &listenerv3.Listener{
 		Name: IngressHTTPSListener,
 		Address: &corev3.Address{
@@ -1221,7 +1238,7 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener {
 						},
 					},
 				},
-				TransportSocket: buildDownstreamTlsTransportSocket(),
+				TransportSocket: buildDownstreamTlsTransportSocket(alpn),
 			},
 		},
 	}
@@ -1287,7 +1304,10 @@ func (x *XdsServer) buildConnectTerminateTLSListener() *listenerv3.Listener {
 						},
 					},
 				},
-				TransportSocket: buildDownstreamTlsTransportSocket(),
+				// No ALPN: CONNECT-TLS clients speak HTTP/1.1 CONNECT
+				// today, and the HTTPS listener's h2 offer deliberately
+				// leaves this listener alone.
+				TransportSocket: buildDownstreamTlsTransportSocket(nil),
 			},
 		},
 	}

cmd/atenet/internal/router/xds_test.go

@@ -40,6 +40,7 @@ import (
 	routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	hcmv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
 	tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
 	discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
 	secretgrpc "github.com/envoyproxy/go-control-plane/envoy/service/secret/v3"
 	cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
@@ -829,6 +830,55 @@ func TestXdsServer_BuildOriginalDstCluster_UsesMetadataKey(t *testing.T) {
 	}
 }
 
+// TestXdsServer_ActorClusterProtocolOptions pins where downstream-protocol
+// mirroring is allowed: only on the mTLS atunnel leg, where atunnel guards
+// HTTP/1.1-only actors by downgrading non-gRPC HTTP/2 (see
+// atunnel.protocolMirrorTransport). The legacy plaintext cluster dials the
+// actor directly with no such guard, so it must carry no protocol options at
+// all — Envoy's implicit HTTP/1.1.
+func TestXdsServer_ActorClusterProtocolOptions(t *testing.T) {
+	x := NewXdsServer(18000)
+
+	if opts := x.buildOriginalDstCluster().GetTypedExtensionProtocolOptions(); len(opts) != 0 {
+		t.Errorf("legacy plaintext actor cluster has protocol options %v, want none (implicit HTTP/1.1)", opts)
+	}
+
+	x.SetUpstreamTls("/run/bundle.pem", "/run/trust.pem", "spiffe://ate.dev/")
+	cluster := x.buildOriginalDstCluster()
+	ts := cluster.GetTransportSocket()
+	if ts == nil {
+		t.Fatal("mTLS actor cluster is missing its transport socket")
+	}
+	// The upstream TLS context must NOT set alpn_protocols: with
+	// use_downstream_protocol_config, Envoy already offers the single ALPN
+	// matching each connection pool's protocol. A static ["h2","http/1.1"]
+	// list would override that, and Go's atunnel server (which negotiates by
+	// server preference) would pick h2 on connections belonging to the
+	// HTTP/1.1 pool, breaking it.
+	upstreamTls := &tlsv3.UpstreamTlsContext{}
+	if err := ts.GetTypedConfig().UnmarshalTo(upstreamTls); err != nil {
+		t.Fatalf("Failed to unmarshal UpstreamTlsContext: %v", err)
+	}
+	if alpn := upstreamTls.GetCommonTlsContext().GetAlpnProtocols(); len(alpn) != 0 {
+		t.Errorf("upstream TLS context ALPN = %v, want none (per-pool ALPN comes from use_downstream_protocol_config)", alpn)
+	}
+	raw, ok := cluster.GetTypedExtensionProtocolOptions()[httpProtocolOptionsName]
+	if !ok {
+		t.Fatalf("mTLS actor cluster is missing %q protocol options", httpProtocolOptionsName)
+	}
+	protoOpts := &httpv3.HttpProtocolOptions{}
+	if err := raw.UnmarshalTo(protoOpts); err != nil {
+		t.Fatalf("Failed to unmarshal HttpProtocolOptions: %v", err)
+	}
+	downstream := protoOpts.GetUseDownstreamProtocolConfig()
+	if downstream == nil {
+		t.Fatalf("mTLS actor cluster protocol options = %v, want use_downstream_protocol_config", protoOpts)
+	}
+	if downstream.GetHttp2ProtocolOptions() == nil {
+		t.Error("use_downstream_protocol_config must enable HTTP/2 so gRPC keeps trailers on the atunnel leg")
+	}
+}
+
 // TestXdsServer_BuildRoutes_DerivesTargetPortHeader covers the fix for atunnel
 // needing the target port as a real header (it can't read Envoy's dynamic
 // metadata directly): rather than ext_proc building that header mutation
@@ -1055,3 +1105,66 @@ func TestSnapshotVersionsUniqueAcrossRestarts(t *testing.T) {
 		seen[v] = true
 	}
 }
+
+// downstreamTLS extracts the DownstreamTlsContext from a listener's first
+// filter chain.
+func downstreamTLS(t *testing.T, raw any) *tlsv3.DownstreamTlsContext {
+	t.Helper()
+	l := raw.(*listenerv3.Listener)
+	dtc := &tlsv3.DownstreamTlsContext{}
+	if err := l.GetFilterChains()[0].GetTransportSocket().GetTypedConfig().UnmarshalTo(dtc); err != nil {
+		t.Fatalf("Failed to unmarshal DownstreamTlsContext: %v", err)
+	}
+	return dtc
+}
+
+// TestXdsServer_ALPN pins the per-listener ALPN contract. The HTTPS ingress
+// listener always offers h2 before http/1.1: gRPC over TLS requires a
+// negotiated "h2", and the offer is unconditional because atunnel downgrades
+// every non-gRPC request to HTTP/1.1 on the actor leg (see
+// atunnel.protocolMirrorTransport and TestProtocolMirrorTransport), so an
+// HTTP/1.1-only actor cannot tell what the client negotiated at the edge. The
+// CONNECT-TLS listener stays ALPN-free: its clients speak HTTP/1.1 CONNECT,
+// and an h2 offer there would move them onto extended CONNECT the tunnel path
+// does not serve.
+func TestXdsServer_ALPN(t *testing.T) {
+	const certPath = "/run/servicedns.podcert.ate.dev/credential-bundle.pem"
+
+	server := NewXdsServer(18000)
+	server.SetConfig(8085, 50053, "127.0.0.1")
+	server.SetTlsConfig(8443, certPath)
+	server.SetConnectPorts(0, 8444)
+	if err := server.UpdateSnapshot(); err != nil {
+		t.Fatalf("UpdateSnapshot failed: %v", err)
+	}
+	res, err := server.snapshot.GetSnapshot(NodeID)
+	if err != nil {
+		t.Fatalf("Failed to get snapshot: %v", err)
+	}
+	listeners := map[string]any{}
+	for name, l := range res.(*cachev3.Snapshot).GetResources(resourcev3.ListenerType) {
+		listeners[name] = l
+	}
+
+	// h2 first: ALPN is server-preference, and a client that can speak HTTP/2
+	// must land on it rather than on http/1.1.
+	alpn := downstreamTLS(t, listeners[IngressHTTPSListener]).GetCommonTlsContext().GetAlpnProtocols()
+	if len(alpn) != 2 || alpn[0] != "h2" || alpn[1] != "http/1.1" {
+		t.Errorf("HTTPS listener ALPN = %v, want [h2 http/1.1]", alpn)
+	}
+	if alpn := downstreamTLS(t, listeners["connect_terminate_tls"]).GetCommonTlsContext().GetAlpnProtocols(); len(alpn) != 0 {
+		t.Errorf("CONNECT-TLS listener ALPN = %v, want none", alpn)
+	}
+
+	// The offer is only half the contract: the HCM must honor whatever ALPN
+	// negotiated. An explicit HTTP1 codec here would turn every h2 client
+	// into a connection error while the ALPN list still looked right.
+	https := listeners[IngressHTTPSListener].(*listenerv3.Listener)
+	hcm := &hcmv3.HttpConnectionManager{}
+	if err := https.GetFilterChains()[0].GetFilters()[0].GetTypedConfig().UnmarshalTo(hcm); err != nil {
+		t.Fatalf("Failed to unmarshal the HTTPS listener's HCM config: %v", err)
+	}
+	if hcm.GetCodecType() != hcmv3.HttpConnectionManager_AUTO {
+		t.Errorf("HTTPS listener HCM codec = %v, want AUTO so the negotiated protocol is honored", hcm.GetCodecType())
+	}
+}

internal/atunnel/ingress.go

@@ -126,7 +126,7 @@ func NewServer(cfg Config) (*Server, error) {
 		return nil, fmt.Errorf("atunnel: trust bundle %q contains no certificates", cfg.TrustBundlePath)
 	}
 
-	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport := newProtocolMirrorTransport()
 	proxy := &httputil.ReverseProxy{
 		Rewrite: func(pr *httputil.ProxyRequest) {
 			pr.SetURL(cfg.Upstream)
@@ -181,6 +181,65 @@ func NewServer(cfg Config) (*Server, error) {
 	return s, nil
 }
 
+// isGRPC reports whether an incoming HTTP request conforms to the gRPC over
+// HTTP/2 specification: HTTP/2 framing, POST method, and a Content-Type of
+// "application/grpc" (optionally with a "+<subtype>" or ";<parameters>").
+// Notably this excludes gRPC-Web that works over HTTP/1.1.
+// See https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests
+func isGRPC(r *http.Request) bool {
+	if r.ProtoMajor != 2 || r.Method != http.MethodPost {
+		return false
+	}
+	ct := strings.ToLower(r.Header.Get("Content-Type"))
+	const base = "application/grpc"
+	if ct == base {
+		return true
+	}
+	if strings.HasPrefix(ct, base) {
+		switch ct[len(base)] {
+		case '+', ';':
+			return true
+		}
+	}
+	return false
+}
+
+// protocolMirrorTransport picks the actor-leg protocol per request. gRPC goes
+// out as prior-knowledge h2c, since it needs HTTP/2 trailers and full-duplex
+// streaming; everything else is translated to HTTP/1.1 even if it arrived over
+// HTTP/2 at the edge. This is to not break previous logic so an HTTP/1.1-only actor keeps working whatever the
+// client negotiated.
+type protocolMirrorTransport struct {
+	h1, h2c *http.Transport
+}
+
+func newProtocolMirrorTransport() protocolMirrorTransport {
+	h1 := http.DefaultTransport.(*http.Transport).Clone()
+	h2c := http.DefaultTransport.(*http.Transport).Clone()
+	protocols := new(http.Protocols)
+	protocols.SetUnencryptedHTTP2(true)
+	h2c.Protocols = protocols
+	return protocolMirrorTransport{h1: h1, h2c: h2c}
+}
+
+func (t protocolMirrorTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	if isGRPC(r) {
+		return t.h2c.RoundTrip(r)
+	}
+	return t.h1.RoundTrip(r)
+}
+
+// CloseIdleConnections keeps Deactivate's idle-connection cleanup working:
+// closeIdleUpstreamConnections discovers it through a duck-typed interface
+// assertion that silently returns false if the method disappears, so pin it
+// at compile time.
+var _ interface{ CloseIdleConnections() } = protocolMirrorTransport{}
+
+func (t protocolMirrorTransport) CloseIdleConnections() {
+	t.h1.CloseIdleConnections()
+	t.h2c.CloseIdleConnections()
+}
+
 func loadCredentialBundle(path string) (*tls.Certificate, error) {
 	pemBytes, err := os.ReadFile(path)
 	if err != nil {
@@ -202,9 +261,15 @@ func (s *Server) Serve(ctx context.Context, lis net.Listener) error {
 // separate listener so ordinary actor ingress remains a request proxy, while
 // the router can use this listener for a bidirectional tunnel.
 func (s *Server) ServeConnect(ctx context.Context, lis net.Listener) error {
-	// Keep ALPN confined to the CONNECT listener. The established ingress
-	// listener is explicitly HTTP/1.1 in the router's upstream cluster, while
-	// the CONNECT listener supports either HTTP/1.1 or HTTP/2.
+	// Offer both protocols explicitly, matching the ingress listener (whose
+	// ServeTLS advertises h2 and http/1.1 by default). The router's actor
+	// cluster mirrors the downstream protocol, so either can arrive here.
+	// The tunnel itself relays opaque bytes, so protocolMirrorTransport's
+	// gRPC-only gate does not apply to CONNECT traffic — and that is the
+	// point: this listener is the basis of the planned tunnel-based ingress,
+	// where ordinary actor traffic arrives here as a spliced tunnel and all
+	// protocol decisions move to the router's route config, leaving atunnel
+	// with no request parsing at all.
 	tlsConfig := s.tlsConfig.Clone()
 	tlsConfig.NextProtos = []string{"h2", "http/1.1"}
 	return s.serve(ctx, lis, http.HandlerFunc(s.ServeConnectHTTP), tlsConfig)