From 8cd2a012691dcca9951c1e975dbfce4a6ec000bb Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 27 Aug 2026 15:52:14 -0700 Subject: [PATCH 01/22] hack: drop the IPv6 kubeconfig repoint On an IPv6-only cluster the script rewrote kind's `https://[::1]:PORT` kubeconfig entry to `https://localhost:PORT` unconditionally. That breaks any host whose `/etc/hosts` leaves `localhost` off the `::1` line, including the Ubuntu cloud image Lima runs: `localhost` resolves v4-only and every later kubectl fails at connect. The rewrite existed for one case, a macOS client reaching kind inside a Lima VM, where limactl re-forwards the published port to the host's v4 loopback only. Running the loop inside the guest reaches `[::1]` directly and avoids that path entirely. --- hack/create-kind-cluster.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index feb8732540..86857c7849 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -145,21 +145,6 @@ if [[ "${IP_FAMILY}" != "ipv4" && exit 1 fi -# For ipv6 kind writes a kubeconfig pointing at [::1], the address it published -# the apiserver on, which only works for a client on the Docker host itself: a -# VM-hosted daemon (Lima on macOS) forwards the port to the *v4* loopback, so -# every kubectl below fails at connect. localhost is a SAN on the apiserver -# cert and lets the client pick a family that works from either side. -if [[ "${IP_FAMILY}" == "ipv6" ]]; then - server="$(kubectl config view \ - -o jsonpath="{.clusters[?(@.name==\"${KUBECTL_CONTEXT}\")].cluster.server}")" - if [[ "${server}" == "https://[::1]:"* ]]; then - echo "Repointing the kubeconfig for '${KUBECTL_CONTEXT}' at localhost..." - kubectl config set-cluster "${KUBECTL_CONTEXT}" \ - --server="https://localhost:${server##*:}" >/dev/null - fi -fi - # 2.5 Enable Proxy ARP/NDP on kind nodes for gVisor loopback pod-to-pod networking echo "Enabling Proxy ARP/NDP on kind nodes..." for node in $("${ROOT}"/hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do From 245671050b67e11d2ca58ce020d974fc2b0e95f7 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 27 Aug 2026 15:52:37 -0700 Subject: [PATCH 02/22] hack: fix DNS on IPv6-only kind clusters On a fresh IPv6-only kind cluster nothing resolves from inside a pod, so the install never completes. CoreDNS inherits the node's IPv4 resolver, which a v6-only pod cannot reach, and the in-cluster registry has no name a pod can look up. This gives the cluster its own Corefile, gated on ipv6: a forward to an IPv6 upstream, overridable with IPV6_DNS_UPSTREAM, and a kind-registry:53 server block so atelet can pull from its own network namespace. The Corefile patch runs kubectl straight after `kind create`, which returns before the apiserver answers, so the script now waits for the control plane from inside the node first. --- hack/create-kind-cluster.sh | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index 86857c7849..231785b096 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" +IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" if [[ $# -gt 0 ]]; then case "$1" in @@ -31,6 +32,9 @@ if [[ $# -gt 0 ]]; then echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" + echo " (default: Google Public DNS). These replace the host's resolver, so any" + echo " split-horizon names it served stop resolving from pods." exit 0 ;; esac @@ -135,6 +139,23 @@ fi echo "Creating kind cluster '${KIND_CLUSTER_NAME}'..." "${ROOT}"/hack/kind.sh create cluster --name "${KIND_CLUSTER_NAME}" --config "${ROOT}/bin/kind-config.yaml" +# kind create returns before the apiserver answers, and every kubectl below races +# it. Poll from inside the node, where the answer does not depend on how the +# daemon published the port. +echo "Waiting for the control plane to answer..." +for attempt in $(seq 60); do + if docker exec "${KIND_CLUSTER_NAME}-control-plane" \ + kubectl --kubeconfig=/etc/kubernetes/admin.conf get --raw /healthz >/dev/null 2>&1; then + break + fi + if [[ "${attempt}" == 60 ]]; then + echo "error: the control plane did not answer /healthz within 2m of create:" >&2 + echo " docker logs ${KIND_CLUSTER_NAME}-control-plane" >&2 + exit 1 + fi + sleep 2 +done + # A daemon with IPv6 off hands kind a v4-only network whatever it asked for. if [[ "${IP_FAMILY}" != "ipv4" && "$(docker network inspect kind --format '{{.EnableIPv6}}')" != "true" ]]; then @@ -181,6 +202,50 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name} docker network connect "kind" "${reg_name}" fi +# 4.5. Point CoreDNS at an IPv6 resolver and teach it the registry's name +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..." + reg_v6="$(docker inspect "${reg_name}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}' 2>/dev/null || true)" + if [[ -z "${reg_v6}" ]]; then + echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2 + exit 1 + fi + + # CoreDNS runs dnsPolicy: Default and inherits the node's IPv4 resolver, which + # no pod here can reach. + corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}')" + search="forward . /etc/resolv.conf" + # $search unquoted: bash 3.2 splices the quotes in literally. + patched="${corefile/$search/forward . ${IPV6_DNS_UPSTREAM}}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " the Corefile layout changed upstream; update this block" >&2 + exit 1 + fi + + # Step 3's registry wiring is node-side, while atelet pulls from its own netns, + # where "kind-registry" does not resolve. Own zone, so no fallthrough is needed: + # only this name reaches the hosts stanza. + patched="${patched} +${reg_name}:53 { + errors + hosts { + ${reg_v6} ${reg_name} + } +}" + + # A YAML patch file avoids escaping the Corefile's newlines into JSON. + { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ + > "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \ + --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \ + --timeout=120s +fi + # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." cat < Date: Thu, 27 Aug 2026 12:41:30 -0700 Subject: [PATCH 03/22] hack: add opt-in NAT64 support to IPv6-only kind clusters An IPv6-only cluster on a host with no IPv6 egress comes up but cannot reach anything, and the setup that fixes that lived only inside a CI job, so the cluster it tests could not be reproduced by hand. setup-nat64.sh now brings up the translator and IPV6_DNS64_PREFIX points cluster DNS at it, which also takes the Corefile rewriting back out of the job. Off by default, because translate_all replaces reachable AAAA answers with unreachable ones on any host whose IPv6 egress already works. --- hack/create-kind-cluster.sh | 62 +++++++++++++ hack/setup-nat64.sh | 174 ++++++++++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100755 hack/setup-nat64.sh diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index 231785b096..0bf2df73f5 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -22,6 +22,7 @@ KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="${KIND_REGISTRY_PORT:-5001}" IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" +IPV6_DNS64_PREFIX="${IPV6_DNS64_PREFIX:-}" if [[ $# -gt 0 ]]; then case "$1" in @@ -35,6 +36,11 @@ if [[ $# -gt 0 ]]; then echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" echo " (default: Google Public DNS). These replace the host's resolver, so any" echo " split-horizon names it served stop resolving from pods." + echo " IPV6_DNS64_PREFIX NAT64 prefix cluster DNS synthesizes external names into when" + echo " IP_FAMILY=ipv6, e.g. 64:ff9b::/96 (default: empty, no DNS64)." + echo " Set it only on a host with no IPv6 egress, together with" + echo " hack/setup-nat64.sh: it routes every external name through the" + echo " prefix, including names that already have reachable AAAA records." exit 0 ;; esac @@ -236,6 +242,62 @@ ${reg_name}:53 { } }" + if [[ -n "${IPV6_DNS64_PREFIX}" ]]; then + echo "Synthesizing external names into ${IPV6_DNS64_PREFIX}..." + # Plain DNS64 synthesizes only for names with no AAAA, and the names that + # matter here have real AAAA records pointing at addresses a host with no + # IPv6 egress cannot reach. Only translate_all forces them through the + # prefix -- which is why this is opt-in: where IPv6 egress does work it + # replaces reachable answers with unreachable ones. + # + # translate_all cannot share a server block with the cluster zones. dns64 + # wraps the plugin chain below it and answers AAAA by synthesizing from A, + # so for an AAAA-only name it synthesizes from nothing and returns an empty + # answer. Every ClusterIP here is AAAA-only, so one block would take out all + # in-cluster service discovery. Re-zone what kind shipped to the cluster + # zones, lift its forwarder out, and give dns64 the catch-all. + # + # The prefix goes inside the block, not on the `dns64` line: CoreDNS takes + # `dns64 PREFIX { ... }` without complaint and then never applies the block, + # so translate_all silently does nothing and only AAAA-less names get + # synthesized -- which looks like it works until something with a real AAAA + # is the thing that has to be reached. + rezoned="$(printf '%s\n' "${patched}" | awk ' + NR == 1 && /^\.:53[[:space:]]*\{/ { + print "cluster.local:53 in-addr.arpa:53 ip6.arpa:53 {"; first = 1; next + } + first && /^ forward([[:space:]].*)?\{$/ { skip = 1; next } + first && skip && /^ \}$/ { skip = 0; next } + first && skip { next } + first && /^\}$/ { first = 0 } + { print } + ')" + case "${rezoned}" in + "cluster.local:53"*) ;; + *) echo "error: the Corefile does not open with the '.:53' block kind ships" >&2 + echo " the Corefile layout changed upstream; update this block" >&2 + exit 1 ;; + esac + if printf '%s' "${rezoned}" | grep -q 'forward'; then + echo "error: a forward block survived the re-zone" >&2 + exit 1 + fi + patched="${rezoned} +.:53 { + errors + dns64 { + prefix ${IPV6_DNS64_PREFIX} + translate_all + } + forward . ${IPV6_DNS_UPSTREAM} { + max_concurrent 1000 + } + cache 30 + loop + reload +}" + fi + # A YAML patch file avoids escaping the Corefile's newlines into JSON. { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ > "${ROOT}/bin/coredns-patch.yaml" diff --git a/hack/setup-nat64.sh b/hack/setup-nat64.sh new file mode 100755 index 0000000000..c59bcf7900 --- /dev/null +++ b/hack/setup-nat64.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +# The well-known prefix, which is also what CoreDNS's dns64 plugin defaults to. +PREFIX="64:ff9b::/96" +DEV="nat64" +PROBE_HOST="${NAT64_PROBE_HOST:-storage.googleapis.com}" + +if [[ $# -gt 0 ]]; then + case "$1" in + -h|--help) + echo "Usage: $0" + echo "Sets up NAT64 (${PREFIX}) so an IPv6-only kind cluster can reach IPv4-only" + echo "destinations. Linux only, and needs sudo. Safe to re-run." + echo + echo "Only needed where the host itself cannot reach the internet over IPv6, which" + echo "is true of GitHub Actions runners and false of most Linux boxes and Lima VMs" + echo "on a v6-capable network. The deciding test, which this script also runs:" + echo + echo " curl -6 -sS -o /dev/null https://${PROBE_HOST}/ && echo 'no NAT64 needed'" + echo + echo "Pair it with IPV6_DNS64_PREFIX=${PREFIX} on hack/create-kind-cluster.sh." + echo "That is the half that makes pods resolve names into the prefix; without it" + echo "the translator sits there unused, and without the translator the synthesized" + echo "addresses go nowhere. Neither is any use alone." + echo + echo "Configured through the environment:" + echo " NAT64_PROBE_HOST Name the translation probe resolves and fetches" + echo " (default: ${PROBE_HOST})." + echo " NAT64_FORCE Set to run even where IPv6 egress already works, which this" + echo " otherwise refuses to do -- enabling forwarding costs the host" + echo " any IPv6 route it learned from a router advertisement." + exit 0 + ;; + esac +fi + +if [[ "$(uname -s)" != "Linux" ]]; then + echo "error: tayga is Linux-only; run this inside the Linux VM hosting your Docker daemon" >&2 + exit 1 +fi + +# Refuse where NAT64 is not needed, because here it is not merely redundant. +# Turning on IPv6 forwarding below makes the kernel stop honouring router +# advertisements on every interface left at the default accept_ra=1, so on a +# host whose IPv6 default route came from an RA this takes that route out when +# it expires -- long after the run that caused it. +# +# Twice, and not on a short timeout. A host with no IPv6 egress fails this in +# milliseconds, so the retry costs CI nothing, while a cold TLS handshake on a +# host that does have egress can outrun a tight deadline -- and that misread is +# the one that does the damage. +if [[ -z "${NAT64_FORCE:-}" ]]; then + for _ in 1 2; do + if curl -6 -sS -m 10 -o /dev/null "https://${PROBE_HOST}/" 2>/dev/null; then + echo "error: this host already reaches ${PROBE_HOST} over IPv6, so it does not need NAT64." >&2 + echo " Create the cluster without IPV6_DNS64_PREFIX and external names resolve normally." >&2 + echo " Set NAT64_FORCE=1 to override; see the accept_ra note in this script first." >&2 + exit 1 + fi + done +fi + +echo "Installing tayga..." +if ! command -v tayga >/dev/null 2>&1; then + sudo apt-get update -qq + sudo apt-get install -y -qq tayga +fi +# Debian's package starts a unit the moment apt finishes, so tayga is already +# running against the stock config before this writes its own. Take it over +# rather than reconfiguring around it: one instance, and a log to point at. +if systemctl cat tayga.service >/dev/null 2>&1; then + sudo systemctl disable --now tayga.service >/dev/null 2>&1 || true +fi + +# tayga answers to .1/::1 and the tun holds .2/::2, so host-originated traffic +# is not sourced from tayga's own address, which is self-addressed rather than +# translatable. The dynamic pool avoids both. +sudo tee /etc/tayga.conf >/dev/null </dev/null 2>&1 || sudo tayga --mktun +sudo ip link set "${DEV}" up +ip -4 addr show dev "${DEV}" | grep -q '192\.168\.255\.2' \ + || sudo ip addr add 192.168.255.2/24 dev "${DEV}" +ip -6 addr show dev "${DEV}" | grep -q '2001:db8:64::2' \ + || sudo ip -6 addr add 2001:db8:64::2/128 dev "${DEV}" +ip -6 route show "${PREFIX}" | grep -q . \ + || sudo ip -6 route add "${PREFIX}" dev "${DEV}" src 2001:db8:64::2 + +sudo sysctl -qw net.ipv4.ip_forward=1 +sudo sysctl -qw net.ipv6.conf.all.forwarding=1 + +sudo iptables -t nat -C POSTROUTING -s 192.168.255.0/24 -j MASQUERADE 2>/dev/null \ + || sudo iptables -t nat -A POSTROUTING -s 192.168.255.0/24 -j MASQUERADE +# Insert, not append: docker sets the FORWARD policy to DROP. Re-run after a +# dockerd restart, which rebuilds the chains these rules live in. +for dir in -i -o; do + sudo iptables -C FORWARD "${dir}" "${DEV}" -j ACCEPT 2>/dev/null \ + || sudo iptables -I FORWARD 1 "${dir}" "${DEV}" -j ACCEPT + sudo ip6tables -C FORWARD "${dir}" "${DEV}" -j ACCEPT 2>/dev/null \ + || sudo ip6tables -I FORWARD 1 "${dir}" "${DEV}" -j ACCEPT +done + +# Unconditionally ours, so a re-run picks up an edited config and the log below +# is never some earlier instance's. +sudo pkill -x tayga 2>/dev/null || true +# -d keeps tayga in the foreground and logs a reason for every packet it +# declines to translate; detaching hides the failures worth diagnosing. +sudo sh -c "nohup tayga -d --config /etc/tayga.conf >/tmp/tayga.log 2>&1 &" +sleep 3 +pgrep -a tayga || { + echo "error: tayga is not running" >&2 + sudo cat /tmp/tayga.log >&2 || true + exit 1 +} + +# A cluster built on a broken translator takes minutes to fail and does it as a +# rollout timeout, so gate here where the message is unambiguous. Map a live A +# record rather than hardcoding one. +v4="$(getent ahostsv4 "${PROBE_HOST}" | awk 'NR==1{print $1}')" +if [[ -z "${v4}" ]]; then + echo "error: cannot resolve an IPv4 address for ${PROBE_HOST}" >&2 + exit 1 +fi +# shellcheck disable=SC2086 +set -- ${v4//./ } +v6="$(printf '64:ff9b::%02x%02x:%02x%02x' "$1" "$2" "$3" "$4")" +echo "NAT64 maps ${v4} -> ${v6}" +# No `|| echo 000`: curl already writes 000 on a connect failure, and a second +# one appends rather than replaces. +code="$(curl -6 -sS -m 15 -o /dev/null -w '%{http_code}' \ + --resolve "${PROBE_HOST}:443:[${v6}]" "https://${PROBE_HOST}/" 2>/dev/null || true)" +case "${code}" in + # Any HTTP status proves the translator carried a TCP stream; GCS answers a + # bare / with 400. ICMP is blocked separately, so a ping here would report a + # failure that does not matter. + 2*|3*|4*) + echo "NAT64 is translating (HTTP ${code})" + echo + echo "Half of the setup. Cluster DNS has to point at the prefix too:" + echo " IP_FAMILY=ipv6 IPV6_DNS64_PREFIX=${PREFIX} $(dirname "$0")/create-kind-cluster.sh" + ;; + *) + echo "error: NAT64 is not translating (HTTP ${code})" >&2 + sudo cat /tmp/tayga.log >&2 || true + exit 1 + ;; +esac From 00c5406d0b335f64514fe4f9ee7d30abf1f894f6 Mon Sep 17 00:00:00 2001 From: lubingtan Date: Wed, 5 Aug 2026 20:58:54 +0800 Subject: [PATCH 04/22] atunnel: support IPv6 original destination lookup --- internal/atunnel/original_dst_linux.go | 68 +++- internal/atunnel/original_dst_linux_test.go | 395 ++++++++++++++++++++ 2 files changed, 444 insertions(+), 19 deletions(-) create mode 100644 internal/atunnel/original_dst_linux_test.go diff --git a/internal/atunnel/original_dst_linux.go b/internal/atunnel/original_dst_linux.go index 07dd0f9344..8b4309119d 100644 --- a/internal/atunnel/original_dst_linux.go +++ b/internal/atunnel/original_dst_linux.go @@ -18,6 +18,7 @@ package atunnel import ( "encoding/binary" + "errors" "fmt" "net" "strconv" @@ -26,10 +27,12 @@ import ( "golang.org/x/sys/unix" ) -// TCPOriginalDestination reads the IPv4 destination preserved by a Linux -// REDIRECT rule. Actor networking is currently IPv4-only. -// TODO(liorlieberman) add the IPv6 IP6T_SO_ORIGINAL_DST variant -// when actor veth setup gains dual-stack support. +// IP6T_SO_ORIGINAL_DST is not generated by golang.org/x/sys/unix. It is +// defined as 80 in linux/netfilter_ipv6/ip6_tables.h. +const ip6tSOOriginalDst = 80 + +// TCPOriginalDestination reads the IPv4 or IPv6 destination preserved by a +// Linux REDIRECT rule. func TCPOriginalDestination(conn net.Conn) (string, error) { tcpConn, ok := conn.(*net.TCPConn) if !ok { @@ -40,21 +43,15 @@ func TCPOriginalDestination(conn net.Conn) (string, error) { return "", fmt.Errorf("atunnel: acquiring TCP syscall connection: %w", err) } - var addr unix.RawSockaddrInet4 var sockoptErr error + var destination string if err := rawConn.Control(func(fd uintptr) { - size := uint32(unsafe.Sizeof(addr)) - _, _, errno := unix.Syscall6( - unix.SYS_GETSOCKOPT, - fd, - unix.SOL_IP, - unix.SO_ORIGINAL_DST, - uintptr(unsafe.Pointer(&addr)), - uintptr(unsafe.Pointer(&size)), - 0, - ) - if errno != 0 { - sockoptErr = errno + destination, sockoptErr = originalIPv4Destination(fd) + // Linux returns ENOENT when the IPv4 original-destination option is + // queried on a redirected IPv6 connection. Only then try the IPv6 + // equivalent, so unrelated IPv4 failures retain their original error. + if errors.Is(sockoptErr, unix.ENOENT) { + destination, sockoptErr = originalIPv6Destination(fd) } }); err != nil { return "", fmt.Errorf("atunnel: accessing TCP socket: %w", err) @@ -62,11 +59,44 @@ func TCPOriginalDestination(conn net.Conn) (string, error) { if sockoptErr != nil { return "", fmt.Errorf("atunnel: reading original TCP destination: %w", sockoptErr) } + return destination, nil +} + +func originalIPv4Destination(fd uintptr) (string, error) { + var addr unix.RawSockaddrInet4 + if errno := getOriginalDestination(fd, unix.SOL_IP, unix.SO_ORIGINAL_DST, unsafe.Pointer(&addr), unsafe.Sizeof(addr)); errno != 0 { + return "", errno + } + return formatOriginalDestination(addr.Addr[:], addr.Port) +} + +func originalIPv6Destination(fd uintptr) (string, error) { + var addr unix.RawSockaddrInet6 + if errno := getOriginalDestination(fd, unix.SOL_IPV6, ip6tSOOriginalDst, unsafe.Pointer(&addr), unsafe.Sizeof(addr)); errno != 0 { + return "", errno + } + return formatOriginalDestination(addr.Addr[:], addr.Port) +} + +func getOriginalDestination(fd uintptr, level, option int, addr unsafe.Pointer, addrSize uintptr) unix.Errno { + size := uint32(addrSize) + _, _, errno := unix.Syscall6( + unix.SYS_GETSOCKOPT, + fd, + uintptr(level), + uintptr(option), + uintptr(addr), + uintptr(unsafe.Pointer(&size)), + 0, + ) + return errno +} - portBytes := (*[2]byte)(unsafe.Pointer(&addr.Port)) +func formatOriginalDestination(ip []byte, rawPort uint16) (string, error) { + portBytes := (*[2]byte)(unsafe.Pointer(&rawPort)) port := binary.BigEndian.Uint16(portBytes[:]) if port == 0 { return "", fmt.Errorf("atunnel: original TCP destination has port zero") } - return net.JoinHostPort(net.IP(addr.Addr[:]).String(), strconv.Itoa(int(port))), nil + return net.JoinHostPort(net.IP(ip).String(), strconv.Itoa(int(port))), nil } diff --git a/internal/atunnel/original_dst_linux_test.go b/internal/atunnel/original_dst_linux_test.go new file mode 100644 index 0000000000..4f26d98055 --- /dev/null +++ b/internal/atunnel/original_dst_linux_test.go @@ -0,0 +1,395 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atunnel + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" + + "github.com/agent-substrate/substrate/internal/ateomnet" + "github.com/agent-substrate/substrate/internal/roottest" +) + +func TestTCPOriginalDestination(t *testing.T) { + roottest.Require(t, "CAP_NET_ADMIN + CAP_SYS_ADMIN for an actor-like network namespace and nftables REDIRECT rule") + + // Model the production path rather than redirecting a locally generated + // connection through OUTPUT. Actor egress enters the worker netns through a + // veth and is redirected in PREROUTING; that is the path on which Linux + // preserves SO_ORIGINAL_DST for atunnel. + actorNS := newTestNetNS(t) + actorIP, hostIP := setupTestVeth(t, actorNS) + // targetListener reserves the port the actor intends to reach. The NAT rule + // below must prevent connections from reaching it. + // + // redirectListener represents atunnel's local egress listener. It receives + // the redirected connection and is therefore the connection on which we ask + // Linux for the original destination. + redirectListener := listenTCP(t, hostIP) + defer redirectListener.Close() + targetListener := listenTCP(t, hostIP) + defer targetListener.Close() + targetPort := targetListener.Addr().(*net.TCPAddr).Port + + table := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: fmt.Sprintf("atunnel_original_dst_test_%d", os.Getpid())} + installOriginalDstRedirect(t, table, actorIP, targetPort, redirectListener.Addr().(*net.TCPAddr).Port) + + clientDone := make(chan error, 1) + go func() { + // From the actor's perspective this is an ordinary connection to + // hostIP:targetPort. The worker's PREROUTING rule redirects it before + // it reaches the host network stack's local delivery path. + clientDone <- ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { + conn, err := net.DialTimeout("tcp4", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), time.Second) + if err == nil { + _ = conn.Close() + } + return err + }) + }() + + if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + redirected, err := redirectListener.Accept() + if err != nil { + t.Fatalf("accepting redirected connection: %v", err) + } + defer redirected.Close() + + // The accepted socket is addressed to redirectListener, but the kernel's + // SO_ORIGINAL_DST record must still contain the destination chosen by the + // actor before nftables rewrote it. + got, err := TCPOriginalDestination(redirected) + if err != nil { + t.Fatalf("TCPOriginalDestination: %v", err) + } + want := net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)) + if got != want { + t.Errorf("original destination = %q, want %q", got, want) + } + if err := <-clientDone; err != nil { + t.Fatalf("dialing redirected connection: %v", err) + } +} + +func TestTCPOriginalDestinationIPv6(t *testing.T) { + roottest.Require(t, "CAP_NET_ADMIN + CAP_SYS_ADMIN for an actor-like network namespace and nftables REDIRECT rule") + + actorNS := newTestNetNS(t) + actorIP, hostIP := setupTestIPv6Veth(t, actorNS) + redirectListener := listenTCP6(t, hostIP) + defer redirectListener.Close() + targetListener := listenTCP6(t, hostIP) + defer targetListener.Close() + targetPort := targetListener.Addr().(*net.TCPAddr).Port + + table := &nftables.Table{Family: nftables.TableFamilyIPv6, Name: fmt.Sprintf("atunnel_original_dst_ipv6_test_%d", os.Getpid())} + installOriginalDstIPv6Redirect(t, table, actorIP, targetPort, redirectListener.Addr().(*net.TCPAddr).Port) + + clientDone := make(chan error, 1) + go func() { + clientDone <- ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { + conn, err := net.DialTimeout("tcp6", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), time.Second) + if err == nil { + _ = conn.Close() + } + return err + }) + }() + + if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + redirected, err := redirectListener.Accept() + if err != nil { + t.Fatalf("accepting redirected IPv6 connection: %v", err) + } + defer redirected.Close() + + // This assertion captures the IPv6 behavior required by #686. + got, err := TCPOriginalDestination(redirected) + if err != nil { + t.Fatalf("TCPOriginalDestination: %v", err) + } + want := net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)) + if got != want { + t.Errorf("original IPv6 destination = %q, want %q", got, want) + } + if err := <-clientDone; err != nil { + t.Fatalf("dialing redirected IPv6 connection: %v", err) + } +} + +func newTestNetNS(t *testing.T) netns.NsHandle { + t.Helper() + name := fmt.Sprintf("atunnel-original-dst-%d", os.Getpid()) + ns, err := ateomnet.CreateNetNSWithoutSwitching(name) + if err != nil { + if errors.Is(err, unix.EPERM) || strings.Contains(err.Error(), "operation not permitted") { + t.Skipf("needs CAP_SYS_ADMIN to create network namespace: %v", err) + } + t.Fatal(err) + } + t.Cleanup(func() { + _ = ns.Close() + if err := netns.DeleteNamed(name); err != nil { + t.Errorf("deleting test network namespace: %v", err) + } + }) + return ns +} + +func setupTestVeth(t *testing.T, actorNS netns.NsHandle) (actorIP, hostIP net.IP) { + t.Helper() + hostName := fmt.Sprintf("atod%d", os.Getpid()) + peerName := fmt.Sprintf("atop%d", os.Getpid()) + if err := netlink.LinkAdd(&netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: hostName}, PeerName: peerName}); err != nil { + if errors.Is(err, unix.EPERM) || strings.Contains(err.Error(), "operation not permitted") { + t.Skipf("needs CAP_NET_ADMIN to create veth: %v", err) + } + t.Fatal(err) + } + t.Cleanup(func() { + if link, err := netlink.LinkByName(hostName); err == nil { + if err := netlink.LinkDel(link); err != nil { + t.Errorf("deleting test veth: %v", err) + } + } + }) + hostLink, err := netlink.LinkByName(hostName) + if err != nil { + t.Fatal(err) + } + // Allocate one of the /30s in 198.18.0.0/16 from the PID so concurrent + // test processes do not try to use the same host-side address. + network := uint16(os.Getpid() % (1 << 14)) + thirdOctet := byte(network >> 6) + fourthOctet := byte(network&0x3f) << 2 + hostIP = net.IPv4(198, 18, thirdOctet, fourthOctet+1) + actorIP = net.IPv4(198, 18, thirdOctet, fourthOctet+2) + if err := netlink.AddrAdd(hostLink, &netlink.Addr{IPNet: &net.IPNet{IP: hostIP, Mask: net.CIDRMask(30, 32)}}); err != nil { + t.Fatal(err) + } + if err := netlink.LinkSetUp(hostLink); err != nil { + t.Fatal(err) + } + peer, err := netlink.LinkByName(peerName) + if err != nil { + t.Fatal(err) + } + if err := netlink.LinkSetNsFd(peer, int(actorNS)); err != nil { + t.Fatal(err) + } + // Complete the actor end of the point-to-point link inside its own netns. + if err := ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { + lo, err := netlink.LinkByName("lo") + if err != nil { + return err + } + if err := netlink.LinkSetUp(lo); err != nil { + return err + } + link, err := netlink.LinkByName(peerName) + if err != nil { + return err + } + if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &net.IPNet{IP: actorIP, Mask: net.CIDRMask(30, 32)}}); err != nil { + return err + } + return netlink.LinkSetUp(link) + }); err != nil { + t.Fatal(err) + } + return actorIP, hostIP +} + +func listenTCP(t *testing.T, hostIP net.IP) net.Listener { + t.Helper() + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: hostIP, Port: 0}) + if err != nil { + t.Fatal(err) + } + return listener +} + +func setupTestIPv6Veth(t *testing.T, actorNS netns.NsHandle) (actorIP, hostIP net.IP) { + t.Helper() + hostName := fmt.Sprintf("atod6%d", os.Getpid()) + peerName := fmt.Sprintf("atop6%d", os.Getpid()) + if err := netlink.LinkAdd(&netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: hostName}, PeerName: peerName}); err != nil { + if errors.Is(err, unix.EPERM) || strings.Contains(err.Error(), "operation not permitted") { + t.Skipf("needs CAP_NET_ADMIN to create veth: %v", err) + } + t.Fatal(err) + } + t.Cleanup(func() { + if link, err := netlink.LinkByName(hostName); err == nil { + if err := netlink.LinkDel(link); err != nil { + t.Errorf("deleting test IPv6 veth: %v", err) + } + } + }) + hostLink, err := netlink.LinkByName(hostName) + if err != nil { + t.Fatal(err) + } + prefix := uint16(os.Getpid()) + hostIP = net.ParseIP(fmt.Sprintf("fd00:198:18:%x::1", prefix)) + actorIP = net.ParseIP(fmt.Sprintf("fd00:198:18:%x::2", prefix)) + // This isolated veth has no competing IPv6 peers. Suppress DAD so the + // address can be bound immediately instead of remaining tentative while + // the test is trying to start its listener. + if err := netlink.AddrAdd(hostLink, &netlink.Addr{IPNet: &net.IPNet{IP: hostIP, Mask: net.CIDRMask(64, 128)}, Flags: unix.IFA_F_NODAD}); err != nil { + t.Fatal(err) + } + if err := netlink.LinkSetUp(hostLink); err != nil { + t.Fatal(err) + } + peer, err := netlink.LinkByName(peerName) + if err != nil { + t.Fatal(err) + } + if err := netlink.LinkSetNsFd(peer, int(actorNS)); err != nil { + t.Fatal(err) + } + if err := ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { + lo, err := netlink.LinkByName("lo") + if err != nil { + return err + } + if err := netlink.LinkSetUp(lo); err != nil { + return err + } + link, err := netlink.LinkByName(peerName) + if err != nil { + return err + } + if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &net.IPNet{IP: actorIP, Mask: net.CIDRMask(64, 128)}, Flags: unix.IFA_F_NODAD}); err != nil { + return err + } + return netlink.LinkSetUp(link) + }); err != nil { + t.Fatal(err) + } + return actorIP, hostIP +} + +func listenTCP6(t *testing.T, hostIP net.IP) net.Listener { + t.Helper() + listener, err := net.ListenTCP("tcp6", &net.TCPAddr{IP: hostIP, Port: 0}) + if err != nil { + t.Fatal(err) + } + return listener +} + +func installOriginalDstRedirect(t *testing.T, table *nftables.Table, actorIP net.IP, targetPort, redirectPort int) { + t.Helper() + c := &nftables.Conn{} + c.AddTable(table) + chain := c.AddChain(&nftables.Chain{ + Name: "prerouting", + Table: table, + Type: nftables.ChainTypeNAT, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityNATDest, + }) + c.AddRule(&nftables.Rule{ + Table: table, + Chain: chain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}}, + // Restrict the rule to this test's actor so the temporary table cannot + // affect unrelated local TCP traffic. + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: actorIP.To4()}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(uint16(targetPort))}, + &expr.Immediate{Register: 1, Data: binaryutil.BigEndian.PutUint16(uint16(redirectPort))}, + &expr.Redir{RegisterProtoMin: 1}, + }, + }) + if err := c.Flush(); err != nil { + if errors.Is(err, unix.EPERM) || strings.Contains(err.Error(), "operation not permitted") { + t.Skipf("needs CAP_NET_ADMIN to install nftables rule: %v", err) + } + t.Fatalf("installing nftables redirect: %v", err) + } + t.Cleanup(func() { + cleanup := &nftables.Conn{} + cleanup.DelTable(table) + if err := cleanup.Flush(); err != nil { + t.Errorf("removing nftables redirect: %v", err) + } + }) +} + +func installOriginalDstIPv6Redirect(t *testing.T, table *nftables.Table, actorIP net.IP, targetPort, redirectPort int) { + t.Helper() + c := &nftables.Conn{} + c.AddTable(table) + chain := c.AddChain(&nftables.Chain{ + Name: "prerouting", + Table: table, + Type: nftables.ChainTypeNAT, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityNATDest, + }) + c.AddRule(&nftables.Rule{ + Table: table, + Chain: chain, + Exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}}, + // An IPv6 source address begins eight bytes into the IPv6 header. + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 8, Len: 16}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: actorIP.To16()}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: binaryutil.BigEndian.PutUint16(uint16(targetPort))}, + &expr.Immediate{Register: 1, Data: binaryutil.BigEndian.PutUint16(uint16(redirectPort))}, + &expr.Redir{RegisterProtoMin: 1}, + }, + }) + if err := c.Flush(); err != nil { + if errors.Is(err, unix.EPERM) || strings.Contains(err.Error(), "operation not permitted") { + t.Skipf("needs CAP_NET_ADMIN to install IPv6 nftables rule: %v", err) + } + t.Fatalf("installing IPv6 nftables redirect: %v", err) + } + t.Cleanup(func() { + cleanup := &nftables.Conn{} + cleanup.DelTable(table) + if err := cleanup.Flush(); err != nil { + t.Errorf("removing IPv6 nftables redirect: %v", err) + } + }) +} From f3f3814b59c62c552d8e2b5fd2c27ed98d88b3ca Mon Sep 17 00:00:00 2001 From: lubingtan Date: Fri, 21 Aug 2026 11:46:21 +0800 Subject: [PATCH 05/22] atunnel: preserve IPv4 original destination errors --- internal/atunnel/original_dst_linux.go | 16 +++++-- internal/atunnel/original_dst_linux_test.go | 47 +++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/internal/atunnel/original_dst_linux.go b/internal/atunnel/original_dst_linux.go index 8b4309119d..b9d1717479 100644 --- a/internal/atunnel/original_dst_linux.go +++ b/internal/atunnel/original_dst_linux.go @@ -42,15 +42,23 @@ func TCPOriginalDestination(conn net.Conn) (string, error) { if err != nil { return "", fmt.Errorf("atunnel: acquiring TCP syscall connection: %w", err) } + // The IPv6 option is only meaningful on an AF_INET6 socket: on AF_INET the + // kernel returns EOPNOTSUPP, which would mask the real IPv4 error. A + // v4-mapped local address still means an IPv4 flow, so To4 is the test. + local, ok := tcpConn.LocalAddr().(*net.TCPAddr) + if !ok { + return "", fmt.Errorf("atunnel: original destination requires a TCP local address, got %T", tcpConn.LocalAddr()) + } + isIPv6 := local.IP.To4() == nil var sockoptErr error var destination string if err := rawConn.Control(func(fd uintptr) { destination, sockoptErr = originalIPv4Destination(fd) - // Linux returns ENOENT when the IPv4 original-destination option is - // queried on a redirected IPv6 connection. Only then try the IPv6 - // equivalent, so unrelated IPv4 failures retain their original error. - if errors.Is(sockoptErr, unix.ENOENT) { + // A pure-IPv6 socket leaves the inet addresses zeroed, so the IPv4 + // conntrack lookup always misses with ENOENT. That is the redirected + // IPv6 connection, and the only case worth retrying. + if isIPv6 && errors.Is(sockoptErr, unix.ENOENT) { destination, sockoptErr = originalIPv6Destination(fd) } }); err != nil { diff --git a/internal/atunnel/original_dst_linux_test.go b/internal/atunnel/original_dst_linux_test.go index 4f26d98055..ef647c331b 100644 --- a/internal/atunnel/original_dst_linux_test.go +++ b/internal/atunnel/original_dst_linux_test.go @@ -37,6 +37,53 @@ import ( "github.com/agent-substrate/substrate/internal/roottest" ) +// TestTCPOriginalDestinationPreservesErrno covers the failure path on an +// ordinary connection that no REDIRECT rule touched. The IPv4 lookup misses +// and reports ENOENT; that error must reach the caller. Retrying the IPv6 +// option on an AF_INET socket would replace it with EOPNOTSUPP, which says +// nothing about why the lookup failed. +// +// It runs in a fresh namespace because conntrack tracks loopback in any +// namespace that has nftables rules — including the one Docker runs in — and a +// tracked connection returns its real destination instead of missing. +func TestTCPOriginalDestinationPreservesErrno(t *testing.T) { + roottest.Require(t, "CAP_SYS_ADMIN for a network namespace with no conntrack hooks") + + ns := newTestNetNS(t) + if err := ateomnet.NetNSDo(context.Background(), ns, func(context.Context) error { + loopback, err := netlink.LinkByName("lo") + if err != nil { + return err + } + if err := netlink.LinkSetUp(loopback); err != nil { + return err + } + + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + return err + } + defer listener.Close() + client, err := net.DialTimeout("tcp4", listener.Addr().String(), time.Second) + if err != nil { + return err + } + defer client.Close() + server, err := listener.Accept() + if err != nil { + return err + } + defer server.Close() + + if _, err := TCPOriginalDestination(server); !errors.Is(err, unix.ENOENT) { + return fmt.Errorf("want the IPv4 lookup's ENOENT, got %w", err) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + func TestTCPOriginalDestination(t *testing.T) { roottest.Require(t, "CAP_NET_ADMIN + CAP_SYS_ADMIN for an actor-like network namespace and nftables REDIRECT rule") From ef064bb6b6f2c5a9ad7810aba2da1434da53f0c5 Mon Sep 17 00:00:00 2001 From: lubingtan Date: Fri, 21 Aug 2026 11:50:58 +0800 Subject: [PATCH 06/22] atunnel: stabilize original destination tests --- internal/atunnel/original_dst_linux_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/atunnel/original_dst_linux_test.go b/internal/atunnel/original_dst_linux_test.go index ef647c331b..3a9dbfca2a 100644 --- a/internal/atunnel/original_dst_linux_test.go +++ b/internal/atunnel/original_dst_linux_test.go @@ -114,7 +114,7 @@ func TestTCPOriginalDestination(t *testing.T) { // hostIP:targetPort. The worker's PREROUTING rule redirects it before // it reaches the host network stack's local delivery path. clientDone <- ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { - conn, err := net.DialTimeout("tcp4", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), time.Second) + conn, err := net.DialTimeout("tcp4", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), 10*time.Second) if err == nil { _ = conn.Close() } @@ -122,7 +122,7 @@ func TestTCPOriginalDestination(t *testing.T) { }) }() - if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(time.Second)); err != nil { + if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(10 * time.Second)); err != nil { t.Fatal(err) } redirected, err := redirectListener.Accept() @@ -164,7 +164,7 @@ func TestTCPOriginalDestinationIPv6(t *testing.T) { clientDone := make(chan error, 1) go func() { clientDone <- ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { - conn, err := net.DialTimeout("tcp6", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), time.Second) + conn, err := net.DialTimeout("tcp6", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), 10*time.Second) if err == nil { _ = conn.Close() } @@ -172,7 +172,7 @@ func TestTCPOriginalDestinationIPv6(t *testing.T) { }) }() - if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(time.Second)); err != nil { + if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(10 * time.Second)); err != nil { t.Fatal(err) } redirected, err := redirectListener.Accept() From a9bade052f1118fc5644043d75fe217de5a3c25c Mon Sep 17 00:00:00 2001 From: lubingtan Date: Mon, 24 Aug 2026 12:58:45 +0800 Subject: [PATCH 07/22] atunnel: select original destination by family --- internal/atunnel/original_dst_linux.go | 9 +-- internal/atunnel/original_dst_linux_test.go | 87 +++++++++++++-------- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/internal/atunnel/original_dst_linux.go b/internal/atunnel/original_dst_linux.go index b9d1717479..09e2e23140 100644 --- a/internal/atunnel/original_dst_linux.go +++ b/internal/atunnel/original_dst_linux.go @@ -18,7 +18,6 @@ package atunnel import ( "encoding/binary" - "errors" "fmt" "net" "strconv" @@ -54,13 +53,11 @@ func TCPOriginalDestination(conn net.Conn) (string, error) { var sockoptErr error var destination string if err := rawConn.Control(func(fd uintptr) { - destination, sockoptErr = originalIPv4Destination(fd) - // A pure-IPv6 socket leaves the inet addresses zeroed, so the IPv4 - // conntrack lookup always misses with ENOENT. That is the redirected - // IPv6 connection, and the only case worth retrying. - if isIPv6 && errors.Is(sockoptErr, unix.ENOENT) { + if isIPv6 { destination, sockoptErr = originalIPv6Destination(fd) + return } + destination, sockoptErr = originalIPv4Destination(fd) }); err != nil { return "", fmt.Errorf("atunnel: accessing TCP socket: %w", err) } diff --git a/internal/atunnel/original_dst_linux_test.go b/internal/atunnel/original_dst_linux_test.go index 3a9dbfca2a..c409ace4c4 100644 --- a/internal/atunnel/original_dst_linux_test.go +++ b/internal/atunnel/original_dst_linux_test.go @@ -38,10 +38,10 @@ import ( ) // TestTCPOriginalDestinationPreservesErrno covers the failure path on an -// ordinary connection that no REDIRECT rule touched. The IPv4 lookup misses -// and reports ENOENT; that error must reach the caller. Retrying the IPv6 -// option on an AF_INET socket would replace it with EOPNOTSUPP, which says -// nothing about why the lookup failed. +// ordinary connection that no REDIRECT rule touched. Each IPv4 lookup misses +// and reports ENOENT; that error must reach the caller. A dual-stack listener +// receives the IPv4 connection with a v4-mapped local address, so it must also +// select the IPv4 option. // // It runs in a fresh namespace because conntrack tracks loopback in any // namespace that has nftables rules — including the one Docker runs in — and a @@ -49,38 +49,57 @@ import ( func TestTCPOriginalDestinationPreservesErrno(t *testing.T) { roottest.Require(t, "CAP_SYS_ADMIN for a network namespace with no conntrack hooks") - ns := newTestNetNS(t) - if err := ateomnet.NetNSDo(context.Background(), ns, func(context.Context) error { - loopback, err := netlink.LinkByName("lo") - if err != nil { - return err - } - if err := netlink.LinkSetUp(loopback); err != nil { - return err - } + for _, test := range []struct { + name string + listenNetwork string + listenAddress string + }{ + {name: "IPv4", listenNetwork: "tcp4", listenAddress: "127.0.0.1:0"}, + // An unspecified "tcp" listener is a dual-stack AF_INET6 socket on + // Linux. A tcp4 client reaches it through a v4-mapped local address, + // so TCPOriginalDestination must select the IPv4 socket option via + // local.IP.To4(), rather than the listener's socket domain. + {name: "dual-stack v4-mapped", listenNetwork: "tcp", listenAddress: ":0"}, + } { + t.Run(test.name, func(t *testing.T) { + ns := newTestNetNS(t) + if err := ateomnet.NetNSDo(context.Background(), ns, func(context.Context) error { + loopback, err := netlink.LinkByName("lo") + if err != nil { + return err + } + if err := netlink.LinkSetUp(loopback); err != nil { + return err + } - listener, err := net.Listen("tcp4", "127.0.0.1:0") - if err != nil { - return err - } - defer listener.Close() - client, err := net.DialTimeout("tcp4", listener.Addr().String(), time.Second) - if err != nil { - return err - } - defer client.Close() - server, err := listener.Accept() - if err != nil { - return err - } - defer server.Close() + listener, err := net.Listen(test.listenNetwork, test.listenAddress) + if err != nil { + return err + } + defer listener.Close() + _, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + return err + } + client, err := net.DialTimeout("tcp4", net.JoinHostPort("127.0.0.1", port), time.Second) + if err != nil { + return err + } + defer client.Close() + server, err := listener.Accept() + if err != nil { + return err + } + defer server.Close() - if _, err := TCPOriginalDestination(server); !errors.Is(err, unix.ENOENT) { - return fmt.Errorf("want the IPv4 lookup's ENOENT, got %w", err) - } - return nil - }); err != nil { - t.Fatal(err) + if _, err := TCPOriginalDestination(server); !errors.Is(err, unix.ENOENT) { + return fmt.Errorf("want the IPv4 lookup's ENOENT, got %w", err) + } + return nil + }); err != nil { + t.Fatal(err) + } + }) } } From af37c2acb837f0720f99ce2fd68bf8c1ead4748e Mon Sep 17 00:00:00 2001 From: lubingtan Date: Mon, 24 Aug 2026 13:05:54 +0800 Subject: [PATCH 08/22] atunnel: identify original destination family in errors --- internal/atunnel/original_dst_linux.go | 6 +++++- internal/atunnel/original_dst_linux_test.go | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/atunnel/original_dst_linux.go b/internal/atunnel/original_dst_linux.go index 09e2e23140..5021269c75 100644 --- a/internal/atunnel/original_dst_linux.go +++ b/internal/atunnel/original_dst_linux.go @@ -62,7 +62,11 @@ func TCPOriginalDestination(conn net.Conn) (string, error) { return "", fmt.Errorf("atunnel: accessing TCP socket: %w", err) } if sockoptErr != nil { - return "", fmt.Errorf("atunnel: reading original TCP destination: %w", sockoptErr) + family := "IPv4" + if isIPv6 { + family = "IPv6" + } + return "", fmt.Errorf("atunnel: reading original %s TCP destination: %w", family, sockoptErr) } return destination, nil } diff --git a/internal/atunnel/original_dst_linux_test.go b/internal/atunnel/original_dst_linux_test.go index c409ace4c4..ccd97fc8df 100644 --- a/internal/atunnel/original_dst_linux_test.go +++ b/internal/atunnel/original_dst_linux_test.go @@ -92,8 +92,12 @@ func TestTCPOriginalDestinationPreservesErrno(t *testing.T) { } defer server.Close() - if _, err := TCPOriginalDestination(server); !errors.Is(err, unix.ENOENT) { - return fmt.Errorf("want the IPv4 lookup's ENOENT, got %w", err) + _, lookupErr := TCPOriginalDestination(server) + if !errors.Is(lookupErr, unix.ENOENT) { + return fmt.Errorf("want the IPv4 lookup's ENOENT, got %v", lookupErr) + } + if !strings.Contains(lookupErr.Error(), "original IPv4 TCP destination") { + return fmt.Errorf("want the error to name the IPv4 lookup, got %v", lookupErr) } return nil }); err != nil { From 5506f7a1a3d420e955ee69ddc9985d5d440fb2e5 Mon Sep 17 00:00:00 2001 From: lubingtan Date: Mon, 24 Aug 2026 13:35:13 +0800 Subject: [PATCH 09/22] atunnel: isolate original destination tests Run the worker side of the IPv4 and IPv6 redirect tests in a private network namespace. This keeps the veth, listeners, and nftables PREROUTING rule outside host INPUT policies such as ufw or firewalld default-deny rules. The private namespace owns the nftables tables, so its teardown also releases the tables, chains, and veths. Remove the explicit cleanup handlers, which otherwise run after returning to the host namespace and fail to find the private tables. --- internal/atunnel/original_dst_linux_test.go | 209 ++++++++++---------- 1 file changed, 108 insertions(+), 101 deletions(-) diff --git a/internal/atunnel/original_dst_linux_test.go b/internal/atunnel/original_dst_linux_test.go index ccd97fc8df..44d53a6cdf 100644 --- a/internal/atunnel/original_dst_linux_test.go +++ b/internal/atunnel/original_dst_linux_test.go @@ -23,6 +23,7 @@ import ( "net" "os" "strings" + "sync/atomic" "testing" "time" @@ -37,6 +38,8 @@ import ( "github.com/agent-substrate/substrate/internal/roottest" ) +var testNetNSSequence uint64 + // TestTCPOriginalDestinationPreservesErrno covers the failure path on an // ordinary connection that no REDIRECT rule touched. Each IPv4 lookup misses // and reports ENOENT; that error must reach the caller. A dual-stack listener @@ -115,112 +118,130 @@ func TestTCPOriginalDestination(t *testing.T) { // veth and is redirected in PREROUTING; that is the path on which Linux // preserves SO_ORIGINAL_DST for atunnel. actorNS := newTestNetNS(t) - actorIP, hostIP := setupTestVeth(t, actorNS) - // targetListener reserves the port the actor intends to reach. The NAT rule - // below must prevent connections from reaching it. - // - // redirectListener represents atunnel's local egress listener. It receives - // the redirected connection and is therefore the connection on which we ask - // Linux for the original destination. - redirectListener := listenTCP(t, hostIP) - defer redirectListener.Close() - targetListener := listenTCP(t, hostIP) - defer targetListener.Close() - targetPort := targetListener.Addr().(*net.TCPAddr).Port + withTestWorkerNS(t, func() { + actorIP, hostIP := setupTestVeth(t, actorNS) + // targetListener reserves the port the actor intends to reach. The NAT rule + // below must prevent connections from reaching it. + // + // redirectListener represents atunnel's local egress listener. It receives + // the redirected connection and is therefore the connection on which we ask + // Linux for the original destination. + redirectListener := listenTCP(t, hostIP) + defer redirectListener.Close() + targetListener := listenTCP(t, hostIP) + defer targetListener.Close() + targetPort := targetListener.Addr().(*net.TCPAddr).Port - table := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: fmt.Sprintf("atunnel_original_dst_test_%d", os.Getpid())} - installOriginalDstRedirect(t, table, actorIP, targetPort, redirectListener.Addr().(*net.TCPAddr).Port) + table := &nftables.Table{Family: nftables.TableFamilyIPv4, Name: fmt.Sprintf("atunnel_original_dst_test_%d", os.Getpid())} + installOriginalDstRedirect(t, table, actorIP, targetPort, redirectListener.Addr().(*net.TCPAddr).Port) - clientDone := make(chan error, 1) - go func() { - // From the actor's perspective this is an ordinary connection to - // hostIP:targetPort. The worker's PREROUTING rule redirects it before - // it reaches the host network stack's local delivery path. - clientDone <- ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { - conn, err := net.DialTimeout("tcp4", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), 10*time.Second) - if err == nil { - _ = conn.Close() - } - return err - }) - }() + clientDone := make(chan error, 1) + go func() { + // From the actor's perspective this is an ordinary connection to + // hostIP:targetPort. The worker's PREROUTING rule redirects it before + // it reaches the host network stack's local delivery path. + clientDone <- ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { + conn, err := net.DialTimeout("tcp4", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), 10*time.Second) + if err == nil { + _ = conn.Close() + } + return err + }) + }() - if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(10 * time.Second)); err != nil { - t.Fatal(err) - } - redirected, err := redirectListener.Accept() - if err != nil { - t.Fatalf("accepting redirected connection: %v", err) - } - defer redirected.Close() + if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + t.Fatal(err) + } + redirected, err := redirectListener.Accept() + if err != nil { + t.Fatalf("accepting redirected connection: %v", err) + } + defer redirected.Close() - // The accepted socket is addressed to redirectListener, but the kernel's - // SO_ORIGINAL_DST record must still contain the destination chosen by the - // actor before nftables rewrote it. - got, err := TCPOriginalDestination(redirected) - if err != nil { - t.Fatalf("TCPOriginalDestination: %v", err) - } - want := net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)) - if got != want { - t.Errorf("original destination = %q, want %q", got, want) - } - if err := <-clientDone; err != nil { - t.Fatalf("dialing redirected connection: %v", err) - } + // The accepted socket is addressed to redirectListener, but the kernel's + // SO_ORIGINAL_DST record must still contain the destination chosen by the + // actor before nftables rewrote it. + got, err := TCPOriginalDestination(redirected) + if err != nil { + t.Fatalf("TCPOriginalDestination: %v", err) + } + want := net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)) + if got != want { + t.Errorf("original destination = %q, want %q", got, want) + } + if err := <-clientDone; err != nil { + t.Fatalf("dialing redirected connection: %v", err) + } + }) } func TestTCPOriginalDestinationIPv6(t *testing.T) { roottest.Require(t, "CAP_NET_ADMIN + CAP_SYS_ADMIN for an actor-like network namespace and nftables REDIRECT rule") actorNS := newTestNetNS(t) - actorIP, hostIP := setupTestIPv6Veth(t, actorNS) - redirectListener := listenTCP6(t, hostIP) - defer redirectListener.Close() - targetListener := listenTCP6(t, hostIP) - defer targetListener.Close() - targetPort := targetListener.Addr().(*net.TCPAddr).Port + withTestWorkerNS(t, func() { + actorIP, hostIP := setupTestIPv6Veth(t, actorNS) + redirectListener := listenTCP6(t, hostIP) + defer redirectListener.Close() + targetListener := listenTCP6(t, hostIP) + defer targetListener.Close() + targetPort := targetListener.Addr().(*net.TCPAddr).Port - table := &nftables.Table{Family: nftables.TableFamilyIPv6, Name: fmt.Sprintf("atunnel_original_dst_ipv6_test_%d", os.Getpid())} - installOriginalDstIPv6Redirect(t, table, actorIP, targetPort, redirectListener.Addr().(*net.TCPAddr).Port) + table := &nftables.Table{Family: nftables.TableFamilyIPv6, Name: fmt.Sprintf("atunnel_original_dst_ipv6_test_%d", os.Getpid())} + installOriginalDstIPv6Redirect(t, table, actorIP, targetPort, redirectListener.Addr().(*net.TCPAddr).Port) - clientDone := make(chan error, 1) - go func() { - clientDone <- ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { - conn, err := net.DialTimeout("tcp6", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), 10*time.Second) - if err == nil { - _ = conn.Close() - } - return err - }) - }() + clientDone := make(chan error, 1) + go func() { + clientDone <- ateomnet.NetNSDo(context.Background(), actorNS, func(context.Context) error { + conn, err := net.DialTimeout("tcp6", net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)), 10*time.Second) + if err == nil { + _ = conn.Close() + } + return err + }) + }() - if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(10 * time.Second)); err != nil { - t.Fatal(err) - } - redirected, err := redirectListener.Accept() - if err != nil { - t.Fatalf("accepting redirected IPv6 connection: %v", err) - } - defer redirected.Close() + if err := redirectListener.(*net.TCPListener).SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + t.Fatal(err) + } + redirected, err := redirectListener.Accept() + if err != nil { + t.Fatalf("accepting redirected IPv6 connection: %v", err) + } + defer redirected.Close() - // This assertion captures the IPv6 behavior required by #686. - got, err := TCPOriginalDestination(redirected) - if err != nil { - t.Fatalf("TCPOriginalDestination: %v", err) - } - want := net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)) - if got != want { - t.Errorf("original IPv6 destination = %q, want %q", got, want) - } - if err := <-clientDone; err != nil { - t.Fatalf("dialing redirected IPv6 connection: %v", err) + // This assertion captures the IPv6 behavior required by #686. + got, err := TCPOriginalDestination(redirected) + if err != nil { + t.Fatalf("TCPOriginalDestination: %v", err) + } + want := net.JoinHostPort(hostIP.String(), fmt.Sprint(targetPort)) + if got != want { + t.Errorf("original IPv6 destination = %q, want %q", got, want) + } + if err := <-clientDone; err != nil { + t.Fatalf("dialing redirected IPv6 connection: %v", err) + } + }) +} + +// withTestWorkerNS runs the worker half of the test in a private namespace. +// The worker's listeners and nftables PREROUTING rule then cannot be affected +// by default-deny INPUT rules in the host namespace. +func withTestWorkerNS(t *testing.T, fn func()) { + t.Helper() + workerNS := newTestNetNS(t) + if err := ateomnet.NetNSDo(context.Background(), workerNS, func(context.Context) error { + fn() + return nil + }); err != nil { + t.Fatal(err) } } func newTestNetNS(t *testing.T) netns.NsHandle { t.Helper() - name := fmt.Sprintf("atunnel-original-dst-%d", os.Getpid()) + name := fmt.Sprintf("atunnel-original-dst-%d-%d", os.Getpid(), atomic.AddUint64(&testNetNSSequence, 1)) ns, err := ateomnet.CreateNetNSWithoutSwitching(name) if err != nil { if errors.Is(err, unix.EPERM) || strings.Contains(err.Error(), "operation not permitted") { @@ -414,13 +435,6 @@ func installOriginalDstRedirect(t *testing.T, table *nftables.Table, actorIP net } t.Fatalf("installing nftables redirect: %v", err) } - t.Cleanup(func() { - cleanup := &nftables.Conn{} - cleanup.DelTable(table) - if err := cleanup.Flush(); err != nil { - t.Errorf("removing nftables redirect: %v", err) - } - }) } func installOriginalDstIPv6Redirect(t *testing.T, table *nftables.Table, actorIP net.IP, targetPort, redirectPort int) { @@ -455,11 +469,4 @@ func installOriginalDstIPv6Redirect(t *testing.T, table *nftables.Table, actorIP } t.Fatalf("installing IPv6 nftables redirect: %v", err) } - t.Cleanup(func() { - cleanup := &nftables.Conn{} - cleanup.DelTable(table) - if err := cleanup.Flush(); err != nil { - t.Errorf("removing IPv6 nftables redirect: %v", err) - } - }) } From d1469ad2c7c43e2433045ae4fc747e53282a3976 Mon Sep 17 00:00:00 2001 From: lubingtan Date: Mon, 24 Aug 2026 13:40:23 +0800 Subject: [PATCH 10/22] atunnel: test original destination formatting --- .../atunnel/original_dst_format_linux_test.go | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 internal/atunnel/original_dst_format_linux_test.go diff --git a/internal/atunnel/original_dst_format_linux_test.go b/internal/atunnel/original_dst_format_linux_test.go new file mode 100644 index 0000000000..069f481df9 --- /dev/null +++ b/internal/atunnel/original_dst_format_linux_test.go @@ -0,0 +1,74 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atunnel + +import ( + "encoding/binary" + "net" + "testing" +) + +// networkOrderPort produces the raw field value the kernel leaves in +// RawSockaddrInet4.Port and RawSockaddrInet6.Port: a uint16 whose in-memory +// bytes are the port in network order, which on a little-endian host is not +// the port's numeric value. +func networkOrderPort(port uint16) uint16 { + return binary.NativeEndian.Uint16(binary.BigEndian.AppendUint16(nil, port)) +} + +func TestFormatOriginalDestination(t *testing.T) { + tests := []struct { + name string + ip []byte + port uint16 + want string + wantErr bool + }{ + {name: "IPv4", ip: []byte{198, 18, 0, 1}, port: 443, want: "198.18.0.1:443"}, + { + name: "IPv6 is bracketed", + ip: net.ParseIP("fd00:198:18::1").To16(), + port: 443, + want: "[fd00:198:18::1]:443", + }, + { + name: "v4-mapped IPv6 renders as IPv4", + ip: net.ParseIP("::ffff:198.18.0.1").To16(), + port: 8080, + want: "198.18.0.1:8080", + }, + {name: "high port is not sign-extended", ip: []byte{198, 18, 0, 1}, port: 65535, want: "198.18.0.1:65535"}, + {name: "port zero is rejected", ip: []byte{198, 18, 0, 1}, wantErr: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := formatOriginalDestination(test.ip, networkOrderPort(test.port)) + if test.wantErr { + if err == nil { + t.Fatalf("formatOriginalDestination() = %q, want an error", got) + } + return + } + if err != nil { + t.Fatalf("formatOriginalDestination() error = %v", err) + } + if got != test.want { + t.Errorf("formatOriginalDestination() = %q, want %q", got, test.want) + } + }) + } +} From cf0fbcd555573ad81d698f2a674a632434fd00e4 Mon Sep 17 00:00:00 2001 From: lubingtan Date: Mon, 24 Aug 2026 13:44:19 +0800 Subject: [PATCH 11/22] atunnel: document original destination buffer sizes --- internal/atunnel/original_dst_linux.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/atunnel/original_dst_linux.go b/internal/atunnel/original_dst_linux.go index 5021269c75..0c2d44bef9 100644 --- a/internal/atunnel/original_dst_linux.go +++ b/internal/atunnel/original_dst_linux.go @@ -88,6 +88,9 @@ func originalIPv6Destination(fd uintptr) (string, error) { } func getOriginalDestination(fd uintptr, level, option int, addr unsafe.Pointer, addrSize uintptr) unix.Errno { + // getsockopt treats size as both input and output. Callers provide the exact + // size of RawSockaddrInet4 (16 bytes) or RawSockaddrInet6 (28 bytes), which + // the kernel validates before writing the original destination into addr. size := uint32(addrSize) _, _, errno := unix.Syscall6( unix.SYS_GETSOCKOPT, From 404ec608a73836755a7f8d24d07eb79c9956d61f Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 19 Aug 2026 10:22:32 -0700 Subject: [PATCH 12/22] ateom: drop the family from the atunnel ingress listen defaults Both ateom herders defaulted the actor ingress flags to "0.0.0.0:443" and "0.0.0.0:444", which reads as IPv4-only. It never was: Go treats an unspecified address as a wildcard and binds it dual-stack, so the sockets already served both families. Spell the defaults ":443" and ":444" so the flag says what it does, and note why in a comment. Part of the dual-stack actor networking series; no behavior change. --- cmd/ateom-gvisor/main.go | 7 +++++-- cmd/ateom-microvm/main.go | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index bdc31803b5..1b117f1403 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -65,8 +65,11 @@ var ( podUID = pflag.String("pod-uid", "", "The UID of the current pod") // TODO(liorlieberman) have a sub package for all atunnel releated things like that - atunnelListenAddress = pflag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS") - atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", "0.0.0.0:444", "Address for actor ingress mTLS CONNECT") + // + // Every listen address here is an unspecified wildcard, which Go binds as a + // dual-stack socket. + atunnelListenAddress = pflag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS") + atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = pflag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = pflag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") atunnelClientIdentity = pflag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index fdfaf7b4f5..6a4b635e67 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -71,8 +71,10 @@ var ( otlpRelaySocket = flag.String("otlp-relay-socket", ateompath.AteletOTLPSocketPath(), "Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.") - atunnelListenAddress = flag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS") - atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", "0.0.0.0:444", "Address for actor ingress mTLS CONNECT") + // Every listen address here is an unspecified wildcard, which Go binds as a + // dual-stack socket. + atunnelListenAddress = flag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS") + atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") podIdentityTrustBundle = flag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet") atunnelClientIdentity = flag.String("atunnel-client-identity", "spiffe://cluster.local/ns/ate-system/sa/atenet-router", "SPIFFE identity allowed to call actor ingress HTTPS") From b3ef35f9b426ea785ee78af6dd83a7fcb0f2544f Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 19 Aug 2026 22:24:09 -0700 Subject: [PATCH 13/22] atecontroller: drop the family from the atunnel ingress args The worker Deployment passes --atunnel-listen-address=0.0.0.0:443 and --atunnel-connect-listen-address=0.0.0.0:444 explicitly, so the ateom flag defaults never reach a deployed worker and the command line a reader inspects still says IPv4. Spell these ":443" and ":444" to match. Part of the dual-stack actor networking series; no behavior change, as Go binds an unspecified IPv4 wildcard dual-stack either way. --- cmd/atecontroller/internal/controllers/workerpool_apply.go | 4 ++-- .../internal/controllers/workerpool_apply_test.go | 4 ++-- cmd/ateom-gvisor/main.go | 6 +++--- cmd/ateom-microvm/main.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 47ee183e89..c1a80389f1 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -93,8 +93,8 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin WithImage(wp.Spec.AteomImage). WithArgs( "--pod-uid=$(POD_UID)", - "--atunnel-listen-address=0.0.0.0:443", - "--atunnel-connect-listen-address=0.0.0.0:444", + "--atunnel-listen-address=:443", + "--atunnel-connect-listen-address=:444", "--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem", "--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem", "--atunnel-egress-listen-address=0.0.0.0:15001", diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index d1c7ead3b5..b69f490791 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -849,8 +849,8 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithImage(wp.Spec.AteomImage). WithArgs( "--pod-uid=$(POD_UID)", - "--atunnel-listen-address=0.0.0.0:443", - "--atunnel-connect-listen-address=0.0.0.0:444", + "--atunnel-listen-address=:443", + "--atunnel-connect-listen-address=:444", "--atunnel-credential-bundle="+atunnelIdentityMountPath+"/credential-bundle.pem", "--atunnel-trust-bundle="+atunnelIdentityMountPath+"/trust-bundle.pem", "--atunnel-egress-listen-address=0.0.0.0:15001", diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 1b117f1403..652b0475f7 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -65,9 +65,9 @@ var ( podUID = pflag.String("pod-uid", "", "The UID of the current pod") // TODO(liorlieberman) have a sub package for all atunnel releated things like that - // - // Every listen address here is an unspecified wildcard, which Go binds as a - // dual-stack socket. + + // The ingress addresses are unspecified wildcards, which Go binds dual-stack. + // Egress below stays IPv4 until the actor datapath carries v6. atunnelListenAddress = pflag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS") atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = pflag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 6a4b635e67..168e0f7bb0 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -71,8 +71,8 @@ var ( otlpRelaySocket = flag.String("otlp-relay-socket", ateompath.AteletOTLPSocketPath(), "Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.") - // Every listen address here is an unspecified wildcard, which Go binds as a - // dual-stack socket. + // The ingress addresses are unspecified wildcards, which Go binds dual-stack. + // Egress below stays IPv4 until the actor datapath carries v6. atunnelListenAddress = flag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS") atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") From fcb1a8908067bf7481b66ac63ec9a0af6064e6ac Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 20 Aug 2026 12:12:17 -0700 Subject: [PATCH 14/22] ateom: fix the comment about which listen addresses are dual-stack The comment singled out the ingress addresses as unspecified wildcards and said egress stays IPv4 below. 0.0.0.0 is unspecified too, so Go binds the egress listener dual-stack as well and the distinction does not exist -- which reintroduces exactly the misreading the rest of this change removes. Egress keeps the v4 spelling as a marker until the actor datapath carries v6; that is a note to a future reader, not a property of the socket. --- cmd/ateom-gvisor/main.go | 4 ++-- cmd/ateom-microvm/main.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 652b0475f7..ce54af96b0 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -66,8 +66,8 @@ var ( // TODO(liorlieberman) have a sub package for all atunnel releated things like that - // The ingress addresses are unspecified wildcards, which Go binds dual-stack. - // Egress below stays IPv4 until the actor datapath carries v6. + // Every listen address here is an unspecified wildcard, which Go binds as a + // dual-stack socket. atunnelListenAddress = pflag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS") atunnelConnectListenAddress = pflag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = pflag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 168e0f7bb0..6a4b635e67 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -71,8 +71,8 @@ var ( otlpRelaySocket = flag.String("otlp-relay-socket", ateompath.AteletOTLPSocketPath(), "Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.") - // The ingress addresses are unspecified wildcards, which Go binds dual-stack. - // Egress below stays IPv4 until the actor datapath carries v6. + // Every listen address here is an unspecified wildcard, which Go binds as a + // dual-stack socket. atunnelListenAddress = flag.String("atunnel-listen-address", ":443", "Address for actor ingress HTTPS") atunnelConnectListenAddress = flag.String("atunnel-connect-listen-address", ":444", "Address for actor ingress mTLS CONNECT") workerCredentialBundle = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS") From 606825e6a336cd25e52d8059f2652ee7a3acfcc2 Mon Sep 17 00:00:00 2001 From: Suraj Kumar Date: Sat, 15 Aug 2026 12:57:09 +0000 Subject: [PATCH 15/22] ateomnet: enable IPv6 forwarding in worker pod netns EnableIPv4Forwarding now also writes /proc/sys/net/ipv6/conf/all/forwarding so actor IPv6 traffic (including DNS queries) is routed between the actor veth and pod eth0 instead of being dropped by ip6_forward() on dual-stack / IPv6-only clusters. Factor the sysctl write into writeSysctlIfUnset preserving the original read-only remount/restore behavior, and add unit coverage for its fast paths. Fixes: agent-substrate/substrate#945 --- internal/ateomnet/net.go | 28 ++++++++- internal/ateomnet/write_sysctl_test.go | 84 ++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 internal/ateomnet/write_sysctl_test.go diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 91203a8e04..2d3daa4737 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -193,6 +193,9 @@ func PodIPv4() (net.IP, error) { } // EnableIPv4Forwarding enables IPv4 forwarding in the current network namespace. +// It also enables IPv6 forwarding so actor IPv6 traffic (including DNS queries +// on IPv6-capable clusters) is routed between the veth and eth0 instead of +// being dropped by ip6_forward(). func EnableIPv4Forwarding() error { // Forwarding is required because actor packets now enter the worker pod via // the host-side veth and then leave through the pod's eth0. Without this, the @@ -203,20 +206,41 @@ func EnableIPv4Forwarding() error { // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag // is not locked: clear it, write the sysctl, restore ro. const path = "/proc/sys/net/ipv4/ip_forward" + if err := writeSysctlIfUnset(path); err != nil { + return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err) + } + // IPv6 forwarding: actor packets that arrive on the veth and leave via eth0 + // are IPv6 on dual-stack / IPv6-only clusters. Without + // net.ipv6.conf.all.forwarding the kernel drops every IPv6 packet in + // ip6_forward(), including the actor's DNS queries. conf.all.forwarding=1 + // also implies the per-interface default, so a single write covers the veth + // and eth0. + const v6path = "/proc/sys/net/ipv6/conf/all/forwarding" + if err := writeSysctlIfUnset(v6path); err != nil { + return fmt.Errorf("while enabling IPv6 forwarding in worker pod netns: %w", err) + } + return nil +} + +// writeSysctlIfUnset writes "1\n" to a sysctl path unless it already reads "1". +func writeSysctlIfUnset(path string) error { if b, err := os.ReadFile(path); err == nil && len(b) > 0 && b[0] == '1' { return nil } if err := os.WriteFile(path, []byte("1\n"), 0o644); err == nil { return nil } + // Without privileged, the container runtime bind-mounts /proc/sys read-only. + // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag + // is not locked: clear it, write the sysctl, restore ro. if err := unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT, ""); err != nil { - return fmt.Errorf("while remounting /proc/sys read-write to enable IPv4 forwarding: %w", err) + return fmt.Errorf("while remounting /proc/sys read-write to enable forwarding: %w", err) } defer func() { _ = unix.Mount("none", "/proc/sys", "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, "") }() if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil { - return fmt.Errorf("while enabling IPv4 forwarding in worker pod netns: %w", err) + return fmt.Errorf("while writing %s: %w", path, err) } return nil } diff --git a/internal/ateomnet/write_sysctl_test.go b/internal/ateomnet/write_sysctl_test.go new file mode 100644 index 0000000000..b4cb1db169 --- /dev/null +++ b/internal/ateomnet/write_sysctl_test.go @@ -0,0 +1,84 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "os" + "path/filepath" + "testing" +) + +// TestWriteSysctlIfUnset verifies writeSysctlIfUnset's fast paths against a +// temp file standing in for a /proc/sys node: it must not rewrite a value +// that already reads "1", and it must write "1\n" when the value is missing +// or unset. The privileged bind-remount path is covered by the netns +// integration tests (withTestNetNS), which require root. +func TestWriteSysctlIfUnset(t *testing.T) { + dir := t.TempDir() + + t.Run("already_set", func(t *testing.T) { + p := filepath.Join(dir, "already") + // Sentinel content: if writeSysctlIfUnset rewrote the file, the value + // would change to "1\n" and this assertion would fail. Keeping the + // file larger than the helper's output makes a silent rewrite + // detectable. + if err := os.WriteFile(p, []byte("1 other-content\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset: %v", err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if string(b) != "1 other-content\n" { + t.Fatalf("already-set file was rewritten: %q", b) + } + }) + + t.Run("unset_written", func(t *testing.T) { + p := filepath.Join(dir, "unset") + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset: %v", err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if len(b) < 1 || b[0] != '1' { + t.Fatalf("expected '1' written, got %q", b) + } + }) + + t.Run("zero_is_rewritten", func(t *testing.T) { + p := filepath.Join(dir, "zero") + if err := os.WriteFile(p, []byte("0\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset: %v", err) + } + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + if len(b) < 1 || b[0] != '1' { + t.Fatalf("expected '1' written, got %q", b) + } + }) +} From 1cd4703cca11997d96ea8e19a2891dad489fb438 Mon Sep 17 00:00:00 2001 From: Suraj Kumar Date: Sun, 16 Aug 2026 19:12:06 +0000 Subject: [PATCH 16/22] ateomnet: return nil when sysctl path missing in writeSysctlIfUnset IPv6 sysctls are absent on kernels with IPv6 disabled (e.g. some containers set net.ipv6.conf.* only when IPv6 is enabled). Treat a missing path as 'nothing to enable' instead of forcing a remount and failing, matching the documented behavior. --- internal/ateomnet/net.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 2d3daa4737..a2a9501787 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -223,6 +223,8 @@ func EnableIPv4Forwarding() error { } // writeSysctlIfUnset writes "1\n" to a sysctl path unless it already reads "1". +// If the path does not exist (e.g. IPv6 sysctls on a kernel with IPv6 disabled), +// it returns nil — IPv6 forwarding is simply unavailable, not an error. func writeSysctlIfUnset(path string) error { if b, err := os.ReadFile(path); err == nil && len(b) > 0 && b[0] == '1' { return nil @@ -230,6 +232,10 @@ func writeSysctlIfUnset(path string) error { if err := os.WriteFile(path, []byte("1\n"), 0o644); err == nil { return nil } + if _, err := os.Stat(path); os.IsNotExist(err) { + // Path absent (e.g. IPv6 disabled in kernel): nothing to enable. + return nil + } // Without privileged, the container runtime bind-mounts /proc/sys read-only. // The worker holds CAP_SYS_ADMIN and uses no user namespace, so the ro flag // is not locked: clear it, write the sysctl, restore ro. From b357f52c98ca47c3d7c5816ab67c9115ccf542e5 Mon Sep 17 00:00:00 2001 From: Suraj Kumar Date: Fri, 21 Aug 2026 13:04:19 +0000 Subject: [PATCH 17/22] ateomnet: rename EnableIPv4Forwarding to EnableForwarding The helper has enabled both address families since IPv6 forwarding was added; the name now says so. The single call site in SetupActorNetwork is updated along with the doc comment. --- internal/ateomnet/net.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index a2a9501787..1f0fbc92a8 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -192,11 +192,11 @@ func PodIPv4() (net.IP, error) { return nil, fmt.Errorf("pod eth0 has no IPv4 address") } -// EnableIPv4Forwarding enables IPv4 forwarding in the current network namespace. -// It also enables IPv6 forwarding so actor IPv6 traffic (including DNS queries -// on IPv6-capable clusters) is routed between the veth and eth0 instead of -// being dropped by ip6_forward(). -func EnableIPv4Forwarding() error { +// EnableForwarding enables IPv4 and IPv6 forwarding in the current network +// namespace, so actor traffic (including DNS queries on IPv6-capable clusters) +// is routed between the veth and eth0 instead of being dropped by ip_forward() +// or ip6_forward(). +func EnableForwarding() error { // Forwarding is required because actor packets now enter the worker pod via // the host-side veth and then leave through the pod's eth0. Without this, the // kernel would not route traffic between those interfaces even though both @@ -595,7 +595,7 @@ func SetupActorNetwork(ctx context.Context, cfg NetworkConfig) (retErr error) { return fmt.Errorf("while configuring actor veth in interior netns: %w", err) } - if err := EnableIPv4Forwarding(); err != nil { + if err := EnableForwarding(); err != nil { return err } if err := InstallActorNftablesRules(cfg.EgressRedirectPort); err != nil { From a67785c6076407fe75acfb459c965317431f7f4a Mon Sep 17 00:00:00 2001 From: Suraj Kumar Date: Fri, 21 Aug 2026 13:04:19 +0000 Subject: [PATCH 18/22] ateomnet: cover writeSysctlIfUnset's missing-path branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The os.Stat/IsNotExist fallback was never executed by the unit tests: every existing subtest's temp path could be created, so each returned at the os.WriteFile fast path. Point the new subtest at a node under a directory that does not exist — what procfs always does in production — and assert the file stays absent. --- internal/ateomnet/write_sysctl_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/ateomnet/write_sysctl_test.go b/internal/ateomnet/write_sysctl_test.go index b4cb1db169..d687932d8c 100644 --- a/internal/ateomnet/write_sysctl_test.go +++ b/internal/ateomnet/write_sysctl_test.go @@ -65,6 +65,21 @@ func TestWriteSysctlIfUnset(t *testing.T) { } }) + t.Run("missing_path_is_noop", func(t *testing.T) { + // A node under a directory that does not exist stands in for + // /proc/sys/net/ipv6/... on a kernel with IPv6 disabled. The other + // subtests' paths can be created, so they return at the os.WriteFile + // fast path; this is the only one that reaches the os.Stat branch, + // which is what procfs always does in production. + p := filepath.Join(dir, "no-such-dir", "forwarding") + if err := writeSysctlIfUnset(p); err != nil { + t.Fatalf("writeSysctlIfUnset on a missing path: %v", err) + } + if _, err := os.Stat(p); !os.IsNotExist(err) { + t.Fatalf("expected %s to stay absent, stat err = %v", p, err) + } + }) + t.Run("zero_is_rewritten", func(t *testing.T) { p := filepath.Join(dir, "zero") if err := os.WriteFile(p, []byte("0\n"), 0o644); err != nil { From c501a6ee622fb4269972b82a51a7356909eded1b Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 20 Aug 2026 11:50:21 -0700 Subject: [PATCH 19/22] atenet/egress: resolve upstream names on both address families The egress gateway's dynamic forward proxy pinned dns_lookup_family to V4_ONLY at all seven sites, so Envoy only ever resolved the A record. On an IPv6-only cluster there is no A record to find, and the gateway cannot resolve upstream names at all. All seven now use ALL, which returns both families and lets Envoy Happy-Eyeballs between them. AUTO will not do: it returns AAAA whenever the name has one, leaving no A address to retry when that AAAA is unroutable. A test walks the shipped manifests and requires ALL on every dns_cache_config, so a new egress variant cannot reintroduce the pin -- the seventh site arrived with the sdsmint MITM leg while this change was in review, and the test caught it. --- .../router/extproc/egress_manifest_test.go | 136 ++++++++++++++++++ .../atenet-egress-with-sdsmint.yaml | 13 +- manifests/ate-install/atenet-egress.yaml | 7 +- 3 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 cmd/atenet/internal/router/extproc/egress_manifest_test.go diff --git a/cmd/atenet/internal/router/extproc/egress_manifest_test.go b/cmd/atenet/internal/router/extproc/egress_manifest_test.go new file mode 100644 index 0000000000..02ea7addb5 --- /dev/null +++ b/cmd/atenet/internal/router/extproc/egress_manifest_test.go @@ -0,0 +1,136 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extproc + +import ( + "bufio" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/yaml" +) + +// The install tree, not a fixture, so this guards the Envoy config that ships. +const manifestsDir = "../../../../../manifests" + +// What the install ships today. Falling below it means the walk stopped +// matching, not that the config got better. +const minDNSCacheConfigs = 7 + +// TestEgressDNSLookupFamily requires ALL on every dynamic forward proxy DNS +// cache the install ships; atenet-egress.yaml says why ALL. +func TestEgressDNSLookupFamily(t *testing.T) { + found := 0 + for _, path := range manifestPaths(t) { + caches := dnsCacheConfigs(t, path) + if len(caches) == 0 { + continue + } + found += len(caches) + t.Run(filepath.Base(path), func(t *testing.T) { + for _, cache := range caches { + if got := cache["dns_lookup_family"]; got != "ALL" { + t.Errorf("dns_cache_config %v: dns_lookup_family = %v, want ALL", cache["name"], got) + } + } + }) + } + if found < minDNSCacheConfigs { + t.Errorf("found %d dns_cache_config blocks under %s, want at least %d", found, manifestsDir, minDNSCacheConfigs) + } +} + +// manifestPaths covers the whole install tree, so a new egress variant is +// checked the day it is added. +func manifestPaths(t *testing.T) []string { + t.Helper() + var paths []string + err := filepath.WalkDir(manifestsDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && (strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) { + paths = append(paths, path) + } + return nil + }) + if err != nil { + t.Fatalf("walking %s: %v", manifestsDir, err) + } + return paths +} + +func dnsCacheConfigs(t *testing.T, path string) []map[string]any { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatalf("opening %s: %v", path, err) + } + defer f.Close() + + var caches []map[string]any + reader := k8syaml.NewYAMLReader(bufio.NewReader(f)) + for { + doc, err := reader.Read() + if errors.Is(err, io.EOF) { + return caches + } + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + var object struct { + Kind string `json:"kind"` + Data map[string]string `json:"data"` + } + if err := yaml.Unmarshal(doc, &object); err != nil { + t.Fatalf("parsing a document of %s: %v", path, err) + } + if object.Kind != "ConfigMap" { + continue + } + for _, value := range object.Data { + var config any + if err := yaml.Unmarshal([]byte(value), &config); err != nil { + // Not every ConfigMap value is YAML. + continue + } + caches = append(caches, collectDNSCacheConfigs(config)...) + } + } +} + +func collectDNSCacheConfigs(node any) []map[string]any { + var caches []map[string]any + switch node := node.(type) { + case map[string]any: + for key, value := range node { + if cache, ok := value.(map[string]any); ok && key == "dns_cache_config" { + caches = append(caches, cache) + } + caches = append(caches, collectDNSCacheConfigs(value)...) + } + case []any: + for _, value := range node { + caches = append(caches, collectDNSCacheConfigs(value)...) + } + } + return caches +} diff --git a/manifests/ate-install/atenet-egress-with-sdsmint.yaml b/manifests/ate-install/atenet-egress-with-sdsmint.yaml index 161b9d4c71..3d7b85b509 100644 --- a/manifests/ate-install/atenet-egress-with-sdsmint.yaml +++ b/manifests/ate-install/atenet-egress-with-sdsmint.yaml @@ -359,7 +359,10 @@ data: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + # V4_ONLY resolves nothing on an IPv6-only cluster. ALL + # returns both families and enables Happy Eyeballs, so + # neither family is stranded. + dns_lookup_family: ALL - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -457,7 +460,7 @@ data: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: ALL - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -597,7 +600,7 @@ data: "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: ALL # The MITM must not weaken upstream authentication. Envoy decrypted the # actor's TLS with a leaf of its own; it still sends the real SNI here # and still validates the real origin's certificate against the public @@ -632,7 +635,7 @@ data: "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: ALL typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions @@ -668,7 +671,7 @@ data: "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: ALL # Envoy refuses to build a dynamic forward proxy cluster without # auto_sni and auto_san_validation unless this is set, because for # the usual TLS case resolving the host from a header and then not diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 24c38cd58c..463303a612 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -148,7 +148,10 @@ data: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + # V4_ONLY resolves nothing on an IPv6-only cluster. ALL + # returns both families and enables Happy Eyeballs, so + # neither family is stranded. + dns_lookup_family: ALL - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -185,7 +188,7 @@ data: "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig dns_cache_config: name: egress_dns_cache - dns_lookup_family: V4_ONLY + dns_lookup_family: ALL --- apiVersion: apps/v1 kind: Deployment From aeb1b7cfd8b5417890645d5a90711c341565d11d Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Fri, 14 Aug 2026 11:03:44 -0700 Subject: [PATCH 20/22] ci: add an IPv6-only kind e2e job No CI job exercises IPv6-only, so nothing catches a change that breaks it, and the IPv6 work is a set of changes that do little apart. This builds a single-stack IPv6 kind cluster, installs the full system and runs the networking suite, where every test that can tell one address family from another lives. It asserts the cluster really is single-stack, and that cluster DNS is the shape an IPv6-only install needs, before installing anything -- so a green run cannot be vacuous. It carries no copy of what it exercises: dispatch it with a list of pull requests and it merges them onto main for the length of the run. It stays out of the e2e-test gate, so it never blocks a pull request. --- .github/workflows/e2e-ipv6.yaml | 370 ++++++++++++++++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 .github/workflows/e2e-ipv6.yaml diff --git a/.github/workflows/e2e-ipv6.yaml b/.github/workflows/e2e-ipv6.yaml new file mode 100644 index 0000000000..727957d3c0 --- /dev/null +++ b/.github/workflows/e2e-ipv6.yaml @@ -0,0 +1,370 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: e2e-ipv6 +# Separate from pr-workflow.yaml so this can be gated independently -- and so a +# 40-minute IPv6 run never delays that workflow's merge-gating jobs. +# +# Only the networking suite runs. Every test that can tell one address family +# from another lives in it, and the demo suite is a large share of the wall +# clock for coverage that is not family-specific. The demos still deploy, +# because networking builds its actors from their templates. +# +# Two ways in. A pull request that edits this file runs it: the paths filter is +# evaluated before the job, so no unrelated request pays for a v6 cluster, and +# the lane can demonstrate itself without a label someone has to create first. +# Dispatch it with a list of pull requests and it merges them onto main in that +# order and runs the result: the IPv6 work is a set of changes that do nothing +# apart, so a per-request run cannot show the feature working, only that it did +# no harm. The stack exists for the length of the run and is never pushed, +# which is what lets this branch carry the workflow file alone instead of a +# copy of every change it wants to exercise. +# +# Stacking is dispatch-only, so the pull_request path runs main plus this file +# and nothing else. That exercises the lane, not the IPv6 work, and it stays +# red until the changes this gates are on main. +# +# Either way this job stays out of the `e2e-test` gate, so it never blocks a PR. +on: + pull_request: + paths: ['.github/workflows/e2e-ipv6.yaml'] + workflow_dispatch: + inputs: + prs: + description: 'Pull requests to merge onto main, in order, e.g. "958 938 1083". Blank runs main alone.' + required: false + type: string +permissions: + contents: read +jobs: + e2e-test-ipv6: + runs-on: ubuntu-latest + # Nothing else in this workflow sets a timeout, so jobs inherit GitHub's + # 6-hour default. A broken IPv6 cluster does not crash, it misses + # 10-minute ActorTemplate deadlines, so an uncapped job burns hours. + timeout-minutes: 40 + env: + # Non-default name so these steps can be replayed locally without + # touching an existing cluster. install-ate-kind.sh does not derive + # KUBECTL_CONTEXT from the cluster name the way run-e2e-kind.sh does, + # so both have to be set here. + KIND_CLUSTER_NAME: ate-ipv6 + KUBECTL_CONTEXT: kind-ate-ipv6 + # 8.8.8.8 reached through the well-known NAT64 prefix. CoreDNS is a + # v6-only pod on a runner with no IPv6 egress of its own, so this is the + # only shape of upstream resolver it can reach. See "Set up NAT64". + IPV6_DNS_UPSTREAM: 64:ff9b::808:808 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + # The merges below need refs this clone would not otherwise have. + fetch-depth: 0 + - name: Stack the pull requests under test + # Merge, not cherry-pick: a request that has been squash-merged since it + # was opened re-applies as a revert of what is already on main, and only + # a merge notices there is nothing to do. Order is the order given, so a + # conflict names the request that has to rebase rather than the stack. + # + # This builds and runs the requests' code. The token is read-only and no + # secrets reach it; a run happens only when someone asks for one. + if: github.event_name == 'workflow_dispatch' && inputs.prs != '' + env: + GH_TOKEN: ${{ github.token }} + PRS: ${{ inputs.prs }} + # The numbers in `prs` are always upstream pull requests, so they are + # resolved against upstream by name rather than against whichever repo + # the run happens to be in. Reading them needs no token. + UPSTREAM: https://github.com/agent-substrate/substrate + run: | + git config user.name 'e2e-ipv6' + git config user.email 'e2e-ipv6@invalid' + { + echo "### Stack under test" + echo "" + echo "Merged onto \`main\` at \`$(git rev-parse --short HEAD)\`, in order:" + echo "" + echo "| pull request | head | title |" + echo "|---|---|---|" + } >> "$GITHUB_STEP_SUMMARY" + for pr in ${PRS//,/ }; do + case "${pr}" in + ''|*[!0-9]*) echo "::error::'${pr}' is not a pull request number"; exit 1 ;; + esac + git fetch -q "${UPSTREAM}" "refs/pull/${pr}/head:pr-${pr}" \ + || { echo "::error::no such pull request: #${pr}"; exit 1; } + if ! git merge --no-edit -q "pr-${pr}"; then + echo "::error::#${pr} does not merge onto the stack; it needs a rebase" + git merge --abort || true + exit 1 + fi + title=$(gh pr view "${pr}" --repo "${UPSTREAM}" --json title -q .title 2>/dev/null || echo '?') + echo "| #${pr} | \`$(git rev-parse --short "pr-${pr}")\` | ${title} |" >> "$GITHUB_STEP_SUMMARY" + echo "merged #${pr}: ${title}" + done + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: 'go.mod' + - name: Free disk space + # kind node image + control-plane images + snapshots are tight on the + # ~14GB runner disk even without the micro-VM assets. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - name: Enable IPv6 in the Docker daemon + # ubuntu-latest ships dockerd with IPv6 off, so kind would create its + # network v4-only and create-kind-cluster.sh would reject the cluster. + # Merge the two keys into whatever daemon.json the runner image ships + # rather than replacing the file. + run: | + sudo mkdir -p /etc/docker + [ -s /etc/docker/daemon.json ] || echo '{}' | sudo tee /etc/docker/daemon.json >/dev/null + sudo cat /etc/docker/daemon.json \ + | jq '. + {"ipv6": true, "ip6tables": true}' \ + | sudo tee /etc/docker/daemon.json.new >/dev/null + sudo mv /etc/docker/daemon.json.new /etc/docker/daemon.json + sudo systemctl restart docker + # Assert, don't print: if the merge or the restart silently did not take, + # kind builds a v4-only network and the failure surfaces much later as a + # cluster that is not the one this job exists to test. + docker network inspect bridge --format '{{.EnableIPv6}}' | grep -qx true || { + echo "::error::dockerd did not come back with IPv6 enabled" + sudo cat /etc/docker/daemon.json; exit 1; + } + echo "bridge EnableIPv6=true" + - name: Set up NAT64 on the runner + # ubuntu-latest has no IPv6 egress whatsoever -- measured, not assumed: + # every curl -6 fails in ~2ms. A v6-only cluster still has to reach real + # v4 destinations (atelet fetches the gVisor tarball from GCS, + # TestActorEgress fetches example.com), so the runner translates for it. + # Ordered after the dockerd restart, which rebuilds the iptables chains + # these rules live in. The script gates on a translated HTTPS fetch, so a + # broken translator fails here rather than as a rollout timeout ten + # minutes later. + run: | + # Named check rather than letting bash report a missing file: this is + # the first step that touches the IPv6-only kind support, so on a tree + # without it this is where the run stops, and it should say why. + [ -x hack/setup-nat64.sh ] || { + echo "::error::hack/setup-nat64.sh is missing -- the IPv6-only kind support this job gates is not in this tree" + exit 1 + } + hack/setup-nat64.sh + - name: Create cluster + # DNS64 is opt-in in the script and this is the case it exists for: the + # names this job needs have real AAAA records pointing at addresses the + # runner cannot reach, so cluster DNS has to route them through the + # translator set up above. Everything the Corefile ends up containing is + # the script's doing -- the steps below only read it back. + env: + IP_FAMILY: ipv6 + IPV6_DNS64_PREFIX: '64:ff9b::/96' + run: hack/create-kind-cluster.sh + - name: Wait for the node to be ready + # podCIDRs is assigned by the controller-manager after the node registers, + # so the assertion below can read an empty field on a cluster that is + # perfectly fine and report "cannot confirm IP family". Poll for the field + # this job actually reads, then wait on Ready for everything after it. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + for _ in $(seq 60); do + [ -n "$(k get nodes -o jsonpath='{.items[*].spec.podCIDRs[*]}' 2>/dev/null)" ] && break + sleep 2 + done + k wait --for=condition=Ready nodes --all --timeout=180s + - name: Assert the cluster is single-stack IPv6 + # This job is worthless if the cluster is not actually v6-only, and a + # green run leaves no evidence either way -- the diagnostics dump below + # only runs on failure. A kind default change or an IP_FAMILY regression + # would otherwise turn this into a second IPv4 run that reports success. + # Checked here rather than later so it fails as itself. + # + # PreferDualStack Services resolving to a single clusterIP is the + # positive signal: on a dual-stack cluster they would get two. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + pod_cidrs=$(k get nodes -o jsonpath='{.items[*].spec.podCIDRs[*]}') + svc_ips=$(k -n default get svc kubernetes -o jsonpath='{.spec.clusterIPs[*]}') + node_ips=$(k get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}') + for pair in "podCIDRs=${pod_cidrs}" "kubernetes.clusterIPs=${svc_ips}" "node.InternalIP=${node_ips}"; do + case "${pair#*=}" in + *.*) echo "::error::not single-stack IPv6 -- ${pair}"; exit 1 ;; + "") echo "::error::empty, cannot confirm IP family -- ${pair}"; exit 1 ;; + esac + echo " ${pair}" + done + echo "single-stack IPv6 confirmed" + - name: Assert create-kind-cluster.sh built the Corefile this job needs + # The script owns the whole Corefile, so this reads it back instead of + # rewriting it. That is the point: a green run is then evidence for what + # the script produced, where a job that patched the Corefile itself would + # paper over the absence of the very change it is here to gate. Each of + # these is something the install would otherwise fail on ten minutes + # later and far less legibly. + run: | + Corefile=$(kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}') + echo "${Corefile}" + check() { + echo "${Corefile}" | grep -q "$1" || { echo "::error::$2"; exit 1; } + } + if echo "${Corefile}" | grep -q 'forward .* /etc/resolv.conf'; then + echo "::error::CoreDNS still forwards to /etc/resolv.conf -- the IPv6-only DNS fix is not in this tree" + exit 1 + fi + check "forward .* ${IPV6_DNS_UPSTREAM}" \ + "CoreDNS does not forward to ${IPV6_DNS_UPSTREAM}" + check '^kind-registry:53' \ + "no kind-registry server block -- atelet cannot pull from its own netns" + check '^cluster.local:53' \ + "the cluster zones have no server block of their own -- dns64 would take out service discovery" + check 'translate_all' \ + "no dns64 translate_all -- external names resolve to addresses this runner cannot reach" + echo "Corefile is the IPv6-only shape this job needs" + - name: Verify cluster DNS answers both internal and external names + # The install is the next step and it takes ten minutes to fail. A DNS + # regression is the failure this Corefile is most likely to cause, so + # assert all three cases here where the message is unambiguous. + # Reads the log rather than attaching. `kubectl run --attach` has to + # connect before the container writes, and on a cluster still settling it + # loses that race far more often than it wins it -- a probe that fails on + # a healthy cluster is worse than no probe. Create, wait, read the log, + # and retry the whole thing. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + probe() { + k delete pod dnscheck --ignore-not-found --wait --timeout=60s >/dev/null 2>&1 + k run dnscheck --restart=Never --image=busybox:1.36 --command -- \ + sh -c ' + nslookup kubernetes.default.svc.cluster.local >/dev/null 2>&1 \ + || { echo "FAIL: an in-cluster Service does not resolve"; exit 1; } + nslookup storage.googleapis.com 2>/dev/null | grep -q "64:ff9b" \ + || { echo "FAIL: external names are not synthesized through NAT64"; exit 1; } + # Absolute name deliberately. The registry has its own server + # block and no search domain leads to it, so the bare name is + # NXDOMAIN under ndots:5 -- busybox gives up there, while glibc + # and Go go on to try the name as given, which is what atelet + # does when it pulls from its own netns. Querying what those + # resolvers end up querying is what makes this worth failing on. + nslookup kind-registry. >/dev/null 2>&1 \ + || { echo "FAIL: kind-registry does not resolve; atelet cannot pull"; exit 1; } + echo "DNS-OK" + ' >/dev/null 2>&1 || return 1 + k wait --for=jsonpath='{.status.phase}'=Succeeded pod/dnscheck --timeout=120s \ + >/dev/null 2>&1 + k logs dnscheck > /tmp/dnscheck.log 2>&1 || return 1 + grep -q DNS-OK /tmp/dnscheck.log + } + for attempt in $(seq 6); do + if probe; then cat /tmp/dnscheck.log; exit 0; fi + echo "dnscheck attempt ${attempt} failed" + cat /tmp/dnscheck.log 2>/dev/null || true + sleep 10 + done + echo "::error::cluster DNS did not come up" + exit 1 + - name: Install Agent Substrate + run: hack/install-ate-kind.sh --deploy-ate-system + - name: Assert the control plane is up + # install-ate.sh runs under pipefail, so a failed apply does propagate. + # What it would not catch is a Deployment that rolls out and then + # crash-loops. Re-check everything deploy_ate_system waits on -- + # atenet-egress included, since a non-dual-stack Envoy listener fails + # there first, by way of a readiness probe the kubelet cannot reach. + run: | + for r in deployment/ate-api-server deployment/ate-controller \ + deployment/atenet-router deployment/atenet-egress \ + deployment/dns statefulset/postgres daemonset/atelet; do + kubectl --context="$KUBECTL_CONTEXT" -n ate-system rollout status "$r" --timeout=120s + done + kubectl --context="$KUBECTL_CONTEXT" -n podcertificate-controller-system \ + rollout status deployment/podcertificate-controller --timeout=120s + if kubectl --context="$KUBECTL_CONTEXT" -n ate-system get pods \ + -o jsonpath='{.items[*].status.containerStatuses[*].state.waiting.reason}' \ + | grep -q CrashLoopBackOff; then + echo "::error::a pod in ate-system is in CrashLoopBackOff" + exit 1 + fi + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Assert the demo fixtures exist + # A failed demo deploy exits 0: install-ate.sh dispatches demos through + # `if "${demo}_cmdline" "$1"`, which suspends errexit, and _cmdline ends + # in an unconditional `return 0`. Without this the suites fail later with + # "ActorTemplate not found", pointing at the tests instead of the install. + run: | + for ns_tmpl in ate-demo-counter/counter ate-demo-egress/egress; do + ns=${ns_tmpl%/*}; tmpl=${ns_tmpl#*/} + kubectl --context="$KUBECTL_CONTEXT" -n "${ns}" get actortemplate "${tmpl}" \ + || { echo "::error::${ns_tmpl} was not created -- the demo deploy failed silently"; exit 1; } + done + - name: Run E2E tests (networking) + id: e2e-networking + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/networking -v -args --no-color 2>&1 \ + | tee /tmp/e2e-networking.log + - name: Guard against a vacuously green run + # A test that gates on a dual-stack Service skips itself on v6-only and + # exits 0, so a suite that only skipped would otherwise read as a pass. + # The bar is one real PASS, not zero skips: a test with nothing to compare + # on a single-family cluster is right to skip. Skips are printed so a + # growing list gets noticed. + # Skipped when the suite never ran: with no log to count, this step would + # otherwise report a reassuring zero on a job that failed earlier. + if: always() && steps.e2e-networking.outcome != 'skipped' + run: | + for f in /tmp/e2e-networking.log; do + [ -s "$f" ] || { echo "::error::${f} is missing or empty"; exit 1; } + passed=$(grep -c -- '--- PASS' "$f" || true) + echo "${f}: ${passed} passed, $(grep -c -- '--- SKIP' "$f" || true) skipped" + grep -h -- '--- SKIP' "$f" || true + if [ "${passed}" -eq 0 ]; then + echo "::error::${f} has no passing tests -- a suite that only skips proves nothing" + exit 1 + fi + done + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context="$KUBECTL_CONTEXT" get actortemplate,workerpool,pods -A -o wide || true + dump() { + echo "=== logs: $1/$2 ===" + kubectl --context="$KUBECTL_CONTEXT" logs -n "$1" "$2" --all-containers --tail=300 2>/dev/null || true + } + for p in $(kubectl --context="$KUBECTL_CONTEXT" get pods -n ate-system -o name 2>/dev/null); do + dump ate-system "$p" + done + # Every worker pod in any namespace: the demo pools plus the e2e suites' + # randomly-named per-test namespaces, which the suites keep on failure. + kubectl --context="$KUBECTL_CONTEXT" get pods -A -l ate.dev/worker-pool \ + -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ + | while read -r ns name; do dump "$ns" "$name"; done + # IPv6-specific: the Corefile create-kind-cluster.sh built, and what each + # Service actually got assigned, are the two things that differ from the + # IPv4 job. + kubectl --context="$KUBECTL_CONTEXT" -n kube-system logs -l k8s-app=kube-dns --tail=100 || true + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' || true + kubectl --context="$KUBECTL_CONTEXT" get svc -A \ + -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,POLICY:.spec.ipFamilyPolicy,IPS:.spec.clusterIPs || true + # tayga logs a reason for every packet it declines to translate, which + # is the only view of an egress failure that is not a bare timeout. + # Say when there is no log rather than letting tail's stderr through: + # the run that dies before NAT64 starts is the one whose diagnostics + # most need to not end on something that reads like a second failure. + echo "=== tayga ===" + sudo tail -100 /tmp/tayga.log 2>/dev/null || echo "(no log; NAT64 never started)" From 47655ec7256fc7c8c980e65c685c1f392d24039c Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 20 Aug 2026 14:48:33 -0700 Subject: [PATCH 21/22] ateomnet: move the actor nftables table to the inet family The actor's NAT and filter rules lived in an ip table, which can only ever carry IPv4. They are now in an inet table, so one table can hold both address families when the actor veth becomes dual-stack. Every IPv4 match opens with an NFPROTO comparison, because a bare payload match in an inet table would read an IPv4 offset out of an IPv6 header. IPv4 behaviour is unchanged, but the move is not a no-op on a dual-stack pod: inet nat chains register the nat hooks for both families, so IPv6 traffic in the worker pod netns is now conntracked, and the forward accept now covers IPv6. NAT in the inet family needs Linux 5.2 or later. Teardown sweeps ip as well as inet. A table name is unique per family, so the ip table an earlier ateom left behind is invisible to an inet-only cleanup: the dump comes back empty, the "already clean" path reports success, and the stale table keeps redirecting alongside the new one. Part of #246 --- internal/ateomnet/net.go | 69 ++++++++---- internal/ateomnet/net_linux_test.go | 148 +++++++++++++++++++++++++- internal/ateomnet/rules_linux_test.go | 88 +++++++++++++++ 3 files changed, 281 insertions(+), 24 deletions(-) create mode 100644 internal/ateomnet/rules_linux_test.go diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index 1f0fbc92a8..a63377fc2c 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -259,8 +259,15 @@ func InstallActorNftablesRules(egressPort uint16) error { // rules in an ateom-owned table makes cleanup simple and avoids mutating // Kubernetes or CNI-managed chains directly. // - // TODO: Add IPv6 veth addressing, forwarding, and nftables rules once actor - // networking supports dual-stack pods. The current actor network is IPv4-only. + // The table is in the inet family so one table can carry both address + // families once the actor veth is dual-stack. NAT there needs Linux 5.2. + // A bare payload match in that family is ambiguous, so every IPv4 match + // opens with an NFPROTO comparison. An inet nat chain registers the nat + // hooks for both families, so IPv6 traffic in this netns is conntracked + // where an ip table left it untracked. + // + // TODO(#246): Add the IPv6 veth addressing and forwarding this table is + // waiting on. The actor network itself is still IPv4-only. // // The rules do three things: // @@ -278,7 +285,7 @@ func InstallActorNftablesRules(egressPort uint16) error { c := &nftables.Conn{} table := &nftables.Table{ - Family: nftables.TableFamilyIPv4, + Family: nftables.TableFamilyINet, Name: ActorNftTableName, } c.AddTable(table) @@ -304,7 +311,7 @@ func InstallActorNftablesRules(egressPort uint16) error { c.AddRule(&nftables.Rule{ Table: table, Chain: postrouting, - Exprs: append(IPSourceEqual(ActorVethIP), &expr.Masq{}), + Exprs: append(ipv4SourceEqual(ActorVethIP), &expr.Masq{}), }) acceptPolicy := nftables.ChainPolicyAccept @@ -316,6 +323,9 @@ func InstallActorNftablesRules(egressPort uint16) error { Priority: nftables.ChainPriorityFilter, Policy: &acceptPolicy, }) + // Unqualified, so this accepts forwarded IPv6 too -- what the actor needs + // once it is dual-stack. accept is per-table, so it cannot override a drop + // from the CNI's own forward chains. c.AddRule(&nftables.Rule{ Table: table, Chain: forward, @@ -335,30 +345,51 @@ func RemoveActorNftablesRules() error { // Delete the whole ateom nftables table if it exists. The table is // per-worker and currently per-active-actor because this worker path runs at // most one actor at a time. Missing tables are treated as already clean. + // + // Both families are swept, not just the inet one this installs into: a table + // name is unique per family, so an ip table left by an earlier ateom would + // survive every later cleanup and keep redirecting alongside the new one. + // The pod netns outlives an in-place container restart, so an ateom that + // predates the inet table can share a netns with one that follows it + // wherever WorkerPool.spec.ateomImage is a mutable tag -- the dev loop. + // TODO(ypgao): Drop the ip sweep once no live pod can predate this change. c := &nftables.Conn{} - tables, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4) - if err != nil { - return fmt.Errorf("while listing nftables tables: %w", err) - } - for _, table := range tables { - if table.Name != ActorNftTableName { - continue + for _, family := range []struct { + name string + id nftables.TableFamily + }{{"inet", nftables.TableFamilyINet}, {"ip", nftables.TableFamilyIPv4}} { + tables, err := c.ListTablesOfFamily(family.id) + if err != nil { + return fmt.Errorf("while listing %s nftables tables: %w", family.name, err) } - c.DelTable(table) - if err := c.Flush(); err != nil { - return fmt.Errorf("while deleting actor nftables table: %w", err) + for _, table := range tables { + if table.Name != ActorNftTableName { + continue + } + c.DelTable(table) + if err := c.Flush(); err != nil { + return fmt.Errorf("while deleting the %s actor nftables table: %w", family.name, err) + } } - return nil } return nil } -func IPSourceEqual(ip string) []expr.Any { - return IPPayloadEqual(12, ip) +func ipv4SourceEqual(ip string) []expr.Any { + return ipv4PayloadEqual(12, ip) } -func IPPayloadEqual(offset uint32, ip string) []expr.Any { +// ipv4PayloadEqual matches a 4-byte IPv4 network-header field. The leading +// nfproto comparison is what makes it safe in the inet table: without it the +// payload load would read the same offset out of an IPv6 header. +func ipv4PayloadEqual(offset uint32, ip string) []expr.Any { return []expr.Any{ + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.NFPROTO_IPV4}, + }, &expr.Payload{ DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, @@ -391,7 +422,7 @@ func ActorEgressRedirectRule(table *nftables.Table, chain *nftables.Chain, port if port == 0 { return nil } - exprs := append(IPSourceEqual(ActorVethIP), TCPProtocol()...) + exprs := append(ipv4SourceEqual(ActorVethIP), TCPProtocol()...) exprs = append(exprs, &expr.Immediate{ Register: 1, diff --git a/internal/ateomnet/net_linux_test.go b/internal/ateomnet/net_linux_test.go index b9c8ac45f4..0216585f4c 100644 --- a/internal/ateomnet/net_linux_test.go +++ b/internal/ateomnet/net_linux_test.go @@ -17,6 +17,7 @@ package ateomnet import ( + "bytes" "context" "errors" "runtime" @@ -24,6 +25,8 @@ import ( "github.com/agent-substrate/substrate/internal/roottest" "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) @@ -75,17 +78,51 @@ func withTestNetNS(t *testing.T, fn func(interior netns.NsHandle)) { fn(interior) } -// requireNftables skips when the kernel in this environment cannot serve the -// nftables netlink API at all, which SetupActorNetwork needs and which is a -// property of the machine rather than of the code under test. +// requireNftables skips when this kernel cannot serve what SetupActorNetwork +// installs, which is a property of the machine rather than of the code under +// test. Listing the inet family is not a sufficient probe: inet filter is far +// older than inet nat, which needs Linux 5.2, so this builds and drops a nat +// chain in the family the actor table uses. func requireNftables(t *testing.T) { t.Helper() c := &nftables.Conn{} - if _, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4); err != nil { - t.Skipf("nftables unavailable in this environment: %v", err) + probe := c.AddTable(&nftables.Table{Family: nftables.TableFamilyINet, Name: "ateom_nft_probe"}) + c.AddChain(&nftables.Chain{ + Name: "prerouting", + Table: probe, + Type: nftables.ChainTypeNAT, + Hooknum: nftables.ChainHookPrerouting, + Priority: nftables.ChainPriorityNATDest, + }) + if err := c.Flush(); err != nil { + t.Skipf("nftables inet nat unavailable in this environment: %v", err) + } + c.DelTable(probe) + if err := c.Flush(); err != nil { + t.Fatalf("deleting the nftables probe table: %v", err) } } +// actorNftTableExists reports whether the actor table is present in the family +// InstallActorNftablesRules creates it in. The family is load-bearing: +// ListTablesOfFamily puts it in the netlink dump header, so the kernel filters +// the dump and a query for the wrong family comes back empty rather than +// erroring. +func actorNftTableExists(t *testing.T) bool { + t.Helper() + c := &nftables.Conn{} + tables, err := c.ListTablesOfFamily(nftables.TableFamilyINet) + if err != nil { + t.Fatalf("listing inet nftables tables: %v", err) + } + for _, table := range tables { + if table.Name == ActorNftTableName { + return true + } + } + return false +} + // linkByName returns the link, or nil when it does not exist. func linkByName(t *testing.T, name string) netlink.Link { t.Helper() @@ -215,9 +252,20 @@ func TestSetupActorNetworkIsRepeatable(t *testing.T) { if linkByName(t, HostVethName) == nil { t.Fatalf("host veth %q missing after activation %d", HostVethName, i) } + if !actorNftTableExists(t) { + t.Fatalf("nftables table %q missing after activation %d", ActorNftTableName, i) + } if err := CleanupActorNetwork(ctx, interior); err != nil { t.Fatalf("CleanupActorNetwork (activation %d): %v", i, err) } + // Install and teardown have to name the same family. When they do not, + // teardown's dump comes back empty, its "missing tables are already + // clean" path reports success, and the table survives -- so the next + // activation stacks another copy of every chain and rule onto it and + // the leak is invisible to every other assertion here. + if actorNftTableExists(t) { + t.Fatalf("nftables table %q survived cleanup after activation %d", ActorNftTableName, i) + } } // Cleanup is idempotent: the extra call after the loop's last one must @@ -228,6 +276,9 @@ func TestSetupActorNetworkIsRepeatable(t *testing.T) { if stray := linkByName(t, HostVethName); stray != nil { t.Errorf("host veth %q survived cleanup", HostVethName) } + if actorNftTableExists(t) { + t.Errorf("nftables table %q survived a repeated cleanup", ActorNftTableName) + } if err := NetNSDo(ctx, interior, func(context.Context) error { if stray := linkByName(t, ActorVethName); stray != nil { t.Errorf("actor veth %q survived cleanup", ActorVethName) @@ -239,6 +290,93 @@ func TestSetupActorNetworkIsRepeatable(t *testing.T) { }) } +// TestRemoveActorNftablesRulesSweepsIPv4Family covers the upgrade case: a +// worker whose previous ateom created the actor table in the ip family. Table +// names are unique per family, so an inet-only cleanup could never see that +// table, and it would have kept redirecting alongside the inet one installed +// next to it. +func TestRemoveActorNftablesRulesSweepsIPv4Family(t *testing.T) { + roottest.Require(t, "creating network namespaces and nftables rules") + + withTestNetNS(t, func(netns.NsHandle) { + requireNftables(t) + + c := &nftables.Conn{} + c.AddTable(&nftables.Table{Family: nftables.TableFamilyIPv4, Name: ActorNftTableName}) + if err := c.Flush(); err != nil { + t.Fatalf("creating the stand-in ip actor table: %v", err) + } + + if err := RemoveActorNftablesRules(); err != nil { + t.Fatalf("RemoveActorNftablesRules: %v", err) + } + + tables, err := c.ListTablesOfFamily(nftables.TableFamilyIPv4) + if err != nil { + t.Fatalf("listing ip nftables tables: %v", err) + } + for _, table := range tables { + if table.Name == ActorNftTableName { + t.Fatal("the ip actor table survived cleanup") + } + } + }) +} + +// TestSetupActorNetworkInstallsEgressRedirect covers the rule no other test in +// this package builds: they all leave EgressRedirectPort zero, so the kernel +// never sees the redirect. Its acceptance is not implied by the masquerade rule +// next to it -- redirect in the inet family is separate kernel support from the +// nat chain type -- and it is the rule the whole actor egress path rides on. +func TestSetupActorNetworkInstallsEgressRedirect(t *testing.T) { + roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") + ctx := context.Background() + + const egressPort = 15001 + + withTestNetNS(t, func(interior netns.NsHandle) { + requireNftables(t) + + if err := SetupActorNetwork(ctx, NetworkConfig{ + InteriorNetNS: interior, + EgressRedirectPort: egressPort, + }); err != nil { + t.Fatalf("SetupActorNetwork: %v", err) + } + + c := &nftables.Conn{} + rules, err := c.GetRules( + &nftables.Table{Family: nftables.TableFamilyINet, Name: ActorNftTableName}, + &nftables.Chain{Name: "prerouting"}, + ) + if err != nil { + t.Fatalf("listing prerouting rules of the actor table: %v", err) + } + if len(rules) != 1 { + t.Fatalf("prerouting holds %d rules, want the egress redirect alone", len(rules)) + } + + // Read back what the kernel stored rather than what the builder emitted: + // TestActorNftablesRuleExprs already pins the builder, and what is in + // doubt here is whether an inet nat chain takes these expressions at all. + var haveNFProto, havePort, haveRedir bool + for _, e := range rules[0].Exprs { + switch e := e.(type) { + case *expr.Meta: + haveNFProto = haveNFProto || e.Key == expr.MetaKeyNFPROTO + case *expr.Immediate: + havePort = havePort || bytes.Equal(e.Data, binaryutil.BigEndian.PutUint16(egressPort)) + case *expr.Redir: + haveRedir = true + } + } + if !haveNFProto || !havePort || !haveRedir { + t.Errorf("installed redirect has nfproto=%t port=%t redir=%t, want all three, got %v", + haveNFProto, havePort, haveRedir, rules[0].Exprs) + } + }) +} + // TestSetupActorNetworkHostVethHWAddr covers the micro-VM requirement: a CH // snapshot freezes the guest's ARP entry for the gateway, so the worker-side // veth MAC has to be exactly the one the caller asked for, on every pod. diff --git a/internal/ateomnet/rules_linux_test.go b/internal/ateomnet/rules_linux_test.go new file mode 100644 index 0000000000..b6b8b53f3e --- /dev/null +++ b/internal/ateomnet/rules_linux_test.go @@ -0,0 +1,88 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomnet + +import ( + "fmt" + "reflect" + "testing" + + "github.com/google/nftables/expr" +) + +// TestActorNftablesRuleExprs pins the expressions installed into the inet actor +// table. The nfproto guard in front of every IPv4 match is what makes those +// matches safe there -- without it the payload load reads an IPv4 offset out of +// an IPv6 header -- and no other test in the package can see it: an IPv4-only +// datapath still behaves correctly with the guard removed. +// +// The wants are spelled out as literal bytes rather than built from the same +// helpers as the code, so they pin the wire encoding and not just its spelling. +func TestActorNftablesRuleExprs(t *testing.T) { + // meta nfproto ipv4; ip saddr 169.254.17.2 + actorSourceIsIPv4 := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{2}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 12, Len: 4}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{169, 254, 17, 2}}, + } + + tests := []struct { + name string + got []expr.Any + want []expr.Any + }{{ + name: "source match guards the payload load with nfproto", + got: ipv4SourceEqual(ActorVethIP), + want: actorSourceIsIPv4, + }, { + name: "egress redirect matches actor IPv4 TCP and redirects to the port", + got: ActorEgressRedirectRule(nil, nil, 15001).Exprs, + want: append(append([]expr.Any{}, actorSourceIsIPv4...), + // meta l4proto tcp; redirect to :15001 + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}}, + &expr.Immediate{Register: 1, Data: []byte{0x3a, 0x99}}, + &expr.Redir{RegisterProtoMin: 1}, + ), + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if !reflect.DeepEqual(test.got, test.want) { + t.Errorf("rule exprs mismatch:\ngot:\n%s\nwant:\n%s", formatExprs(test.got), formatExprs(test.want)) + } + }) + } +} + +// TestActorEgressRedirectRuleDisabled covers the zero port: no rule at all, so +// actor egress stays on the masquerade path instead of being redirected to a +// listener that is not there. +func TestActorEgressRedirectRuleDisabled(t *testing.T) { + if rule := ActorEgressRedirectRule(nil, nil, 0); rule != nil { + t.Errorf("ActorEgressRedirectRule(0) = %v, want nil", rule.Exprs) + } +} + +func formatExprs(exprs []expr.Any) string { + var s string + for _, e := range exprs { + s += fmt.Sprintf(" %T%+v\n", e, e) + } + return s +} From 703b8ee9f77b76493a64f2d2d9b808b5f8c3675a Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 20 Aug 2026 14:56:11 -0700 Subject: [PATCH 22/22] ateomnet: give the actor an IPv6 address when the pod has one Actor networking was IPv4-only, so an actor on a dual-stack worker pod could not reach an IPv6-only destination at all. SetupActorNetwork now assigns the fd00:169:254::/126 counterparts of the existing point-to-point pair to both ends of the actor veth, installs an IPv6 default route in the interior netns, and adds the matching rules to the inet-family actor table. Whether the actor gets IPv6 is decided once in the worker pod netns and carried into the interior one, which is created fresh and so always reports IPv6 available whatever the cluster's families are. Both halves have to hold: the pod needs a global IPv6 address of its own, and the veth has to accept an IPv6 address -- IPv4-only GKE sets disable_ipv6 and netlink then rejects the assignment with EPERM. Addresses carry IFA_F_NODAD rather than the accept_dad sysctl, which the unprivileged ateom container cannot write. Part of #246 --- cmd/ateom-microvm/run.go | 4 + internal/ateomnet/net.go | 194 +++++++++++++++++-- internal/ateomnet/net_linux_test.go | 268 ++++++++++++++++++++++++-- internal/ateomnet/rules_linux_test.go | 39 +++- 4 files changed, 467 insertions(+), 38 deletions(-) diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 7930c9e108..5b968304f7 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -1042,6 +1042,10 @@ func tailString(s string, n int) string { // agent: configure eth0 (IP/MAC/MTU), install the connected + default routes, and // pin the gateway's ARP entry to its fixed MAC (so a restored guest's frozen // neighbor entry stays valid). +// +// TODO(#246): the guest is configured IPv4-only, so a micro-VM actor sees no +// IPv6 even on a dual-stack pod where the host veth has one. gVisor reads the +// interior netns and picks the address up; this path has to be told. func (s *AteomService) configureGuestNetwork(ctx context.Context, ac *kata.AgentClient, mtu uint64) error { if err := ac.UpdateInterface(ctx, &agentpb.Interface{ Device: ateomnet.ActorVethName, diff --git a/internal/ateomnet/net.go b/internal/ateomnet/net.go index a63377fc2c..2576669c1a 100644 --- a/internal/ateomnet/net.go +++ b/internal/ateomnet/net.go @@ -43,6 +43,22 @@ const ( ActorVethIP = "169.254.17.2" ActorNftTableName = "ateom_actor" + // podPrimaryIfaceName is the worker pod's own interface, the one the CNI + // gave it. It is not ActorVethName despite the identical value: that one + // names the actor's end of the veth, which lives in the interior netns. + podPrimaryIfaceName = "eth0" + + // The IPv6 counterparts of the point-to-point pair above, chosen to echo the + // v4 addresses digit for digit. A fixed ULA rather than an RFC 4193 random + // prefix so the pair stays as readable in a packet dump as 169.254.17.x, and + // not fe80::/10 because a link-local source would need a scope id everywhere + // it is used. It must not overlap the cluster's pod CIDR; kind's dual-stack + // default is fd00:10:244::/56. + HostVethIPv6CIDR = "fd00:169:254::1/126" + ActorVethIPv6CIDR = "fd00:169:254::2/126" + ActorVethIPv6Gateway = "fd00:169:254::1" + ActorVethIPv6IP = "fd00:169:254::2" + // ActorVethSubnet is the point-to-point /30 the actor veth lives on. ActorVethSubnet = "169.254.17.0/30" ) @@ -51,6 +67,10 @@ var ( HostVethAddr = MustParseAddr(HostVethCIDR) ActorVethAddr = MustParseAddr(ActorVethCIDR) ActorVethGwIP = MustParseIP(ActorVethGateway) + + HostVethIPv6Addr = mustParseNoDADAddr(HostVethIPv6CIDR) + ActorVethIPv6Addr = mustParseNoDADAddr(ActorVethIPv6CIDR) + ActorVethIPv6GwIP = MustParseIPv6(ActorVethIPv6Gateway) ) // MustParseAddr parses a CIDR string into a netlink.Addr, panicking on error. @@ -62,6 +82,18 @@ func MustParseAddr(cidr string) *netlink.Addr { return a } +// mustParseNoDADAddr parses a CIDR into an address flagged IFA_F_NODAD. +// +// Per-address flag rather than the interface-wide accept_dad sysctl because the +// ateom container is unprivileged, so containerd mounts /proc/sys read-only. +// DAD is pointless on a point-to-point veth nobody else can reach, and it would +// otherwise hold the address tentative for ~1s on every resume. +func mustParseNoDADAddr(cidr string) *netlink.Addr { + a := MustParseAddr(cidr) + a.Flags = unix.IFA_F_NODAD + return a +} + // MustParseIP parses an IPv4 string into a net.IP, panicking on error. func MustParseIP(s string) net.IP { ip := net.ParseIP(s).To4() @@ -71,6 +103,57 @@ func MustParseIP(s string) net.IP { return ip } +// MustParseIPv6 parses an IPv6 string into a net.IP, panicking on error. An +// IPv4 string is an error: net.IP holds it as a 16-byte v4-mapped address, so +// it would pass a length check and then compare against no IPv6 header. +func MustParseIPv6(s string) net.IP { + ip := net.ParseIP(s) + if ip == nil || ip.To4() != nil { + panic(fmt.Sprintf("parsing constant IPv6 %q", s)) + } + return ip.To16() +} + +// linkIPv6Enabled reports whether IPv6 addresses can be assigned to the named +// link in the current netns. It answers a kernel capability question, not a +// cluster one: IPv4-only GKE leaves disable_ipv6=1 and netlink then rejects +// every IPv6 address with EPERM, but IPv4-only kind leaves it at 0 because the +// node kernel has IPv6 compiled in. A kernel built without IPv6 has no sysctl +// at all. Pair it with linkHasGlobalIPv6 to decide whether the actor gets IPv6; +// on its own it says yes on clusters that have no IPv6 anywhere. +func linkIPv6Enabled(name string) bool { + b, err := os.ReadFile("/proc/sys/net/ipv6/conf/" + name + "/disable_ipv6") + if err != nil { + return false + } + return len(b) > 0 && b[0] == '0' +} + +// linkHasGlobalIPv6 reports whether link carries a global IPv6 address. Called +// on the worker pod's own interface, that is what decides the families the +// actor can egress on. +// +// It answers whether the pod has an address to egress from, not whether that +// address routes anywhere: IsGlobalUnicast is true for a ULA, and dual-stack +// kind hands pods a ULA with no path off the host. Reachability is the +// cluster's problem, not something this can decide from inside the netns. +func linkHasGlobalIPv6(ctx context.Context, link netlink.Link) bool { + // netlink can report ErrDumpInterrupted alongside a valid partial answer. + // Trust a positive result either way: reporting false on a dual-stack pod + // silently strands the actor on IPv4, which is the costlier mistake. + addrs, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + slog.WarnContext(ctx, "listing IPv6 addresses of the worker pod interface", + "link", link.Attrs().Name, "error", err, "addressesRead", len(addrs)) + } + for _, addr := range addrs { + if addr.IP.IsGlobalUnicast() { + return true + } + } + return false +} + // MustParseMAC parses a MAC address string into a net.HardwareAddr, panicking on error. func MustParseMAC(s string) net.HardwareAddr { m, err := net.ParseMAC(s) @@ -82,7 +165,9 @@ func MustParseMAC(s string) net.HardwareAddr { // ConfigureActorVeth configures the actor veth inside the interior netns. // It assumes it is already running inside the target network namespace. -func ConfigureActorVeth(ctx context.Context) error { +// ipv6 comes from SetupActorNetwork, which decides it in the worker pod netns; +// this namespace cannot answer the question for itself. +func ConfigureActorVeth(ctx context.Context, ipv6 bool) error { // Run inside the gVisor interior netns. SetupActorNetwork has already created // the veth peer here, under its final name, so this only has to address it. // gVisor reads link names, addresses, and routes from this namespace when the @@ -107,6 +192,12 @@ func ConfigureActorVeth(ctx context.Context) error { if err := netlink.AddrReplace(actorLink, ActorVethAddr); err != nil { return fmt.Errorf("while assigning actor veth address: %w", err) } + if ipv6 { + if err := netlink.AddrReplace(actorLink, ActorVethIPv6Addr); err != nil { + return fmt.Errorf("while assigning actor veth ipv6 address: %w", err) + } + } + if err := netlink.LinkSetUp(actorLink); err != nil { return fmt.Errorf("while bringing up actor veth: %w", err) } @@ -117,6 +208,15 @@ func ConfigureActorVeth(ctx context.Context) error { }); err != nil { return fmt.Errorf("while installing actor default route: %w", err) } + if ipv6 { + if err := netlink.RouteReplace(&netlink.Route{ + LinkIndex: actorLink.Attrs().Index, + Gw: ActorVethIPv6GwIP, + Dst: &net.IPNet{IP: net.ParseIP("::"), Mask: net.CIDRMask(0, 128)}, + }); err != nil { + return fmt.Errorf("while installing actor default ipv6 route: %w", err) + } + } return nil } @@ -173,7 +273,7 @@ func PodIPv4() (net.IP, error) { // Resolve the worker pod IPv4 address from the pod namespace's real eth0. // Because eth0 now stays in the pod namespace, this IP remains available for // both normal worker connectivity and the temporary inbound DNAT rule. - eth0Link, err := netlink.LinkByName("eth0") + eth0Link, err := netlink.LinkByName(podPrimaryIfaceName) if err != nil { return nil, fmt.Errorf("while getting pod eth0: %w", err) } @@ -259,15 +359,10 @@ func InstallActorNftablesRules(egressPort uint16) error { // rules in an ateom-owned table makes cleanup simple and avoids mutating // Kubernetes or CNI-managed chains directly. // - // The table is in the inet family so one table can carry both address - // families once the actor veth is dual-stack. NAT there needs Linux 5.2. - // A bare payload match in that family is ambiguous, so every IPv4 match - // opens with an NFPROTO comparison. An inet nat chain registers the nat - // hooks for both families, so IPv6 traffic in this netns is conntracked - // where an ip table left it untracked. - // - // TODO(#246): Add the IPv6 veth addressing and forwarding this table is - // waiting on. The actor network itself is still IPv4-only. + // The table is in the inet family so one table carries both address + // families. NAT there needs Linux 5.2. A bare payload match in that family + // is ambiguous, so every family-specific match opens with an NFPROTO + // comparison. // // The rules do three things: // @@ -300,6 +395,9 @@ func InstallActorNftablesRules(egressPort uint16) error { if redirectRule := ActorEgressRedirectRule(table, prerouting, egressPort); redirectRule != nil { c.AddRule(redirectRule) } + if redirectRuleIPv6 := ActorIPv6EgressRedirectRule(table, prerouting, egressPort); redirectRuleIPv6 != nil { + c.AddRule(redirectRuleIPv6) + } postrouting := c.AddChain(&nftables.Chain{ Name: "postrouting", @@ -313,6 +411,11 @@ func InstallActorNftablesRules(egressPort uint16) error { Chain: postrouting, Exprs: append(ipv4SourceEqual(ActorVethIP), &expr.Masq{}), }) + c.AddRule(&nftables.Rule{ + Table: table, + Chain: postrouting, + Exprs: append(ipv6SourceEqual(ActorVethIPv6IP), &expr.Masq{}), + }) acceptPolicy := nftables.ChainPolicyAccept forward := c.AddChain(&nftables.Chain{ @@ -404,6 +507,35 @@ func ipv4PayloadEqual(offset uint32, ip string) []expr.Any { } } +func ipv6SourceEqual(ip string) []expr.Any { + return ipv6PayloadEqual(8, ip) +} + +// ipv6PayloadEqual matches a 16-byte IPv6 network-header field. Offset 8 is the +// source address, where the IPv4 source sits at 12: the nfproto comparison in +// front is what keeps each rule off the other family's packets. +func ipv6PayloadEqual(offset uint32, ip string) []expr.Any { + return []expr.Any{ + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.NFPROTO_IPV6}, + }, + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: offset, + Len: 16, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: MustParseIPv6(ip), + }, + } +} + func TCPProtocol() []expr.Any { return []expr.Any{ &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, @@ -433,6 +565,24 @@ func ActorEgressRedirectRule(table *nftables.Table, chain *nftables.Chain, port return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} } +// ActorIPv6EgressRedirectRule is ActorEgressRedirectRule for the actor's IPv6 +// source address. Both rules live in the same inet table, so each carries its +// own NFPROTO match to keep it off the other family's packets. +func ActorIPv6EgressRedirectRule(table *nftables.Table, chain *nftables.Chain, port uint16) *nftables.Rule { + if port == 0 { + return nil + } + exprs := append(ipv6SourceEqual(ActorVethIPv6IP), TCPProtocol()...) + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.BigEndian.PutUint16(port), + }, + &expr.Redir{RegisterProtoMin: 1}, + ) + return &nftables.Rule{Table: table, Chain: chain, Exprs: exprs} +} + // CreateNetNSWithoutSwitching creates a named netns and returns its handle, // restoring the caller's current netns before returning. func CreateNetNSWithoutSwitching(name string) (netns.NsHandle, error) { @@ -618,11 +768,31 @@ func SetupActorNetwork(ctx context.Context, cfg NetworkConfig) (retErr error) { if err := netlink.AddrReplace(hostLink, HostVethAddr); err != nil { return fmt.Errorf("while assigning host veth address: %w", err) } + // Decided once, here in the worker pod netns, and carried into the interior + // netns below. Probing separately on each side would let them disagree: the + // interior netns is freshly created, so its sysctl is always the permissive + // kernel default whatever the pod's families are. + var podIPv6 bool + if podLink, err := netlink.LinkByName(podPrimaryIfaceName); err == nil { + podIPv6 = linkHasGlobalIPv6(ctx, podLink) + } + vethIPv6 := linkIPv6Enabled(HostVethName) + actorIPv6 := podIPv6 && vethIPv6 + if actorIPv6 { + if err := netlink.AddrReplace(hostLink, HostVethIPv6Addr); err != nil { + return fmt.Errorf("while assigning host veth ipv6 address: %w", err) + } + } else { + slog.InfoContext(ctx, "actor networking is IPv4-only", + "link", HostVethName, "podHasGlobalIPv6", podIPv6, "vethIPv6Enabled", vethIPv6) + } if err := netlink.LinkSetUp(hostLink); err != nil { return fmt.Errorf("while bringing up host veth: %w", err) } - if err := NetNSDo(ctx, cfg.InteriorNetNS, ConfigureActorVeth); err != nil { + if err := NetNSDo(ctx, cfg.InteriorNetNS, func(ctx context.Context) error { + return ConfigureActorVeth(ctx, actorIPv6) + }); err != nil { return fmt.Errorf("while configuring actor veth in interior netns: %w", err) } diff --git a/internal/ateomnet/net_linux_test.go b/internal/ateomnet/net_linux_test.go index 0216585f4c..2e4dd38f03 100644 --- a/internal/ateomnet/net_linux_test.go +++ b/internal/ateomnet/net_linux_test.go @@ -20,6 +20,8 @@ import ( "bytes" "context" "errors" + "net" + "os" "runtime" "testing" @@ -29,6 +31,7 @@ import ( "github.com/google/nftables/expr" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" + "golang.org/x/sys/unix" ) // withTestNetNS runs fn with the calling thread inside a throwaway netns @@ -325,9 +328,10 @@ func TestRemoveActorNftablesRulesSweepsIPv4Family(t *testing.T) { // TestSetupActorNetworkInstallsEgressRedirect covers the rule no other test in // this package builds: they all leave EgressRedirectPort zero, so the kernel -// never sees the redirect. Its acceptance is not implied by the masquerade rule -// next to it -- redirect in the inet family is separate kernel support from the -// nat chain type -- and it is the rule the whole actor egress path rides on. +// never sees either redirect. Their acceptance is not implied by the masquerade +// rule next to them -- redirect in the inet family is separate kernel support +// from the nat chain type -- and they are what the whole actor egress path +// rides on. func TestSetupActorNetworkInstallsEgressRedirect(t *testing.T) { roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") ctx := context.Background() @@ -352,31 +356,259 @@ func TestSetupActorNetworkInstallsEgressRedirect(t *testing.T) { if err != nil { t.Fatalf("listing prerouting rules of the actor table: %v", err) } - if len(rules) != 1 { - t.Fatalf("prerouting holds %d rules, want the egress redirect alone", len(rules)) + if len(rules) != 2 { + t.Fatalf("prerouting holds %d rules, want one redirect per family", len(rules)) } // Read back what the kernel stored rather than what the builder emitted: // TestActorNftablesRuleExprs already pins the builder, and what is in // doubt here is whether an inet nat chain takes these expressions at all. - var haveNFProto, havePort, haveRedir bool - for _, e := range rules[0].Exprs { - switch e := e.(type) { - case *expr.Meta: - haveNFProto = haveNFProto || e.Key == expr.MetaKeyNFPROTO - case *expr.Immediate: - havePort = havePort || bytes.Equal(e.Data, binaryutil.BigEndian.PutUint16(egressPort)) - case *expr.Redir: - haveRedir = true + // Both rules are installed whatever the pod's families are; the one whose + // source address the actor never gets simply matches nothing. + for i, rule := range rules { + var nfproto []byte + var havePort, haveRedir bool + for _, e := range rule.Exprs { + switch e := e.(type) { + case *expr.Cmp: + if nfproto == nil && len(e.Data) == 1 { + nfproto = e.Data + } + case *expr.Immediate: + havePort = havePort || bytes.Equal(e.Data, binaryutil.BigEndian.PutUint16(egressPort)) + case *expr.Redir: + haveRedir = true + } + } + wantNFProto := []byte{unix.NFPROTO_IPV4} + if i == 1 { + wantNFProto = []byte{unix.NFPROTO_IPV6} + } + if !bytes.Equal(nfproto, wantNFProto) || !havePort || !haveRedir { + t.Errorf("prerouting rule %d has nfproto=%v port=%t redir=%t, want nfproto=%v and both, got %v", + i, nfproto, havePort, haveRedir, wantNFProto, rule.Exprs) } - } - if !haveNFProto || !havePort || !haveRedir { - t.Errorf("installed redirect has nfproto=%t port=%t redir=%t, want all three, got %v", - haveNFProto, havePort, haveRedir, rules[0].Exprs) } }) } +// addPodEth0 plants a dummy link carrying cidrs in the current netns, standing +// in for the worker pod's own primary interface. withTestNetNS hands out a bare +// namespace, and the families on that interface are what SetupActorNetwork reads +// to decide the families the actor gets. +// +// The name has to be exactly podPrimaryIfaceName: the probe is link-scoped, so +// under any other name it answers false and the test asserts the opposite of +// what it means to. +func addPodEth0(t *testing.T, cidrs ...string) { + t.Helper() + + link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: podPrimaryIfaceName}} + if err := netlink.LinkAdd(link); err != nil { + t.Fatalf("creating the stand-in pod %s: %v", podPrimaryIfaceName, err) + } + if err := netlink.LinkSetUp(link); err != nil { + t.Fatalf("bringing up the stand-in pod %s: %v", podPrimaryIfaceName, err) + } + for _, cidr := range cidrs { + addr := MustParseAddr(cidr) + addr.Flags |= unix.IFA_F_NODAD // else an IPv6 address stays tentative + if err := netlink.AddrAdd(link, addr); err != nil { + t.Fatalf("assigning %s to the stand-in pod %s: %v", cidr, podPrimaryIfaceName, err) + } + } +} + +// writeSysctl turns an IPv6 knob off in the current netns. "all" flushes the +// addresses already assigned; "default" only reaches links created afterwards. +func writeSysctl(t *testing.T, knob string) { + t.Helper() + path := "/proc/sys/net/ipv6/conf/" + knob + "/disable_ipv6" + if err := os.WriteFile(path, []byte("1\n"), 0o644); err != nil { + t.Fatalf("disabling IPv6 via %s: %v", path, err) + } +} + +// assertIPv6AddrNoDAD requires cidr to be present on link and to carry +// IFA_F_NODAD. +// +// The flag is the whole point: the ateom container is unprivileged, so the +// accept_dad sysctl this replaced could not be written and setup failed outright +// on a real worker. It passes as root, where /proc/sys is writable either way, +// so nothing else here would catch a regression back to the sysctl. +func assertIPv6AddrNoDAD(t *testing.T, link netlink.Link, cidr string) { + t.Helper() + addrs, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + t.Fatalf("listing IPv6 addresses of %q: %v", link.Attrs().Name, err) + } + want := MustParseAddr(cidr) + for _, addr := range addrs { + if addr.IPNet == nil || addr.IPNet.String() != want.IPNet.String() { + continue + } + if addr.Flags&unix.IFA_F_NODAD == 0 { + t.Errorf("%s on %q has flags %#x, want IFA_F_NODAD (%#x) set", cidr, link.Attrs().Name, addr.Flags, unix.IFA_F_NODAD) + } + return + } + t.Errorf("%q does not carry %s, got %v", link.Attrs().Name, cidr, addrs) +} + +// assertNoGlobalIPv6Addr requires link to carry no IPv6 address beyond the +// fe80::/64 the kernel gives every up link wherever IPv6 is enabled at all. +// That link-local is not what strands an actor -- the routable address is. +func assertNoGlobalIPv6Addr(t *testing.T, link netlink.Link) { + t.Helper() + addrs, err := netlink.AddrList(link, netlink.FAMILY_V6) + if err != nil { + t.Fatalf("listing IPv6 addresses of %q: %v", link.Attrs().Name, err) + } + for _, addr := range addrs { + if addr.IP.IsGlobalUnicast() { + t.Errorf("%q carries global IPv6 address %s, want none", link.Attrs().Name, addr) + } + } +} + +// assertDefaultRoute requires link to carry -- or, when want is false, to not +// carry -- a default route via gw in the given family. +func assertDefaultRoute(t *testing.T, link netlink.Link, family int, gw net.IP, want bool) { + t.Helper() + + dst := "0.0.0.0/0" + if family == netlink.FAMILY_V6 { + dst = "::/0" + } + routes, err := netlink.RouteList(link, family) + if err != nil { + t.Fatalf("listing %s routes of %q: %v", dst, link.Attrs().Name, err) + } + var got bool + for _, route := range routes { + // A default route reports its destination either as nil or as an + // explicit zero-length mask, depending on how the kernel rendered it. + ones := 0 + if route.Dst != nil { + ones, _ = route.Dst.Mask.Size() + } + if ones == 0 && route.Gw.Equal(gw) { + got = true + } + } + switch { + case want && !got: + t.Errorf("%q has no %s route via %s, got %v", link.Attrs().Name, dst, gw, routes) + case !want && got: + t.Errorf("%q has a %s route via %s, want none, got %v", link.Attrs().Name, dst, gw, routes) + } +} + +// TestSetupActorNetworkIPv6Gate is the truth table for who gets actor IPv6. +// Both halves have to hold: the worker pod needs a global IPv6 address of its +// own, or the actor prefers the AAAA of a dual-stack destination and the +// connection dies with nowhere to go; and the veth has to accept an IPv6 +// address at all, or the assignment fails with EPERM on the path of every +// SetupActorNetwork call and the actor never starts. +// +// The IPv4 half must come out identical in every case. +func TestSetupActorNetworkIPv6Gate(t *testing.T) { + roottest.Require(t, "creating network namespaces, veth pairs, and nftables rules") + ctx := context.Background() + + for _, tc := range []struct { + name string + // podAddrs go on the stand-in pod interface before setup runs. + podAddrs []string + // disable, when set, runs in the pod netns after podAddrs are assigned. + disable func(*testing.T) + wantIPv6 bool + }{{ + name: "dual-stack pod", + podAddrs: []string{"10.244.0.7/24", "fd00:10:244::7/64"}, + wantIPv6: true, + }, { + // A probe that reads the wrong link or the wrong scope fails closed, + // which every IPv4 case here would happily accept. This one notices. + name: "IPv6-only pod", + podAddrs: []string{"fd00:10:244::7/64"}, + wantIPv6: true, + }, { + // An IPv4-only cluster whose kernel still has IPv6 compiled in, so every + // capability probe says yes. This is the case that turned the IPv4 e2e + // job red. + name: "pod without IPv6", + podAddrs: []string{"10.244.0.7/24"}, + }, { + // The default on IPv4-only GKE. Writing "all" also flushes podAddrs, so + // both halves of the gate are false here. + name: "IPv6 disabled for the whole netns", + podAddrs: []string{"10.244.0.7/24", "fd00:10:244::7/64"}, + disable: func(t *testing.T) { + writeSysctl(t, "all") + writeSysctl(t, "default") + }, + }, { + // The one case the capability half is there for: the pod keeps its + // address, but the veth created next inherits disable_ipv6=1. + name: "IPv6 disabled per link", + podAddrs: []string{"10.244.0.7/24", "fd00:10:244::7/64"}, + disable: func(t *testing.T) { writeSysctl(t, "default") }, + }} { + t.Run(tc.name, func(t *testing.T) { + withTestNetNS(t, func(interior netns.NsHandle) { + requireNftables(t) + + addPodEth0(t, tc.podAddrs...) + if tc.disable != nil { + tc.disable(t) + } + + if err := SetupActorNetwork(ctx, NetworkConfig{InteriorNetNS: interior}); err != nil { + t.Fatalf("SetupActorNetwork: %v", err) + } + + host := linkByName(t, HostVethName) + if host == nil { + t.Fatalf("host veth %q missing from the pod netns", HostVethName) + } + if !hasAddr(t, host, HostVethCIDR) { + t.Errorf("host veth %q does not carry %s", HostVethName, HostVethCIDR) + } + if tc.wantIPv6 { + assertIPv6AddrNoDAD(t, host, HostVethIPv6CIDR) + } else { + assertNoGlobalIPv6Addr(t, host) + } + + if err := NetNSDo(ctx, interior, func(context.Context) error { + actor := linkByName(t, ActorVethName) + if actor == nil { + t.Fatalf("actor veth %q missing from the interior netns", ActorVethName) + } + if !hasAddr(t, actor, ActorVethCIDR) { + t.Errorf("actor veth %q does not carry %s", ActorVethName, ActorVethCIDR) + } + assertDefaultRoute(t, actor, netlink.FAMILY_V4, ActorVethGwIP, true) + + // The interior netns is created fresh, so its own sysctls always + // say IPv6 is available whatever the pod's families are. Only a + // decision carried across from the pod netns gets this right. + if tc.wantIPv6 { + assertIPv6AddrNoDAD(t, actor, ActorVethIPv6CIDR) + } else { + assertNoGlobalIPv6Addr(t, actor) + } + assertDefaultRoute(t, actor, netlink.FAMILY_V6, ActorVethIPv6GwIP, tc.wantIPv6) + return nil + }); err != nil { + t.Fatalf("inspecting interior netns: %v", err) + } + }) + }) + } +} + // TestSetupActorNetworkHostVethHWAddr covers the micro-VM requirement: a CH // snapshot freezes the guest's ARP entry for the gateway, so the worker-side // veth MAC has to be exactly the one the caller asked for, on every pod. diff --git a/internal/ateomnet/rules_linux_test.go b/internal/ateomnet/rules_linux_test.go index b6b8b53f3e..d0bd59301c 100644 --- a/internal/ateomnet/rules_linux_test.go +++ b/internal/ateomnet/rules_linux_test.go @@ -41,24 +41,44 @@ func TestActorNftablesRuleExprs(t *testing.T) { &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{169, 254, 17, 2}}, } + // meta nfproto ipv6; ip6 saddr fd00:169:254::2 + actorSourceIsIPv6 := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{10}}, + &expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 8, Len: 16}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{ + 0xfd, 0x00, 0x01, 0x69, 0x02, 0x54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x02, + }}, + } + + // meta l4proto tcp; redirect to :15001 + tcpRedirectTo15001 := []expr.Any{ + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}}, + &expr.Immediate{Register: 1, Data: []byte{0x3a, 0x99}}, + &expr.Redir{RegisterProtoMin: 1}, + } + tests := []struct { name string got []expr.Any want []expr.Any }{{ - name: "source match guards the payload load with nfproto", + name: "IPv4 source match guards the payload load with nfproto", got: ipv4SourceEqual(ActorVethIP), want: actorSourceIsIPv4, + }, { + name: "IPv6 source match guards the payload load with nfproto", + got: ipv6SourceEqual(ActorVethIPv6IP), + want: actorSourceIsIPv6, }, { name: "egress redirect matches actor IPv4 TCP and redirects to the port", got: ActorEgressRedirectRule(nil, nil, 15001).Exprs, - want: append(append([]expr.Any{}, actorSourceIsIPv4...), - // meta l4proto tcp; redirect to :15001 - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{6}}, - &expr.Immediate{Register: 1, Data: []byte{0x3a, 0x99}}, - &expr.Redir{RegisterProtoMin: 1}, - ), + want: append(append([]expr.Any{}, actorSourceIsIPv4...), tcpRedirectTo15001...), + }, { + name: "egress redirect matches actor IPv6 TCP and redirects to the port", + got: ActorIPv6EgressRedirectRule(nil, nil, 15001).Exprs, + want: append(append([]expr.Any{}, actorSourceIsIPv6...), tcpRedirectTo15001...), }} for _, test := range tests { @@ -77,6 +97,9 @@ func TestActorEgressRedirectRuleDisabled(t *testing.T) { if rule := ActorEgressRedirectRule(nil, nil, 0); rule != nil { t.Errorf("ActorEgressRedirectRule(0) = %v, want nil", rule.Exprs) } + if rule := ActorIPv6EgressRedirectRule(nil, nil, 0); rule != nil { + t.Errorf("ActorIPv6EgressRedirectRule(0) = %v, want nil", rule.Exprs) + } } func formatExprs(exprs []expr.Any) string {