From f75afbbe85765aa74de7494efe595603d6e7f218 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Tue, 28 Jul 2026 22:35:34 +0100
Subject: [PATCH 01/19] fix(agent): verify KVM openability; wire real-agent +
 datapath e2e

capability.Check now opens /dev/kvm (O_RDWR) instead of stat, catching
permission-blocked-but-existing devices (#34).

e2e_suite_test.go conditionally drops the no-agent nodeSelector and sets
IMP_SCALE_TO_ZERO=true when IMP_E2E_REAL_AGENT=true, enabling a real
Firecracker agent in the suite.

scaletozero_datapath_test.go un-pends the wake-on-traffic datapath spec:
second always-on VM execs a ping into the suspended target's overlay IP
via the agent's vsock guest-exec API, asserting Suspended -> Resuming ->
Running.

New CI job "E2E Datapath (Kind, real agent)" (workflow_dispatch only):
Kind cluster with /dev/kvm passed through to the node, Firecracker binary
+ public quickstart kernel staged via docker cp (no imp-specific guest
kernel exists yet), runs the datapath-labeled spec.

UNVALIDATED end to end until this CI job actually runs green.
---
 .github/workflows/ci.yml              |  70 ++++++--
 internal/capability/probe.go          |   3 +-
 internal/capability/probe_test.go     |  19 +++
 test/e2e/e2e_suite_test.go            |  26 ++-
 test/e2e/kind-datapath.yaml           |  11 ++
 test/e2e/scaletozero_datapath_test.go | 226 +++++++++++++++++++++++---
 6 files changed, 307 insertions(+), 48 deletions(-)
 create mode 100644 test/e2e/kind-datapath.yaml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5e14bac..6ffef7c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -5,6 +5,7 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  workflow_dispatch:
 
 env:
   GO_VERSION: "1.26"
@@ -124,27 +125,68 @@ jobs:
         with:
           files: coverage.out
 
-  e2e:
-    name: E2E (Talos)
-    if: vars.E2E_RUNNER_LABEL != ''
-    runs-on: ${{ vars.E2E_RUNNER_LABEL }}
-    needs: test
+  e2e-datapath:
+    name: E2E Datapath (Kind, real agent)
+    # Boots a real Firecracker agent via nested KVM — slower and less proven than
+    # the Kind smoke job, so it's manual-dispatch only rather than on every push.
+    # See docs (e2e runner runbook) for why: proven on ubuntu-latest (run 29019535180),
+    # but the datapath spec itself has never executed end to end yet.
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    needs: [lint, build]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
 
-      - name: Create Talos E2E cluster
-        run: talosctl cluster create --provisioner docker --name imp-e2e
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: ${{ env.GO_VERSION }}
 
-      - name: Apply CRDs
-        run: kubectl apply -f config/crd/bases/
+      - name: Allow unprivileged access to /dev/kvm
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+          ls -l /dev/kvm
 
-      - name: Run E2E suite
-        run: go test -v -tags e2e ./test/e2e/...
+      - name: Create Kind cluster
+        uses: helm/kind-action@v1
+        with:
+          cluster_name: imp-e2e-datapath
+          config: test/e2e/kind-datapath.yaml
 
-      - name: Destroy Talos E2E cluster
-        if: always()
-        run: talosctl cluster destroy --name imp-e2e
+      - name: Install Helm CLI
+        uses: azure/setup-helm@v4
+
+      - name: Build local E2E images
+        run: |
+          docker build -f Dockerfile.operator -t local/imp-operator:e2e .
+          docker build -f Dockerfile.agent -t local/imp-agent:e2e .
+
+      - name: Load local E2E images into Kind
+        run: |
+          kind load docker-image local/imp-operator:e2e --name imp-e2e-datapath
+          kind load docker-image local/imp-agent:e2e --name imp-e2e-datapath
+
+      - name: Stage Firecracker binary + guest kernel on the Kind node
+        # No imp-specific guest kernel exists yet (planned "imp-guest-kernel" repo,
+        # not built) — every UNVALIDATED-marked path in this repo uses the public
+        # Firecracker quickstart kernel as a stopgap; same here.
+        run: |
+          node=$(kind get nodes --name imp-e2e-datapath | head -1)
+          fc_ver=v1.15.0
+          curl -fsSL "https://github.com/firecracker-microvm/firecracker/releases/download/${fc_ver}/firecracker-${fc_ver}-x86_64.tgz" -o /tmp/firecracker.tgz
+          tar -xzf /tmp/firecracker.tgz -C /tmp
+          docker cp "/tmp/release-${fc_ver}-x86_64/firecracker-${fc_ver}-x86_64" "${node}:/usr/local/bin/firecracker"
+          docker exec "${node}" chmod +x /usr/local/bin/firecracker
+          curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin" -o /tmp/vmlinux
+          docker exec "${node}" mkdir -p /var/lib/imp
+          docker cp /tmp/vmlinux "${node}:/var/lib/imp/vmlinux"
+
+      - name: Run datapath E2E suite
+        run: |
+          IMP_E2E_REAL_AGENT=true go test -tags e2e ./test/e2e/... -v -timeout 30m -ginkgo.label-filter="datapath"
 
   e2e-kind:
     name: E2E Smoke (Kind)
diff --git a/internal/capability/probe.go b/internal/capability/probe.go
index 8639636..2598350 100644
--- a/internal/capability/probe.go
+++ b/internal/capability/probe.go
@@ -29,9 +29,10 @@ func (r Result) OK() bool {
 func Check(kvmPath, binPath string) Result {
 	var r Result
 
-	if _, err := os.Stat(kvmPath); err != nil {
+	if f, err := os.OpenFile(kvmPath, os.O_RDWR, 0); err != nil { //nolint:gosec // G304: kvmPath is an operator-supplied device path, not user input
 		r.KVMError = fmt.Sprintf("%s not available: %v", kvmPath, err)
 	} else {
+		_ = f.Close()
 		r.KVMAvailable = true
 	}
 
diff --git a/internal/capability/probe_test.go b/internal/capability/probe_test.go
index 84c06c2..2440d6d 100644
--- a/internal/capability/probe_test.go
+++ b/internal/capability/probe_test.go
@@ -53,6 +53,25 @@ func TestCheck_MissingDevice(t *testing.T) {
 	}
 }
 
+func TestCheck_UnopenableDevice(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("running as root: permission bits don't block access")
+	}
+	dir := t.TempDir()
+	kvmPath := filepath.Join(dir, "kvm")
+	binPath := filepath.Join(dir, "firecracker")
+	if err := os.WriteFile(kvmPath, nil, 0o000); err != nil {
+		t.Fatalf("write unopenable kvm device stub: %v", err)
+	}
+	writeExecutable(t, binPath)
+
+	got := Check(kvmPath, binPath)
+
+	if got.KVMAvailable || got.KVMError == "" {
+		t.Errorf("expected KVM unavailable with an error, got %+v", got)
+	}
+}
+
 func TestCheck_MissingBinary(t *testing.T) {
 	dir := t.TempDir()
 	kvmPath := filepath.Join(dir, "kvm")
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index e941189..5c32af3 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -87,17 +87,29 @@ var _ = BeforeSuite(func() {
 	agentRepo := getenvOrDefault("IMP_E2E_AGENT_IMAGE_REPOSITORY", "local/imp-agent")
 	agentTag := getenvOrDefault("IMP_E2E_AGENT_IMAGE_TAG", "e2e")
 
-	impCmd := exec.Command("helm", "install", helmRelease, "charts/imp",
+	impArgs := []string{"install", helmRelease, "charts/imp",
 		"--namespace", namespace,
-		"--set", "operator.image.repository="+operatorRepo,
-		"--set", "operator.image.tag="+operatorTag,
-		"--set", "agent.image.repository="+agentRepo,
-		"--set", "agent.image.tag="+agentTag,
+		"--set", "operator.image.repository=" + operatorRepo,
+		"--set", "operator.image.tag=" + operatorTag,
+		"--set", "agent.image.repository=" + agentRepo,
+		"--set", "agent.image.tag=" + agentTag,
 		"--set", "agent.env.kernelPath=/var/lib/imp/vmlinux",
-		"--set-string", "agent.nodeSelector.imp\\.dev/no-agent=true",
 		"--set", "metrics.serviceMonitor.enabled=false",
 		"--set", "metrics.podMonitor.enabled=false",
-		"--wait", "--timeout", "10m")
+	}
+	if os.Getenv("IMP_E2E_REAL_AGENT") == "true" {
+		// Real KVM runner: let the agent schedule (no no-agent nodeSelector)
+		// and enable the scale-to-zero datapath under test.
+		impArgs = append(impArgs,
+			"--set", "agent.extraEnv[0].name=IMP_SCALE_TO_ZERO",
+			"--set", "agent.extraEnv[0].value=true",
+		)
+	} else {
+		impArgs = append(impArgs, "--set-string", "agent.nodeSelector.imp\\.dev/no-agent=true")
+	}
+	impArgs = append(impArgs, "--wait", "--timeout", "10m")
+
+	impCmd := exec.Command("helm", impArgs...)
 	_, err = utils.Run(impCmd)
 	Expect(err).NotTo(HaveOccurred(), "helm install imp failed")
 })
diff --git a/test/e2e/kind-datapath.yaml b/test/e2e/kind-datapath.yaml
new file mode 100644
index 0000000..7e0f1b2
--- /dev/null
+++ b/test/e2e/kind-datapath.yaml
@@ -0,0 +1,11 @@
+# Kind cluster configuration for the real-agent ScaleToZero datapath e2e.
+# Mounts host /dev/kvm into the control-plane node so a real Firecracker agent
+# can boot microVMs. See docs (e2e runner runbook) for the proven nested-KVM path.
+# Usage: kind create cluster --name imp-e2e-datapath --config test/e2e/kind-datapath.yaml
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraMounts:
+      - hostPath: /dev/kvm
+        containerPath: /dev/kvm
diff --git a/test/e2e/scaletozero_datapath_test.go b/test/e2e/scaletozero_datapath_test.go
index 43bcb71..d395954 100644
--- a/test/e2e/scaletozero_datapath_test.go
+++ b/test/e2e/scaletozero_datapath_test.go
@@ -20,40 +20,214 @@ limitations under the License.
 package e2e
 
 import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/syscode-labs/imp/test/utils"
 )
 
-// Wake-on-traffic datapath e2e — DELIBERATELY Pending (PIt).
+// Wake-on-traffic datapath e2e.
 //
 // This validates the one thing the host-only unit + envtest coverage cannot: that
 // the agent's AF_PACKET hook actually observes the first frame destined to a
 // TAP-less (suspended) ScaleToZero VM on imp's VXLAN overlay, and that a real
-// Firecracker suspend/resume round-trips. It is UNVALIDATED and cannot run yet —
-// see docs (runner runbook) for why the harness must change before it can.
-//
-// Prerequisites that do NOT exist in the current e2e harness (all are runbook
-// decisions, which is why this spec stays Pending until the runbook lands them):
+// Firecracker suspend/resume round-trips.
 //
-//  1. A KVM-capable node. BeforeSuite currently pins the agent to
-//     nodeSelector imp.dev/no-agent=true (agent off) and the Kind smoke runner has
-//     no /dev/kvm. A real Firecracker agent needs nested virt + the Firecracker
-//     Talos system extension + a guest kernel at agent.env.kernelPath.
-//  2. The agent enabled with IMP_SCALE_TO_ZERO=true. BeforeSuite must conditionally
-//     drop the no-agent selector and set the env when targeting the KVM runner
-//     (e.g. gate on an IMP_E2E_REAL_AGENT env var).
-//  3. A traffic source on the same ImpNetwork. OPEN QUESTION for first run: how does
-//     a frame reach a suspended VM's overlay IP? The realistic source is a second
-//     always-on VM on the same ImpNetwork pinging the suspended VM — but that pulls
-//     in guest exec / vsock. This is the crux to resolve when the runner is live.
-//
-// Intended flow once the harness supports it:
-//   - Create an ImpNetwork + a ScaleToZero ImpVM with a short idleTimeout (e.g. 15s).
-//   - Wait for Running, then (no traffic) wait for the agent's idle detector to
-//     auto-suspend it → status.phase == Suspended, VTEP retained.
-//   - Send a frame to the VM's overlay IP from the traffic source.
-//   - Assert the VM returns to Running (Suspended → Resuming → Running).
+// UNVALIDATED: this spec has never run against a live KVM node — no CI job wires
+// IMP_E2E_REAL_AGENT=true onto a KVM runner yet. Skips itself unless that env var
+// is set, so it stays inert everywhere else. Traffic source: a second always-on
+// VM on the same ImpNetwork execs `ping` into the suspended VM's overlay IP via
+// the agent's vsock guest-exec API (internal/agent/api/exec.go) — guest agent
+// injection defaults to enabled, so no extra fixture wiring needed for that part.
 var _ = Describe("Imp ScaleToZero datapath", Label("datapath"), func() {
-	PIt("wakes a suspended ScaleToZero VM when a frame arrives for its overlay IP", func() {
-		Skip("requires a KVM node + real Firecracker agent; see the e2e runner runbook")
+	const (
+		networkName = "e2e-sz-datapath-net"
+		className   = "e2e-sz-datapath"
+		pingerName  = "e2e-sz-datapath-pinger"
+		targetName  = "e2e-sz-datapath-target"
+	)
+
+	It("wakes a suspended ScaleToZero VM when a frame arrives for its overlay IP", func() {
+		if os.Getenv("IMP_E2E_REAL_AGENT") != "true" {
+			Skip("requires a KVM node + real Firecracker agent (IMP_E2E_REAL_AGENT=true); see the e2e runner runbook")
+		}
+
+		DeferCleanup(func() {
+			_, _ = utils.Run(exec.Command("kubectl", "delete", "impvm", targetName, "-n", "default", "--ignore-not-found"))
+			_, _ = utils.Run(exec.Command("kubectl", "delete", "impvm", pingerName, "-n", "default", "--ignore-not-found"))
+			_, _ = utils.Run(exec.Command("kubectl", "delete", "impnetwork", networkName, "-n", "default", "--ignore-not-found"))
+			_, _ = utils.Run(exec.Command("kubectl", "delete", "impvmclass", className, "--ignore-not-found"))
+		})
+
+		By("creating the ImpVMClass")
+		classManifest := fmt.Sprintf(`
+apiVersion: imp.dev/v1alpha1
+kind: ImpVMClass
+metadata:
+  name: %s
+spec:
+  vcpu: 1
+  memoryMiB: 256
+  diskGiB: 1
+`, className)
+		applyClass := exec.Command("kubectl", "apply", "-f", "-")
+		applyClass.Stdin = strings.NewReader(classManifest)
+		_, err := utils.Run(applyClass)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("creating the ImpNetwork")
+		netManifest := fmt.Sprintf(`
+apiVersion: imp.dev/v1alpha1
+kind: ImpNetwork
+metadata:
+  name: %s
+  namespace: default
+spec:
+  subnet: 10.45.0.0/24
+`, networkName)
+		applyNet := exec.Command("kubectl", "apply", "-f", "-")
+		applyNet.Stdin = strings.NewReader(netManifest)
+		_, err = utils.Run(applyNet)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("creating the always-on pinger VM")
+		pingerManifest := fmt.Sprintf(`
+apiVersion: imp.dev/v1alpha1
+kind: ImpVM
+metadata:
+  name: %s
+  namespace: default
+spec:
+  classRef:
+    name: %s
+  networkRef:
+    name: %s
+  image: ghcr.io/syscode-labs/test:latest
+`, pingerName, className, networkName)
+		applyPinger := exec.Command("kubectl", "apply", "-f", "-")
+		applyPinger.Stdin = strings.NewReader(pingerManifest)
+		_, err = utils.Run(applyPinger)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("creating the ScaleToZero target VM with a short idleTimeout")
+		targetManifest := fmt.Sprintf(`
+apiVersion: imp.dev/v1alpha1
+kind: ImpVM
+metadata:
+  name: %s
+  namespace: default
+spec:
+  classRef:
+    name: %s
+  networkRef:
+    name: %s
+  image: ghcr.io/syscode-labs/test:latest
+  desiredState: ScaleToZero
+  idleTimeout: 15s
+`, targetName, className, networkName)
+		applyTarget := exec.Command("kubectl", "apply", "-f", "-")
+		applyTarget.Stdin = strings.NewReader(targetManifest)
+		_, err = utils.Run(applyTarget)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("waiting for both VMs to reach Running")
+		Eventually(func(g Gomega) {
+			phase, _ := vmPhaseAndIP(g, pingerName)
+			g.Expect(phase).To(Equal("Running"))
+		}, "5m", "5s").Should(Succeed())
+
+		var targetIP string
+		Eventually(func(g Gomega) {
+			phase, ip := vmPhaseAndIP(g, targetName)
+			g.Expect(phase).To(Equal("Running"))
+			g.Expect(ip).NotTo(BeEmpty())
+			targetIP = ip
+		}, "5m", "5s").Should(Succeed())
+
+		By("waiting for the target to auto-suspend after going idle")
+		Eventually(func(g Gomega) {
+			phase, _ := vmPhaseAndIP(g, targetName)
+			g.Expect(phase).To(Equal("Suspended"))
+		}, "2m", "5s").Should(Succeed())
+
+		By("finding the agent pod colocated with the pinger VM")
+		pingerNode := vmNodeName(pingerName)
+		Expect(pingerNode).NotTo(BeEmpty())
+		agentPod := agentPodOnNode(pingerNode)
+		Expect(agentPod).NotTo(BeEmpty())
+
+		By("port-forwarding to the agent's guest-exec API")
+		pf := exec.Command("kubectl", "port-forward", "pod/"+agentPod, "19091:9091", "-n", namespace)
+		Expect(pf.Start()).To(Succeed())
+		DeferCleanup(func() {
+			if pf.Process != nil {
+				_ = pf.Process.Kill()
+			}
+		})
+		Eventually(func(g Gomega) {
+			conn, dialErr := net.DialTimeout("tcp", "localhost:19091", 2*time.Second)
+			g.Expect(dialErr).NotTo(HaveOccurred())
+			_ = conn.Close()
+		}, "30s", "1s").Should(Succeed())
+
+		By("pinging the suspended VM's overlay IP from the pinger VM's guest agent")
+		body, err := json.Marshal(map[string][]string{"command": {"ping", "-c", "3", "-W", "2", targetIP}})
+		Expect(err).NotTo(HaveOccurred())
+		resp, err := http.Post( //nolint:noctx
+			fmt.Sprintf("http://localhost:19091/v1/exec/default/%s", pingerName),
+			"application/json", strings.NewReader(string(body)))
+		Expect(err).NotTo(HaveOccurred())
+		defer resp.Body.Close() //nolint:errcheck
+		Expect(resp.StatusCode).To(Equal(http.StatusOK))
+
+		By("asserting the target wakes: Suspended -> Resuming -> Running")
+		Eventually(func(g Gomega) {
+			phase, _ := vmPhaseAndIP(g, targetName)
+			g.Expect(phase).To(Equal("Running"))
+		}, "2m", "5s").Should(Succeed())
 	})
 })
+
+func vmPhaseAndIP(g Gomega, name string) (string, string) {
+	out, err := utils.Run(exec.Command("kubectl", "get", "impvm", name, "-n", "default",
+		"-o", "jsonpath={.status.phase} {.status.ip}"))
+	g.Expect(err).NotTo(HaveOccurred())
+	parts := strings.Fields(strings.TrimSpace(out))
+	switch len(parts) {
+	case 0:
+		return "", ""
+	case 1:
+		return parts[0], ""
+	default:
+		return parts[0], parts[1]
+	}
+}
+
+func vmNodeName(name string) string {
+	out, err := utils.Run(exec.Command("kubectl", "get", "impvm", name, "-n", "default",
+		"-o", "jsonpath={.status.nodeName}"))
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(out)
+}
+
+func agentPodOnNode(nodeName string) string {
+	out, err := utils.Run(exec.Command("kubectl", "get", "pods", "-n", namespace,
+		"-l", "app.kubernetes.io/component=agent",
+		"--field-selector", "spec.nodeName="+nodeName,
+		"-o", "jsonpath={.items[0].metadata.name}"))
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(out)
+}

From ef7f2b3d9b7e663e2cd77e3549b678a7c9ed58a5 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Tue, 28 Jul 2026 22:49:42 +0100
Subject: [PATCH 02/19] fix(test): use --set-string for agent.extraEnv value in
 real-agent e2e

Helm --set types agent.extraEnv[0].value=true as bool via server-side
apply; DaemonSet env values must be strings. Found by the first real
CI run of the datapath e2e job (helm install imp failed:
"expected string, got &value.valueUnstructured{Value:true}").
---
 test/e2e/e2e_suite_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index 5c32af3..fdf9ba0 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -102,7 +102,7 @@ var _ = BeforeSuite(func() {
 		// and enable the scale-to-zero datapath under test.
 		impArgs = append(impArgs,
 			"--set", "agent.extraEnv[0].name=IMP_SCALE_TO_ZERO",
-			"--set", "agent.extraEnv[0].value=true",
+			"--set-string", "agent.extraEnv[0].value=true",
 		)
 	} else {
 		impArgs = append(impArgs, "--set-string", "agent.nodeSelector.imp\\.dev/no-agent=true")

From fd31fda68b5a0ef297fb5dff405aa284c831d45a Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Tue, 28 Jul 2026 23:05:21 +0100
Subject: [PATCH 03/19] ci(datapath): dump cluster diagnostics on e2e failure

First real run got the pinger ImpVM stuck at phase=Pending for the full
5m timeout with no visibility into why (operator/agent logs, events).
Cluster is deleted immediately after the job, so add a failure-only dump
step before teardown.
---
 .github/workflows/ci.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6ffef7c..03612a0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -188,6 +188,20 @@ jobs:
         run: |
           IMP_E2E_REAL_AGENT=true go test -tags e2e ./test/e2e/... -v -timeout 30m -ginkgo.label-filter="datapath"
 
+      - name: Dump cluster diagnostics on failure
+        if: failure()
+        run: |
+          kubectl get pods -A -o wide || true
+          kubectl get impvm,impnetwork,impvmclass -A -o wide || true
+          kubectl describe pods -n imp-system || true
+          kubectl describe impvm -n default || true
+          echo "--- operator logs ---"
+          kubectl logs -n imp-system -l app.kubernetes.io/component=operator --tail=200 || true
+          echo "--- agent logs ---"
+          kubectl logs -n imp-system -l app.kubernetes.io/component=agent --tail=200 || true
+          echo "--- events ---"
+          kubectl get events -A --sort-by=.lastTimestamp || true
+
   e2e-kind:
     name: E2E Smoke (Kind)
     runs-on: ubuntu-latest

From 15b8b2c16cb0a3e01ddc1e0adf2e8de03a389f39 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Tue, 28 Jul 2026 23:22:33 +0100
Subject: [PATCH 04/19] fix(test): label Kind node imp/enabled=true for
 datapath scheduling

impvm_scheduler.go's schedule() filters candidate nodes by node label
imp/enabled=true before anything else runs. The datapath test never set
it (nor removed the Kind control-plane taint), so both VMs sat
Unschedulable ("No eligible node with available capacity") for the
entire 5m Eventually window. The existing "Scheduling filter" smoke
test already does this same node prep; datapath test now mirrors it.
---
 test/e2e/scaletozero_datapath_test.go | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/test/e2e/scaletozero_datapath_test.go b/test/e2e/scaletozero_datapath_test.go
index d395954..121ac84 100644
--- a/test/e2e/scaletozero_datapath_test.go
+++ b/test/e2e/scaletozero_datapath_test.go
@@ -68,6 +68,21 @@ var _ = Describe("Imp ScaleToZero datapath", Label("datapath"), func() {
 			_, _ = utils.Run(exec.Command("kubectl", "delete", "impvmclass", className, "--ignore-not-found"))
 		})
 
+		By("getting the Kind node name")
+		nodeOut, err := utils.Run(exec.Command("kubectl", "get", "nodes", "-o", "jsonpath={.items[0].metadata.name}"))
+		Expect(err).NotTo(HaveOccurred())
+		nodeName := strings.TrimSpace(nodeOut)
+		Expect(nodeName).NotTo(BeEmpty())
+
+		By("removing control-plane taint and labeling the node imp/enabled=true")
+		_, _ = utils.Run(exec.Command("kubectl", "taint", "nodes", nodeName,
+			"node-role.kubernetes.io/control-plane:NoSchedule-"))
+		_, err = utils.Run(exec.Command("kubectl", "label", "node", nodeName, "imp/enabled=true", "--overwrite"))
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(func() {
+			_, _ = utils.Run(exec.Command("kubectl", "label", "node", nodeName, "imp/enabled-"))
+		})
+
 		By("creating the ImpVMClass")
 		classManifest := fmt.Sprintf(`
 apiVersion: imp.dev/v1alpha1
@@ -81,7 +96,7 @@ spec:
 `, className)
 		applyClass := exec.Command("kubectl", "apply", "-f", "-")
 		applyClass.Stdin = strings.NewReader(classManifest)
-		_, err := utils.Run(applyClass)
+		_, err = utils.Run(applyClass)
 		Expect(err).NotTo(HaveOccurred())
 
 		By("creating the ImpNetwork")

From b14670db6bffd4bb07f34c3c4e9a3c3eb64dbcd8 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Tue, 28 Jul 2026 23:41:42 +0100
Subject: [PATCH 05/19] ci(test): dump agent/operator logs in AfterSuite before
 teardown
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both pinger and target VMs reached Starting then their Firecracker
process exited ("VM process exited and was marked Failed") on the first
real-agent run, but the CI-level diagnostics step ran after go test
already returned, by which point Ginkgo's own AfterSuite had already
helm-uninstalled the chart and killed the agent/operator pods — no logs
captured. Dump agent+operator logs from inside AfterSuite instead, before
the uninstall, whenever IMP_E2E_REAL_AGENT=true.
---
 test/e2e/e2e_suite_test.go | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index fdf9ba0..bd7bc79 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -115,6 +115,19 @@ var _ = BeforeSuite(func() {
 })
 
 var _ = AfterSuite(func() {
+	if os.Getenv("IMP_E2E_REAL_AGENT") == "true" {
+		By("dumping agent + operator logs before teardown")
+		agentLogs := exec.Command("kubectl", "logs", "-n", namespace,
+			"-l", "app.kubernetes.io/component=agent", "--tail=500", "--prefix")
+		out, _ := utils.Run(agentLogs)
+		GinkgoWriter.Println("--- agent logs ---\n" + out)
+
+		operatorLogs := exec.Command("kubectl", "logs", "-n", namespace,
+			"-l", "app.kubernetes.io/component=operator", "--tail=500", "--prefix")
+		out, _ = utils.Run(operatorLogs)
+		GinkgoWriter.Println("--- operator logs ---\n" + out)
+	}
+
 	By("uninstalling imp chart")
 	unimpCmd := exec.Command("helm", "uninstall", helmRelease, "--namespace", namespace)
 	_, _ = utils.Run(unimpCmd)

From 2882e0728de1b82326d1730b1b45edba2a955c1b Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 00:01:41 +0100
Subject: [PATCH 06/19] fix(test): use plain stdout for AfterSuite log dump,
 not GinkgoWriter

Ginkgo's default reporter only prints a suite node's GinkgoWriter buffer
when that node fails. AfterSuite succeeded, so the agent/operator log
dump from the previous commit never showed up in CI output even though
it ran. fmt.Println bypasses that suppression.
---
 test/e2e/e2e_suite_test.go | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index bd7bc79..f352aa3 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -20,6 +20,7 @@ limitations under the License.
 package e2e
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"testing"
@@ -120,13 +121,11 @@ var _ = AfterSuite(func() {
 		agentLogs := exec.Command("kubectl", "logs", "-n", namespace,
 			"-l", "app.kubernetes.io/component=agent", "--tail=500", "--prefix")
 		out, _ := utils.Run(agentLogs)
-		GinkgoWriter.Println("--- agent logs ---\n" + out)
-
+		fmt.Println("--- agent logs ---\n" + out)
 		operatorLogs := exec.Command("kubectl", "logs", "-n", namespace,
 			"-l", "app.kubernetes.io/component=operator", "--tail=500", "--prefix")
 		out, _ = utils.Run(operatorLogs)
-		GinkgoWriter.Println("--- operator logs ---\n" + out)
-	}
+		fmt.Println("--- operator logs ---\n" + out)	}
 
 	By("uninstalling imp chart")
 	unimpCmd := exec.Command("helm", "uninstall", helmRelease, "--namespace", namespace)

From 5080399e01e44acfe0716eb3673c83e8d0ea9799 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 00:21:20 +0100
Subject: [PATCH 07/19] fix(test): use real pullable image for datapath VMs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Agent logs (now visible after the previous fix) show the actual root
cause of every prior failure: ghcr.io/syscode-labs/test:latest is a
placeholder used across e2e fixtures that was never actually pulled
before, because every prior e2e run disabled the agent. With a real
agent it fails outright: "DENIED: requested access to the resource is
denied". Switch to public docker.io/library/busybox:latest — imp
synthesizes /sbin/init from the image's CMD/ENTRYPOINT itself
(internal/agent/rootfs/init.go), so any real image with a CMD works;
busybox's default CMD (sh) blocks on stdin, keeping PID1 alive, and it
ships a ping applet the pinger VM needs anyway.
---
 test/e2e/scaletozero_datapath_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/scaletozero_datapath_test.go b/test/e2e/scaletozero_datapath_test.go
index 121ac84..beb2cdd 100644
--- a/test/e2e/scaletozero_datapath_test.go
+++ b/test/e2e/scaletozero_datapath_test.go
@@ -126,7 +126,7 @@ spec:
     name: %s
   networkRef:
     name: %s
-  image: ghcr.io/syscode-labs/test:latest
+  image: docker.io/library/busybox:latest
 `, pingerName, className, networkName)
 		applyPinger := exec.Command("kubectl", "apply", "-f", "-")
 		applyPinger.Stdin = strings.NewReader(pingerManifest)
@@ -145,7 +145,7 @@ spec:
     name: %s
   networkRef:
     name: %s
-  image: ghcr.io/syscode-labs/test:latest
+  image: docker.io/library/busybox:latest
   desiredState: ScaleToZero
   idleTimeout: 15s
 `, targetName, className, networkName)

From bdc1f3266d409ac30e734909c20d347f1c456520 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 00:41:49 +0100
Subject: [PATCH 08/19] fix(chart): mount guest kernel hostPath into agent
 DaemonSet
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Agent logs from the real-agent e2e run show the actual blocker: "start
machine: failed to stat kernel image path, \"/var/lib/imp/vmlinux\":
stat /var/lib/imp/vmlinux: no such file or directory". FC_KERNEL was
always set as an env var, but unlike FC_BIN (firecracker-bin hostPath
mount, fixed for #33) there was never a matching hostPath volume for
the kernel — the path only ever existed on the node, never inside the
agent container. Every real deployment would hit this; it just never
surfaced because the agent has never run for real until this e2e run.
Mirrors the firecracker-bin mount exactly. Chart unit tests pass
(`helm unittest charts/imp`, 51/51); added a case for the new mount.
---
 charts/imp/templates/agent/daemonset.yaml  |  7 +++++++
 charts/imp/tests/agent-daemonset_test.yaml | 19 +++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/charts/imp/templates/agent/daemonset.yaml b/charts/imp/templates/agent/daemonset.yaml
index 714da80..a2f51b8 100644
--- a/charts/imp/templates/agent/daemonset.yaml
+++ b/charts/imp/templates/agent/daemonset.yaml
@@ -68,6 +68,9 @@ spec:
         - name: firecracker-bin
           mountPath: {{ .Values.agent.env.fcBinPath | quote }}
           readOnly: true
+        - name: guest-kernel
+          mountPath: {{ .Values.agent.env.kernelPath | quote }}
+          readOnly: true
         - name: socket-dir
           mountPath: /run/imp/sockets
         - name: image-cache
@@ -81,6 +84,10 @@ spec:
         hostPath:
           path: {{ .Values.agent.env.fcBinPath | quote }}
           type: File
+      - name: guest-kernel
+        hostPath:
+          path: {{ .Values.agent.env.kernelPath | quote }}
+          type: File
       - name: socket-dir
         {{- if .Values.agent.hostPaths.socketDir.enabled }}
         hostPath:
diff --git a/charts/imp/tests/agent-daemonset_test.yaml b/charts/imp/tests/agent-daemonset_test.yaml
index 378960e..a99946c 100644
--- a/charts/imp/tests/agent-daemonset_test.yaml
+++ b/charts/imp/tests/agent-daemonset_test.yaml
@@ -50,6 +50,25 @@ tests:
               path: /usr/local/bin/firecracker
               type: File
 
+  - it: mounts the guest kernel hostPath at kernelPath
+    template: templates/agent/daemonset.yaml
+    set:
+      agent.env.kernelPath: /var/lib/imp/vmlinux
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: guest-kernel
+            mountPath: /var/lib/imp/vmlinux
+            readOnly: true
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: guest-kernel
+            hostPath:
+              path: /var/lib/imp/vmlinux
+              type: File
+
   - it: uses emptyDir for socketDir when hostPaths disabled
     template: templates/agent/daemonset.yaml
     set:

From 630f3d321e08258cdeb9511d76582e20910c0503 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 00:59:26 +0100
Subject: [PATCH 09/19] fix(test): use nginx:alpine, not busybox, for datapath
 VM image
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

busybox's default CMD is a bare "sh" — with no interactive console
attached it hits EOF on stdin almost immediately and exits, so the
guest VM finished (ephemeral lifecycle default) within seconds of
boot: agent logs show finishFailed/finishSucceeded firing right after
Running, clearing the driver's process-map entry, so the later idle
suspend's Snapshot() found nothing ("VM ... is not running on this
node") and retried in a loop until timeout. nginx:alpine's default CMD
(nginx -g "daemon off;") blocks forever without needing stdin, and its
Alpine base still ships busybox ping for the guest-exec step.
---
 test/e2e/scaletozero_datapath_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/scaletozero_datapath_test.go b/test/e2e/scaletozero_datapath_test.go
index beb2cdd..44305ae 100644
--- a/test/e2e/scaletozero_datapath_test.go
+++ b/test/e2e/scaletozero_datapath_test.go
@@ -126,7 +126,7 @@ spec:
     name: %s
   networkRef:
     name: %s
-  image: docker.io/library/busybox:latest
+  image: docker.io/library/nginx:alpine
 `, pingerName, className, networkName)
 		applyPinger := exec.Command("kubectl", "apply", "-f", "-")
 		applyPinger.Stdin = strings.NewReader(pingerManifest)
@@ -145,7 +145,7 @@ spec:
     name: %s
   networkRef:
     name: %s
-  image: docker.io/library/busybox:latest
+  image: docker.io/library/nginx:alpine
   desiredState: ScaleToZero
   idleTimeout: 15s
 `, targetName, className, networkName)

From 9f8a6eb1025bb2d955ad7a01c1fbe216fa9064c4 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 01:17:26 +0100
Subject: [PATCH 10/19] ci(test): capture pod restart count +
 previous-container agent logs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Latest run: idle detection correctly transitioned target to Suspending,
but Snapshot() immediately failed with "VM ... is not running on this
node" and never recovered. Root cause: the imp-agent pod's container
actually restarted right at that moment (kubectl events show a second
Created/Started for the same pod, timestamped exactly when the errors
begin), wiping the driver's in-memory procs map — a real reliability
gap, not a test fixture issue. kubectl logs (current container) can't
show why, since the crash was in the container that got replaced. Add
pod status (RESTARTS column) + --previous logs to the AfterSuite dump
so the next run captures the actual crash/exit reason.
---
 test/e2e/e2e_suite_test.go | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index f352aa3..9a5720d 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -117,15 +117,26 @@ var _ = BeforeSuite(func() {
 
 var _ = AfterSuite(func() {
 	if os.Getenv("IMP_E2E_REAL_AGENT") == "true" {
-		By("dumping agent + operator logs before teardown")
+		By("dumping pod status + agent/operator logs before teardown")
+		podStatus := exec.Command("kubectl", "get", "pods", "-n", namespace, "-o", "wide")
+		out, _ := utils.Run(podStatus)
+		fmt.Println("--- pod status (check RESTARTS) ---\n" + out)
+
+		agentLogsPrev := exec.Command("kubectl", "logs", "-n", namespace,
+			"-l", "app.kubernetes.io/component=agent", "--tail=500", "--prefix", "--previous")
+		out, _ = utils.Run(agentLogsPrev)
+		fmt.Println("--- agent logs (previous container, if it restarted) ---\n" + out)
+
 		agentLogs := exec.Command("kubectl", "logs", "-n", namespace,
 			"-l", "app.kubernetes.io/component=agent", "--tail=500", "--prefix")
-		out, _ := utils.Run(agentLogs)
+		out, _ = utils.Run(agentLogs)
 		fmt.Println("--- agent logs ---\n" + out)
+
 		operatorLogs := exec.Command("kubectl", "logs", "-n", namespace,
 			"-l", "app.kubernetes.io/component=operator", "--tail=500", "--prefix")
 		out, _ = utils.Run(operatorLogs)
-		fmt.Println("--- operator logs ---\n" + out)	}
+		fmt.Println("--- operator logs ---\n" + out)
+	}
 
 	By("uninstalling imp chart")
 	unimpCmd := exec.Command("helm", "uninstall", helmRelease, "--namespace", namespace)

From a07e150f0e4ff1b4fd24c1e096090f090d7bba30 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 01:34:35 +0100
Subject: [PATCH 11/19] fix(test): raise agent memory limit for real-agent e2e
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Found the actual restart cause: previous-container agent logs end
abruptly mid-stream with no panic/fatal/shutdown line — the signature
of an external SIGKILL, not a Go crash. Firecracker runs as a child
process of the agent, so guest RAM counts against the agent
container's own cgroup. Chart default agent.resources.limits.memory
is 128Mi, assuming no hosted VMs; two 256MiB-class VMs alone blow
past that, so kubelet OOM-kills the container almost immediately
after a real microVM boots, wiping the driver's in-memory state.
Override to 1Gi for the real-agent e2e path. (The 128Mi chart default
is itself a real footgun for any actual Firecracker deployment — worth
its own follow-up, not fixed here to keep this change scoped.)
---
 test/e2e/e2e_suite_test.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
index 9a5720d..297cb9d 100644
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -104,6 +104,11 @@ var _ = BeforeSuite(func() {
 		impArgs = append(impArgs,
 			"--set", "agent.extraEnv[0].name=IMP_SCALE_TO_ZERO",
 			"--set-string", "agent.extraEnv[0].value=true",
+			// Firecracker runs as a child process of the agent, so guest RAM counts
+			// against the agent container's own memory cgroup. The chart default
+			// (128Mi) assumes no hosted VMs and OOM-kills almost immediately once a
+			// real microVM boots — silently, since a SIGKILL leaves no panic/log line.
+			"--set", "agent.resources.limits.memory=1Gi",
 		)
 	} else {
 		impArgs = append(impArgs, "--set-string", "agent.nodeSelector.imp\\.dev/no-agent=true")

From 05cbf6f2e4d0e1aba559504540a147ef4c2dad0c Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 01:51:59 +0100
Subject: [PATCH 12/19] fix(test): read spec.nodeName, not status.nodeName, in
 vmNodeName
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Suspend/resume itself now works end to end (OOM fix held) — the run
got past suspend and failed on the next step: vmNodeName() returned
empty. status.NodeName exists on the ImpVMStatus type with a doc
comment claiming it's "the node where the VM is running", but nothing
in the codebase actually writes it — only spec.nodeName is set, by the
scheduler (impvm_controller.go). My own test bug, not a product bug.
---
 test/e2e/scaletozero_datapath_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/e2e/scaletozero_datapath_test.go b/test/e2e/scaletozero_datapath_test.go
index 44305ae..eb2cdaf 100644
--- a/test/e2e/scaletozero_datapath_test.go
+++ b/test/e2e/scaletozero_datapath_test.go
@@ -228,8 +228,10 @@ func vmPhaseAndIP(g Gomega, name string) (string, string) {
 }
 
 func vmNodeName(name string) string {
+	// status.nodeName exists on the CRD but nothing in the codebase ever writes it
+	// (only spec.nodeName is set, by the scheduler) — read spec instead.
 	out, err := utils.Run(exec.Command("kubectl", "get", "impvm", name, "-n", "default",
-		"-o", "jsonpath={.status.nodeName}"))
+		"-o", "jsonpath={.spec.nodeName}"))
 	if err != nil {
 		return ""
 	}

From 184ba8ad182a468d2c9ab8e992088c1e2f74c660 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 14:16:23 +0100
Subject: [PATCH 13/19] fix(test): warm the pinger's ARP cache before the
 target suspends
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Diagnosed the actual open research question: the wake ping never
resolves because the target's TAP is torn down on suspend
(handleSuspending frees memory; only the VTEP is kept, for cross-node
routing). The AF_PACKET wake hook is also bound to ETH_P_IP only, not
ARP — so even if it were listening for ARP it wouldn't see it. A
same-node sender with no prior ARP entry can never emit the IP frame
in the first place; there's nothing to catch. This matches the real
intended use case: a client with an already-warm ARP entry (from
earlier traffic) sends the IP packet directly on the next attempt, no
fresh ARP round-trip needed. Ping the target once while still Running
to warm the pinger's neighbor cache, matching that scenario, before
letting it idle-suspend.

Also refactors the raw net/http exec call into execInVM(), decoding
the NDJSON response to an exit code + combined output, so both the
warm-up ping (asserted) and the wake ping (logged for diagnostics) can
inspect actual ping results, not just the HTTP transport status.
---
 test/e2e/scaletozero_datapath_test.go | 79 ++++++++++++++++++++++-----
 1 file changed, 66 insertions(+), 13 deletions(-)

diff --git a/test/e2e/scaletozero_datapath_test.go b/test/e2e/scaletozero_datapath_test.go
index eb2cdaf..4513635 100644
--- a/test/e2e/scaletozero_datapath_test.go
+++ b/test/e2e/scaletozero_datapath_test.go
@@ -168,12 +168,6 @@ spec:
 			targetIP = ip
 		}, "5m", "5s").Should(Succeed())
 
-		By("waiting for the target to auto-suspend after going idle")
-		Eventually(func(g Gomega) {
-			phase, _ := vmPhaseAndIP(g, targetName)
-			g.Expect(phase).To(Equal("Suspended"))
-		}, "2m", "5s").Should(Succeed())
-
 		By("finding the agent pod colocated with the pinger VM")
 		pingerNode := vmNodeName(pingerName)
 		Expect(pingerNode).NotTo(BeEmpty())
@@ -194,15 +188,29 @@ spec:
 			_ = conn.Close()
 		}, "30s", "1s").Should(Succeed())
 
-		By("pinging the suspended VM's overlay IP from the pinger VM's guest agent")
-		body, err := json.Marshal(map[string][]string{"command": {"ping", "-c", "3", "-W", "2", targetIP}})
+		By("warming the pinger's ARP cache for the target before it suspends")
+		// The target's TAP is torn down on suspend (frees memory) — only the VTEP
+		// is kept, so cross-node overlay routing still works, but the pinger's own
+		// ARP resolution for the target's IP can only happen while the target's TAP
+		// exists to answer it. A sender with a warm ARP entry sends the IP frame
+		// directly on wake with no fresh ARP round-trip needed; a cold sender never
+		// gets an IP frame onto the wire in the first place, so the AF_PACKET wake
+		// hook (bound to ETH_P_IP, not ARP) never fires. This mirrors the real
+		// intended use case: traffic to an already-known, now-idle destination.
+		warmExit, warmOut, err := execInVM(pingerName, "ping", "-c", "1", "-W", "5", targetIP)
 		Expect(err).NotTo(HaveOccurred())
-		resp, err := http.Post( //nolint:noctx
-			fmt.Sprintf("http://localhost:19091/v1/exec/default/%s", pingerName),
-			"application/json", strings.NewReader(string(body)))
+		Expect(warmExit).To(Equal(int32(0)), "ARP warm-up ping failed while target was still Running:\n"+warmOut)
+
+		By("waiting for the target to auto-suspend after going idle")
+		Eventually(func(g Gomega) {
+			phase, _ := vmPhaseAndIP(g, targetName)
+			g.Expect(phase).To(Equal("Suspended"))
+		}, "2m", "5s").Should(Succeed())
+
+		By("pinging the suspended VM's overlay IP from the pinger VM's guest agent")
+		wakeExit, wakeOut, err := execInVM(pingerName, "ping", "-c", "3", "-W", "2", targetIP)
 		Expect(err).NotTo(HaveOccurred())
-		defer resp.Body.Close() //nolint:errcheck
-		Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		GinkgoWriter.Printf("wake ping exit=%d output:\n%s\n", wakeExit, wakeOut)
 
 		By("asserting the target wakes: Suspended -> Resuming -> Running")
 		Eventually(func(g Gomega) {
@@ -212,6 +220,51 @@ spec:
 	})
 })
 
+// execInVM runs command inside vmName's guest via the agent's vsock guest-exec
+// API (assumes a port-forward to localhost:19091 is already active) and returns
+// the process exit code plus the combined stdout+stderr NDJSON stream decoded to
+// plain text.
+func execInVM(vmName string, command ...string) (int32, string, error) {
+	body, err := json.Marshal(map[string][]string{"command": command})
+	if err != nil {
+		return 0, "", err
+	}
+	resp, err := http.Post( //nolint:noctx
+		fmt.Sprintf("http://localhost:19091/v1/exec/default/%s", vmName),
+		"application/json", strings.NewReader(string(body)))
+	if err != nil {
+		return 0, "", err
+	}
+	defer resp.Body.Close() //nolint:errcheck
+	if resp.StatusCode != http.StatusOK {
+		return 0, "", fmt.Errorf("exec %s: unexpected status %d", vmName, resp.StatusCode)
+	}
+
+	var out strings.Builder
+	var exitCode int32
+	dec := json.NewDecoder(resp.Body)
+	for dec.More() {
+		var line struct {
+			Stream string `json:"stream"`
+			Line   string `json:"line,omitempty"`
+			Code   *int32 `json:"code,omitempty"`
+		}
+		if err := dec.Decode(&line); err != nil {
+			return 0, out.String(), fmt.Errorf("decode exec response: %w", err)
+		}
+		switch line.Stream {
+		case "exit":
+			if line.Code != nil {
+				exitCode = *line.Code
+			}
+		default:
+			out.WriteString(line.Line)
+			out.WriteString("\n")
+		}
+	}
+	return exitCode, out.String(), nil
+}
+
 func vmPhaseAndIP(g Gomega, name string) (string, string) {
 	out, err := utils.Run(exec.Command("kubectl", "get", "impvm", name, "-n", "default",
 		"-o", "jsonpath={.status.phase} {.status.ip}"))

From abfbdeca5e8b27f898ce759de8dacd04a1edcf98 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 14:34:26 +0100
Subject: [PATCH 14/19] fix(test): retry ARP warm-up ping instead of
 single-shot
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Failed instantly (11ms) after both VMs hit Running: status.phase
reflects the host starting Firecracker, not that the guest kernel has
finished booting and brought its network up. The single-shot warm-up
ping fired before the guest was ready. Wrap in Eventually (up to 1m,
3s interval) instead — the original code never hit this race because
it always waited through the full 15s+ idle-suspend window first, by
which point the guest was long since up.
---
 test/e2e/scaletozero_datapath_test.go | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/test/e2e/scaletozero_datapath_test.go b/test/e2e/scaletozero_datapath_test.go
index 4513635..c1dbab7 100644
--- a/test/e2e/scaletozero_datapath_test.go
+++ b/test/e2e/scaletozero_datapath_test.go
@@ -197,9 +197,18 @@ spec:
 		// gets an IP frame onto the wire in the first place, so the AF_PACKET wake
 		// hook (bound to ETH_P_IP, not ARP) never fires. This mirrors the real
 		// intended use case: traffic to an already-known, now-idle destination.
-		warmExit, warmOut, err := execInVM(pingerName, "ping", "-c", "1", "-W", "5", targetIP)
-		Expect(err).NotTo(HaveOccurred())
-		Expect(warmExit).To(Equal(int32(0)), "ARP warm-up ping failed while target was still Running:\n"+warmOut)
+		// Retry: status.phase==Running reflects the host starting Firecracker, not
+		// that the guest kernel has finished booting and brought its network up —
+		// an immediate single-shot ping can race a guest that isn't ready yet.
+		var warmOut string
+		Eventually(func(g Gomega) {
+			warmExit, out, execErr := execInVM(pingerName, "ping", "-c", "1", "-W", "5", targetIP)
+			warmOut = out
+			g.Expect(execErr).NotTo(HaveOccurred())
+			g.Expect(warmExit).To(Equal(int32(0)))
+		}, "1m", "3s").Should(Succeed(), func() string {
+			return "ARP warm-up ping never succeeded while target was still Running:\n" + warmOut
+		})
 
 		By("waiting for the target to auto-suspend after going idle")
 		Eventually(func(g Gomega) {

From 215fca78fc887e0c7a4f1c15b65d5acdf49cd92f Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 14:51:01 +0100
Subject: [PATCH 15/19] fix(test): use absolute /bin/ping path in guest exec
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Warm-up ping failed consistently for the full 60s retry window with
exit=1 and empty stdout/stderr — not a timeout pattern. Root cause:
internal/guest/server.go's Exec RPC calls exec.CommandContext(command[0], ...)
directly, which Go resolves via the guest-agent process's own PATH env
var. The guest-agent is launched by /.imp/init as "/.imp/guest-agent &"
(absolute path, no PATH lookup needed) from a PID1 shell that never
had PATH set (WithEnv only writes /.imp/env when explicit env vars are
configured, which this test doesn't set) — so "ping" by bare name can
never resolve, hitting the *exec.Error branch that sets exitCode=1
with no output before the process ever starts. Use /bin/ping (present
via alpine's busybox) instead of relying on PATH.
---
 test/e2e/scaletozero_datapath_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/e2e/scaletozero_datapath_test.go b/test/e2e/scaletozero_datapath_test.go
index c1dbab7..bbd3675 100644
--- a/test/e2e/scaletozero_datapath_test.go
+++ b/test/e2e/scaletozero_datapath_test.go
@@ -202,7 +202,7 @@ spec:
 		// an immediate single-shot ping can race a guest that isn't ready yet.
 		var warmOut string
 		Eventually(func(g Gomega) {
-			warmExit, out, execErr := execInVM(pingerName, "ping", "-c", "1", "-W", "5", targetIP)
+			warmExit, out, execErr := execInVM(pingerName, "/bin/ping", "-c", "1", "-W", "5", targetIP)
 			warmOut = out
 			g.Expect(execErr).NotTo(HaveOccurred())
 			g.Expect(warmExit).To(Equal(int32(0)))
@@ -217,7 +217,7 @@ spec:
 		}, "2m", "5s").Should(Succeed())
 
 		By("pinging the suspended VM's overlay IP from the pinger VM's guest agent")
-		wakeExit, wakeOut, err := execInVM(pingerName, "ping", "-c", "3", "-W", "2", targetIP)
+		wakeExit, wakeOut, err := execInVM(pingerName, "/bin/ping", "-c", "3", "-W", "2", targetIP)
 		Expect(err).NotTo(HaveOccurred())
 		GinkgoWriter.Printf("wake ping exit=%d output:\n%s\n", wakeExit, wakeOut)
 

From 34a347b86fe70023139a9c373baacace1c7388e7 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 15:31:05 +0100
Subject: [PATCH 16/19] feat(agent): add diagnostic logging to the
 wake-on-traffic path
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The scaletozero e2e run got all the way to the wake ping (suspend
worked, target reached Suspended cleanly) but the target never
transitioned to Resuming within the test's 2m window — and this code
path had zero logging anywhere, so there was no way to tell from
cluster logs whether the AF_PACKET hook saw nothing, saw traffic that
didn't match, or something else. Purely additive, no logic change:

- afpacketSource.Run: logs socket open/failure, then a throttled
  liveness line (first frame + every 10s) with frame count and last
  dst IP, so a validation run can tell "the socket sees nothing" from
  "the socket sees traffic, none of it matched."
- wakeRegistry.register/onDstIP: logs VM registration, and each of the
  three onDstIP outcomes (matched+enqueued, matched+already-signalled,
  matched+channel-full) at Info; a match against nothing registered
  stays silent by design, since unrelated overlay traffic would
  otherwise flood this log continuously in a busy cluster.

internal/agent unit tests (scaletozero_test.go, scaletozero_reconciler_test.go)
pass unchanged — no behavior touched, existing coverage still green.
---
 internal/agent/scaletozero.go       | 17 ++++++++++++++++-
 internal/agent/scaletozero_linux.go | 17 +++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/internal/agent/scaletozero.go b/internal/agent/scaletozero.go
index 6757fe2..ee37b4f 100644
--- a/internal/agent/scaletozero.go
+++ b/internal/agent/scaletozero.go
@@ -23,6 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	impdevv1alpha1 "github.com/syscode-labs/imp/api/v1alpha1"
 )
@@ -76,25 +77,39 @@ func (w *wakeRegistry) register(ip string, vm client.Object) {
 	w.keyByIP[ip] = key
 	w.ipByKey[key] = ip
 	w.objByKey[key] = vm
+	logf.Log.Info("wake registry: VM registered for wake-on-traffic", "vm", key, "ip", ip)
 }
 
 // onDstIP is the PacketSource callback: a frame arrived for ip. If ip belongs to
 // a registered VM not already signalled, enqueue a reconcile for it. The
 // signalled flag is set only when the event is actually enqueued, so a full
 // channel never silently loses a wake — the next packet retries.
+//
+// Diagnostic logging: this is the one hop the AF_PACKET PacketSource is
+// UNVALIDATED to reach (see scaletozero_linux.go), so every branch here is
+// logged at Info to make a validation run's outcome legible from cluster logs
+// without needing a debugger. Not gated behind a verbosity flag because a
+// suspended VM should see near-zero unmatched traffic — only VMs actually
+// awaiting wake generate log volume here.
 func (w *wakeRegistry) onDstIP(ip string) {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	key, ok := w.keyByIP[ip]
-	if !ok || w.signalled[key] {
+	if !ok {
+		return
+	}
+	if w.signalled[key] {
+		logf.Log.V(1).Info("wake registry: frame matched an already-signalled VM", "vm", key, "ip", ip)
 		return
 	}
 	obj := w.objByKey[key]
 	select {
 	case w.events <- event.GenericEvent{Object: obj}:
 		w.signalled[key] = true
+		logf.Log.Info("wake registry: frame matched, wake reconcile enqueued", "vm", key, "ip", ip)
 	default:
 		// Channel full; leave unsignalled so a later packet retries.
+		logf.Log.Info("wake registry: frame matched but event channel full, will retry", "vm", key, "ip", ip)
 	}
 }
 
diff --git a/internal/agent/scaletozero_linux.go b/internal/agent/scaletozero_linux.go
index d042fab..9f461ad 100644
--- a/internal/agent/scaletozero_linux.go
+++ b/internal/agent/scaletozero_linux.go
@@ -10,6 +10,7 @@ import (
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 	"k8s.io/apimachinery/pkg/types"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 // resetIdle forgets any idle sample for key (called on suspend/resume so the VM
@@ -62,14 +63,24 @@ func htons(v uint16) uint16 { return v<<8 | v>>8 }
 func (afpacketSource) Run(ctx context.Context, onDstIP func(string)) error {
 	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_IP)))
 	if err != nil {
+		logf.Log.Error(err, "afpacketSource: failed to open AF_PACKET socket")
 		return err
 	}
+	logf.Log.Info("afpacketSource: AF_PACKET socket open, capturing inbound IPv4 frames")
 	// Unblock the blocking Recvfrom and release the fd when the manager stops.
 	go func() {
 		<-ctx.Done()
 		_ = unix.Close(fd)
 	}()
 
+	// Diagnostic-only liveness counter (see scaletozero.go: this hook is
+	// UNVALIDATED). Proves whether the socket sees ANY IPv4 traffic at all,
+	// independent of whether it matches a registered wake IP — onDstIP() only
+	// logs on a match, so a validation run with zero matches is otherwise
+	// indistinguishable from a socket that receives nothing.
+	var frameCount uint64
+	lastLog := time.Now()
+
 	buf := make([]byte, 65536)
 	for {
 		n, _, err := unix.Recvfrom(fd, buf, 0)
@@ -85,6 +96,12 @@ func (afpacketSource) Run(ctx context.Context, onDstIP func(string)) error {
 		if n < 34 {
 			continue
 		}
+		frameCount++
+		if frameCount == 1 || time.Since(lastLog) >= 10*time.Second {
+			logf.Log.Info("afpacketSource: capturing IPv4 traffic", "framesSeen", frameCount,
+				"lastDstIP", net.IP(buf[30:34]).String())
+			lastLog = time.Now()
+		}
 		onDstIP(net.IP(buf[30:34]).String())
 	}
 }

From 8c13cee9bfb0c287f92571ee866376b6cab0d638 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 19:30:59 +0100
Subject: [PATCH 17/19] fix(agent): wake hook must use ETH_P_ALL, not ETH_P_IP

Root-caused via a real-KVM CI run + Fable investigation: the socket
opened with a specific protocol (ETH_P_IP) registers into the kernel's
ptype_base hash, which __netif_receive_skb_core only invokes if the
receiving device's rx_handler returns RX_HANDLER_PASS. A bridge port's
rx_handler (br_handle_frame) returns RX_HANDLER_CONSUMED for every
switched/flooded frame -- i.e. every ordinary VM-to-VM frame on an
impbr-* bridge -- so ptype_base delivery never happens for exactly the
traffic this hook exists to see. This exactly matched the evidence: the
diagnostic run captured thousands of frames on lo/the pod's own CNI IP
(neither bridged, no rx_handler) but zero frames from imp's own overlay
subnet, and zero wake matches, across the whole run.

Fix: bind with ETH_P_ALL (registers into ptype_all, invoked before
rx_handler -- same mechanism tcpdump uses to see switched traffic) and
filter to IPv4 in userspace via the EtherType field instead of at
socket() time.
---
 internal/agent/scaletozero_linux.go | 33 +++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/internal/agent/scaletozero_linux.go b/internal/agent/scaletozero_linux.go
index 9f461ad..61dc2f5 100644
--- a/internal/agent/scaletozero_linux.go
+++ b/internal/agent/scaletozero_linux.go
@@ -53,15 +53,31 @@ func netlinkLinkStats(iface string) (uint64, error) {
 // raw socket (unbound, so it sees every overlay bridge) and reports each frame's
 // destination IP. One socket serves all suspended VMs on the node.
 //
-// UNVALIDATED (see scaletozero.go): not yet confirmed to observe the first frame
-// destined to a TAP-less suspended VM. Swap for a tc-BPF PacketSource if the
-// cluster spike shows the frame is dropped before this hook.
+// VALIDATED-then-fixed (2026-07-29 real-KVM CI run, see scaletozero.go): the
+// first cluster spike proved this hook never observed overlay traffic when
+// opened with a specific protocol (ETH_P_IP). Root cause: passing a specific
+// EtherType at socket() time registers the handler in the kernel's per-protocol
+// ptype_base hash, which __netif_receive_skb_core only invokes if the receiving
+// device's rx_handler returns RX_HANDLER_PASS. A bridge port's rx_handler
+// (br_handle_frame) returns RX_HANDLER_CONSUMED for every switched/flooded
+// frame — i.e. every ordinary VM-to-VM frame on an impbr-* bridge — so
+// ptype_base delivery never happens for exactly the traffic this hook exists
+// to see. Only ptype_all (registered by an unbound/protocol-0 socket, same as
+// tcpdump's default) is invoked before rx_handler and sees the frame
+// regardless of what the bridge later does with it. Hence: bind with
+// ETH_P_ALL and filter IPv4 in userspace instead of filtering at socket()
+// time. Do not "simplify" this back to ETH_P_IP — it silently stops seeing
+// all bridged/switched traffic while still working for non-bridged devices
+// (lo, an unbridged pod veth), which is exactly what made the original bug
+// hard to notice.
 type afpacketSource struct{}
 
 func htons(v uint16) uint16 { return v<<8 | v>>8 }
 
+const ethTypeIPv4 = 0x0800
+
 func (afpacketSource) Run(ctx context.Context, onDstIP func(string)) error {
-	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_IP)))
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL)))
 	if err != nil {
 		logf.Log.Error(err, "afpacketSource: failed to open AF_PACKET socket")
 		return err
@@ -91,11 +107,16 @@ func (afpacketSource) Run(ctx context.Context, onDstIP func(string)) error {
 			time.Sleep(10 * time.Millisecond) // avoid a tight spin on a persistent recv error
 			continue
 		}
-		// AF_PACKET/SOCK_RAW frames include the 14-byte Ethernet header; the IPv4
-		// destination address sits at bytes 30..34 (eth[14] + ipv4[16..20]).
+		// AF_PACKET/SOCK_RAW frames include the 14-byte Ethernet header. With
+		// ETH_P_ALL the socket now also receives ARP, IPv6, STP, LLDP, etc., so
+		// filter to IPv4 in userspace via the EtherType at bytes 12..14 before
+		// treating bytes 30..34 (eth[14] + ipv4[16..20]) as an IPv4 dest addr.
 		if n < 34 {
 			continue
 		}
+		if ethType := uint16(buf[12])<<8 | uint16(buf[13]); ethType != ethTypeIPv4 {
+			continue
+		}
 		frameCount++
 		if frameCount == 1 || time.Since(lastLog) >= 10*time.Second {
 			logf.Log.Info("afpacketSource: capturing IPv4 traffic", "framesSeen", frameCount,

From 70801f0e7a9d9bc1bd7d5d78c175234018ffddbe Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 21:34:15 +0100
Subject: [PATCH 18/19] fix(agent): skip FDB entries with no dst IP in SyncFDB
 delete sweep

Root-caused via Fable investigation, confirmed against real CI job
logs (job 90608981396): "delete FDB entry <mac> ...: address family
not supported by protocol" is EAFNOSUPPORT, hit deterministically
every run. When the VXLAN interface attaches to the bridge
(attachToBridge), Linux auto-creates a "self permanent" FDB entry for
the port's own MAC with no tunnel destination. SyncFDB's stale-entry
sweep isn't caught by the existing all-zeros-MAC skip, tries to delete
it anyway, and NeighDel serializes a 0-length NDA_DST from the nil IP,
which the kernel's VXLAN driver rejects.

Not just log noise: the delete loop runs before the add loop and
returns on first error, so SyncFDB could never reach NeighAdd once the
interface was bridged -- meaning real VTEP entries plausibly never got
added either. Not a Kind/nested-container artifact: NeighAdd,
NeighList, and bridge attach all succeed in the same environment: the
failure is data-shape driven (empty attribute), not a permission or
capability gap, and would reproduce identically on Talos nodes.

Fix: skip entries with no resolvable destination IP in the delete
sweep, alongside the existing all-zeros-broadcast-MAC skip.
---
 internal/agent/network/vxlan.go | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/internal/agent/network/vxlan.go b/internal/agent/network/vxlan.go
index 495e38b..6f6d00b 100644
--- a/internal/agent/network/vxlan.go
+++ b/internal/agent/network/vxlan.go
@@ -95,13 +95,24 @@ func (m *LinuxNetManager) SyncFDB(_ context.Context, ifaceName string, entries [
 		return fmt.Errorf("list FDB entries for %s: %w", ifaceName, err)
 	}
 
-	// Remove stale entries — skip the all-zeros broadcast entry.
+	// Remove stale entries — skip the all-zeros broadcast entry, and skip any
+	// entry with no resolvable destination IP. When this VXLAN interface is
+	// attached to a bridge, the kernel auto-creates a "self permanent" FDB
+	// entry for the port's own MAC with no tunnel destination; NeighDel on
+	// that entry serializes an empty NDA_DST attribute, which the kernel's
+	// VXLAN driver rejects with EAFNOSUPPORT ("address family not supported by
+	// protocol") — deterministically, every time, since it's not something
+	// SyncFDB itself ever added. Confirmed via CI: this deletion failure
+	// aborted the whole sync before any real VTEP entry was ever added.
 	allZeros := "00:00:00:00:00:00"
 	for _, n := range current {
 		mac := n.HardwareAddr.String()
 		if mac == allZeros {
 			continue
 		}
+		if n.IP.To4() == nil && n.IP.To16() == nil {
+			continue
+		}
 		if _, ok := desired[mac]; !ok {
 			del := &netlink.Neigh{
 				LinkIndex:    link.Attrs().Index,

From 72c4214c099e6269d89ae8e24d8b4e7b5fb791e5 Mon Sep 17 00:00:00 2001
From: Giovanni Ferri <giovanni@syscode.uk>
Date: Wed, 29 Jul 2026 22:38:31 +0100
Subject: [PATCH 19/19] ci: move real-agent datapath e2e to nightly +
 path-triggered

Was manual-dispatch-only (never ran automatically). Splits it into its
own workflow file matching the existing nightly-e2e.yml /
nightly-e2e-cilium-heavy.yml convention: schedule (04:00 UTC daily,
staggered after the 02:00/03:00 nightly jobs), plus a pull_request
paths filter so PRs touching the fragile suspend/resume/wake code
trigger it automatically. E2E Smoke (Kind) in ci.yml is unchanged and
stays the fast always-on push/PR gate.

Adds a Telegram notify-on-failure step (reusing the org's existing bot
pattern from oci-free-tier-monitor) for schedule/workflow_dispatch
runs -- skips cleanly until TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID repo
secrets are added (not yet configured on this repo).
---
 .github/workflows/ci.yml                   |  77 ------------
 .github/workflows/nightly-e2e-datapath.yml | 130 +++++++++++++++++++++
 2 files changed, 130 insertions(+), 77 deletions(-)
 create mode 100644 .github/workflows/nightly-e2e-datapath.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 03612a0..2502074 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -125,83 +125,6 @@ jobs:
         with:
           files: coverage.out
 
-  e2e-datapath:
-    name: E2E Datapath (Kind, real agent)
-    # Boots a real Firecracker agent via nested KVM — slower and less proven than
-    # the Kind smoke job, so it's manual-dispatch only rather than on every push.
-    # See docs (e2e runner runbook) for why: proven on ubuntu-latest (run 29019535180),
-    # but the datapath spec itself has never executed end to end yet.
-    if: github.event_name == 'workflow_dispatch'
-    runs-on: ubuntu-latest
-    needs: [lint, build]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v6
-
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version: ${{ env.GO_VERSION }}
-
-      - name: Allow unprivileged access to /dev/kvm
-        run: |
-          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
-          sudo udevadm control --reload-rules
-          sudo udevadm trigger --name-match=kvm
-          ls -l /dev/kvm
-
-      - name: Create Kind cluster
-        uses: helm/kind-action@v1
-        with:
-          cluster_name: imp-e2e-datapath
-          config: test/e2e/kind-datapath.yaml
-
-      - name: Install Helm CLI
-        uses: azure/setup-helm@v4
-
-      - name: Build local E2E images
-        run: |
-          docker build -f Dockerfile.operator -t local/imp-operator:e2e .
-          docker build -f Dockerfile.agent -t local/imp-agent:e2e .
-
-      - name: Load local E2E images into Kind
-        run: |
-          kind load docker-image local/imp-operator:e2e --name imp-e2e-datapath
-          kind load docker-image local/imp-agent:e2e --name imp-e2e-datapath
-
-      - name: Stage Firecracker binary + guest kernel on the Kind node
-        # No imp-specific guest kernel exists yet (planned "imp-guest-kernel" repo,
-        # not built) — every UNVALIDATED-marked path in this repo uses the public
-        # Firecracker quickstart kernel as a stopgap; same here.
-        run: |
-          node=$(kind get nodes --name imp-e2e-datapath | head -1)
-          fc_ver=v1.15.0
-          curl -fsSL "https://github.com/firecracker-microvm/firecracker/releases/download/${fc_ver}/firecracker-${fc_ver}-x86_64.tgz" -o /tmp/firecracker.tgz
-          tar -xzf /tmp/firecracker.tgz -C /tmp
-          docker cp "/tmp/release-${fc_ver}-x86_64/firecracker-${fc_ver}-x86_64" "${node}:/usr/local/bin/firecracker"
-          docker exec "${node}" chmod +x /usr/local/bin/firecracker
-          curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin" -o /tmp/vmlinux
-          docker exec "${node}" mkdir -p /var/lib/imp
-          docker cp /tmp/vmlinux "${node}:/var/lib/imp/vmlinux"
-
-      - name: Run datapath E2E suite
-        run: |
-          IMP_E2E_REAL_AGENT=true go test -tags e2e ./test/e2e/... -v -timeout 30m -ginkgo.label-filter="datapath"
-
-      - name: Dump cluster diagnostics on failure
-        if: failure()
-        run: |
-          kubectl get pods -A -o wide || true
-          kubectl get impvm,impnetwork,impvmclass -A -o wide || true
-          kubectl describe pods -n imp-system || true
-          kubectl describe impvm -n default || true
-          echo "--- operator logs ---"
-          kubectl logs -n imp-system -l app.kubernetes.io/component=operator --tail=200 || true
-          echo "--- agent logs ---"
-          kubectl logs -n imp-system -l app.kubernetes.io/component=agent --tail=200 || true
-          echo "--- events ---"
-          kubectl get events -A --sort-by=.lastTimestamp || true
-
   e2e-kind:
     name: E2E Smoke (Kind)
     runs-on: ubuntu-latest
diff --git a/.github/workflows/nightly-e2e-datapath.yml b/.github/workflows/nightly-e2e-datapath.yml
new file mode 100644
index 0000000..8c52eb7
--- /dev/null
+++ b/.github/workflows/nightly-e2e-datapath.yml
@@ -0,0 +1,130 @@
+name: Nightly E2E Datapath (Kind, real agent)
+
+on:
+  schedule:
+    # 04:00 UTC daily (staggered after the 02:00/03:00 nightly smoke/cilium jobs)
+    - cron: "0 4 * * *"
+  pull_request:
+    branches: [main]
+    paths:
+      - "internal/agent/scaletozero*.go"
+      - "internal/agent/network/**"
+      - "internal/agent/reconciler.go"
+      - "internal/capability/**"
+      - "charts/imp/templates/agent/**"
+      - "test/e2e/scaletozero_datapath_test.go"
+      - "test/e2e/kind-datapath.yaml"
+      - ".github/workflows/nightly-e2e-datapath.yml"
+  workflow_dispatch:
+
+concurrency:
+  group: nightly-e2e-datapath
+  cancel-in-progress: false
+
+env:
+  GO_VERSION: "1.26"
+
+jobs:
+  e2e-datapath:
+    name: E2E Datapath (Kind, real agent)
+    # Boots a real Firecracker agent via nested KVM -- slower and less proven
+    # than the Kind smoke job, so it runs nightly + on PRs touching the
+    # datapath's own fragile paths, not on every push. See docs (e2e runner
+    # runbook) for provenance: proven on ubuntu-latest (run 29019535180); wake
+    # hook + FDB sync validated end-to-end in imp#35 (2026-07-29).
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Allow unprivileged access to /dev/kvm
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+          ls -l /dev/kvm
+
+      - name: Create Kind cluster
+        uses: helm/kind-action@v1
+        with:
+          cluster_name: imp-e2e-datapath
+          config: test/e2e/kind-datapath.yaml
+
+      - name: Install Helm CLI
+        uses: azure/setup-helm@v4
+
+      - name: Build local E2E images
+        run: |
+          docker build -f Dockerfile.operator -t local/imp-operator:e2e .
+          docker build -f Dockerfile.agent -t local/imp-agent:e2e .
+
+      - name: Load local E2E images into Kind
+        run: |
+          kind load docker-image local/imp-operator:e2e --name imp-e2e-datapath
+          kind load docker-image local/imp-agent:e2e --name imp-e2e-datapath
+
+      - name: Stage Firecracker binary + guest kernel on the Kind node
+        # No imp-specific guest kernel exists yet (planned "imp-guest-kernel" repo,
+        # not built) — every UNVALIDATED-marked path in this repo uses the public
+        # Firecracker quickstart kernel as a stopgap; same here.
+        run: |
+          node=$(kind get nodes --name imp-e2e-datapath | head -1)
+          fc_ver=v1.15.0
+          curl -fsSL "https://github.com/firecracker-microvm/firecracker/releases/download/${fc_ver}/firecracker-${fc_ver}-x86_64.tgz" -o /tmp/firecracker.tgz
+          tar -xzf /tmp/firecracker.tgz -C /tmp
+          docker cp "/tmp/release-${fc_ver}-x86_64/firecracker-${fc_ver}-x86_64" "${node}:/usr/local/bin/firecracker"
+          docker exec "${node}" chmod +x /usr/local/bin/firecracker
+          curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin" -o /tmp/vmlinux
+          docker exec "${node}" mkdir -p /var/lib/imp
+          docker cp /tmp/vmlinux "${node}:/var/lib/imp/vmlinux"
+
+      - name: Run datapath E2E suite (retry once)
+        run: |
+          set -euo pipefail
+          if ! IMP_E2E_REAL_AGENT=true go test -tags e2e ./test/e2e/... -v -timeout 30m -ginkgo.label-filter="datapath"; then
+            echo "First run failed; retrying once..."
+            IMP_E2E_REAL_AGENT=true go test -tags e2e ./test/e2e/... -v -timeout 30m -ginkgo.label-filter="datapath"
+          fi
+
+      - name: Dump cluster diagnostics on failure
+        if: failure()
+        run: |
+          mkdir -p artifacts
+          kubectl get pods -A -o wide > artifacts/pods-all.txt || true
+          kubectl get impvm,impnetwork,impvmclass -A -o wide > artifacts/impvm-all.txt || true
+          kubectl describe pods -n imp-system > artifacts/imp-system-pods-describe.txt || true
+          kubectl describe impvm -n default > artifacts/impvm-describe.txt || true
+          kubectl logs -n imp-system -l app.kubernetes.io/component=operator --tail=500 > artifacts/operator-logs.txt || true
+          kubectl logs -n imp-system -l app.kubernetes.io/component=agent --tail=500 > artifacts/agent-logs.txt || true
+          kubectl get events -A --sort-by=.lastTimestamp > artifacts/events.txt || true
+
+      - name: Upload diagnostics
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: nightly-e2e-datapath-diagnostics
+          path: artifacts/
+
+      - name: Notify Telegram on failure
+        # Reuses the org's existing bot pattern (see oci-free-tier-monitor).
+        # Skips cleanly (not a failed step) until TELEGRAM_BOT_TOKEN and
+        # TELEGRAM_CHAT_ID are added as repo secrets.
+        if: failure() && github.event_name != 'pull_request'
+        env:
+          TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
+          TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
+        run: |
+          if [ -z "${TELEGRAM_BOT_TOKEN:-}" ] || [ -z "${TELEGRAM_CHAT_ID:-}" ]; then
+            echo "TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID not set, skipping notification"
+            exit 0
+          fi
+          run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          curl -fsS -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
+            --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
+            --data-urlencode "text=imp: Nightly E2E Datapath failed on ${GITHUB_REF_NAME} (${GITHUB_SHA:0:7}) -- ${run_url}"