From c205baedfc2be4048e58dd1b0e48d6e53b762840 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Wed, 29 Jul 2026 22:43:10 +0100 Subject: [PATCH 1/2] ci: move real-agent datapath e2e to nightly + path-triggered Was manual-dispatch-only (never ran automatically). Splits it into its own workflow file matching the existing nightly-e2e.yml / nightly-e2e-cilium-heavy.yml convention: schedule (04:00 UTC daily, staggered after the 02:00/03:00 nightly jobs), plus a pull_request paths filter so PRs touching the fragile suspend/resume/wake code trigger it automatically. E2E Smoke (Kind) in ci.yml is unchanged and stays the fast always-on push/PR gate. Adds a Telegram notify-on-failure step (reusing the org's existing bot pattern from oci-free-tier-monitor) for schedule/workflow_dispatch runs -- skips cleanly until TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID repo secrets are added (not yet configured on this repo). --- .github/workflows/ci.yml | 77 ------------ .github/workflows/nightly-e2e-datapath.yml | 130 +++++++++++++++++++++ 2 files changed, 130 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/nightly-e2e-datapath.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03612a0..2502074 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,83 +125,6 @@ jobs: with: files: coverage.out - e2e-datapath: - name: E2E Datapath (Kind, real agent) - # Boots a real Firecracker agent via nested KVM — slower and less proven than - # the Kind smoke job, so it's manual-dispatch only rather than on every push. - # See docs (e2e runner runbook) for why: proven on ubuntu-latest (run 29019535180), - # but the datapath spec itself has never executed end to end yet. - if: github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - needs: [lint, build] - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - - - name: Allow unprivileged access to /dev/kvm - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - ls -l /dev/kvm - - - name: Create Kind cluster - uses: helm/kind-action@v1 - with: - cluster_name: imp-e2e-datapath - config: test/e2e/kind-datapath.yaml - - - name: Install Helm CLI - uses: azure/setup-helm@v4 - - - name: Build local E2E images - run: | - docker build -f Dockerfile.operator -t local/imp-operator:e2e . - docker build -f Dockerfile.agent -t local/imp-agent:e2e . - - - name: Load local E2E images into Kind - run: | - kind load docker-image local/imp-operator:e2e --name imp-e2e-datapath - kind load docker-image local/imp-agent:e2e --name imp-e2e-datapath - - - name: Stage Firecracker binary + guest kernel on the Kind node - # No imp-specific guest kernel exists yet (planned "imp-guest-kernel" repo, - # not built) — every UNVALIDATED-marked path in this repo uses the public - # Firecracker quickstart kernel as a stopgap; same here. - run: | - node=$(kind get nodes --name imp-e2e-datapath | head -1) - fc_ver=v1.15.0 - curl -fsSL "https://github.com/firecracker-microvm/firecracker/releases/download/${fc_ver}/firecracker-${fc_ver}-x86_64.tgz" -o /tmp/firecracker.tgz - tar -xzf /tmp/firecracker.tgz -C /tmp - docker cp "/tmp/release-${fc_ver}-x86_64/firecracker-${fc_ver}-x86_64" "${node}:/usr/local/bin/firecracker" - docker exec "${node}" chmod +x /usr/local/bin/firecracker - curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin" -o /tmp/vmlinux - docker exec "${node}" mkdir -p /var/lib/imp - docker cp /tmp/vmlinux "${node}:/var/lib/imp/vmlinux" - - - name: Run datapath E2E suite - run: | - IMP_E2E_REAL_AGENT=true go test -tags e2e ./test/e2e/... -v -timeout 30m -ginkgo.label-filter="datapath" - - - name: Dump cluster diagnostics on failure - if: failure() - run: | - kubectl get pods -A -o wide || true - kubectl get impvm,impnetwork,impvmclass -A -o wide || true - kubectl describe pods -n imp-system || true - kubectl describe impvm -n default || true - echo "--- operator logs ---" - kubectl logs -n imp-system -l app.kubernetes.io/component=operator --tail=200 || true - echo "--- agent logs ---" - kubectl logs -n imp-system -l app.kubernetes.io/component=agent --tail=200 || true - echo "--- events ---" - kubectl get events -A --sort-by=.lastTimestamp || true - e2e-kind: name: E2E Smoke (Kind) runs-on: ubuntu-latest diff --git a/.github/workflows/nightly-e2e-datapath.yml b/.github/workflows/nightly-e2e-datapath.yml new file mode 100644 index 0000000..8c52eb7 --- /dev/null +++ b/.github/workflows/nightly-e2e-datapath.yml @@ -0,0 +1,130 @@ +name: Nightly E2E Datapath (Kind, real agent) + +on: + schedule: + # 04:00 UTC daily (staggered after the 02:00/03:00 nightly smoke/cilium jobs) + - cron: "0 4 * * *" + pull_request: + branches: [main] + paths: + - "internal/agent/scaletozero*.go" + - "internal/agent/network/**" + - "internal/agent/reconciler.go" + - "internal/capability/**" + - "charts/imp/templates/agent/**" + - "test/e2e/scaletozero_datapath_test.go" + - "test/e2e/kind-datapath.yaml" + - ".github/workflows/nightly-e2e-datapath.yml" + workflow_dispatch: + +concurrency: + group: nightly-e2e-datapath + cancel-in-progress: false + +env: + GO_VERSION: "1.26" + +jobs: + e2e-datapath: + name: E2E Datapath (Kind, real agent) + # Boots a real Firecracker agent via nested KVM -- slower and less proven + # than the Kind smoke job, so it runs nightly + on PRs touching the + # datapath's own fragile paths, not on every push. See docs (e2e runner + # runbook) for provenance: proven on ubuntu-latest (run 29019535180); wake + # hook + FDB sync validated end-to-end in imp#35 (2026-07-29). + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Allow unprivileged access to /dev/kvm + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + ls -l /dev/kvm + + - name: Create Kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: imp-e2e-datapath + config: test/e2e/kind-datapath.yaml + + - name: Install Helm CLI + uses: azure/setup-helm@v4 + + - name: Build local E2E images + run: | + docker build -f Dockerfile.operator -t local/imp-operator:e2e . + docker build -f Dockerfile.agent -t local/imp-agent:e2e . + + - name: Load local E2E images into Kind + run: | + kind load docker-image local/imp-operator:e2e --name imp-e2e-datapath + kind load docker-image local/imp-agent:e2e --name imp-e2e-datapath + + - name: Stage Firecracker binary + guest kernel on the Kind node + # No imp-specific guest kernel exists yet (planned "imp-guest-kernel" repo, + # not built) — every UNVALIDATED-marked path in this repo uses the public + # Firecracker quickstart kernel as a stopgap; same here. + run: | + node=$(kind get nodes --name imp-e2e-datapath | head -1) + fc_ver=v1.15.0 + curl -fsSL "https://github.com/firecracker-microvm/firecracker/releases/download/${fc_ver}/firecracker-${fc_ver}-x86_64.tgz" -o /tmp/firecracker.tgz + tar -xzf /tmp/firecracker.tgz -C /tmp + docker cp "/tmp/release-${fc_ver}-x86_64/firecracker-${fc_ver}-x86_64" "${node}:/usr/local/bin/firecracker" + docker exec "${node}" chmod +x /usr/local/bin/firecracker + curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin" -o /tmp/vmlinux + docker exec "${node}" mkdir -p /var/lib/imp + docker cp /tmp/vmlinux "${node}:/var/lib/imp/vmlinux" + + - name: Run datapath E2E suite (retry once) + run: | + set -euo pipefail + if ! IMP_E2E_REAL_AGENT=true go test -tags e2e ./test/e2e/... -v -timeout 30m -ginkgo.label-filter="datapath"; then + echo "First run failed; retrying once..." + IMP_E2E_REAL_AGENT=true go test -tags e2e ./test/e2e/... -v -timeout 30m -ginkgo.label-filter="datapath" + fi + + - name: Dump cluster diagnostics on failure + if: failure() + run: | + mkdir -p artifacts + kubectl get pods -A -o wide > artifacts/pods-all.txt || true + kubectl get impvm,impnetwork,impvmclass -A -o wide > artifacts/impvm-all.txt || true + kubectl describe pods -n imp-system > artifacts/imp-system-pods-describe.txt || true + kubectl describe impvm -n default > artifacts/impvm-describe.txt || true + kubectl logs -n imp-system -l app.kubernetes.io/component=operator --tail=500 > artifacts/operator-logs.txt || true + kubectl logs -n imp-system -l app.kubernetes.io/component=agent --tail=500 > artifacts/agent-logs.txt || true + kubectl get events -A --sort-by=.lastTimestamp > artifacts/events.txt || true + + - name: Upload diagnostics + if: failure() + uses: actions/upload-artifact@v7 + with: + name: nightly-e2e-datapath-diagnostics + path: artifacts/ + + - name: Notify Telegram on failure + # Reuses the org's existing bot pattern (see oci-free-tier-monitor). + # Skips cleanly (not a failed step) until TELEGRAM_BOT_TOKEN and + # TELEGRAM_CHAT_ID are added as repo secrets. + if: failure() && github.event_name != 'pull_request' + env: + TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} + TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} + run: | + if [ -z "${TELEGRAM_BOT_TOKEN:-}" ] || [ -z "${TELEGRAM_CHAT_ID:-}" ]; then + echo "TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID not set, skipping notification" + exit 0 + fi + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + curl -fsS -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ + --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \ + --data-urlencode "text=imp: Nightly E2E Datapath failed on ${GITHUB_REF_NAME} (${GITHUB_SHA:0:7}) -- ${run_url}" From e1242a561049324c0b631b16fcfa4a98e494b2f0 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Wed, 29 Jul 2026 23:06:38 +0100 Subject: [PATCH 2/2] ci: use org secrets SYSCODE_LABS_BOT_TOKEN/CHATID for notify Confirmed via gh api organization-secrets: both already exist as org-level secrets visible to this repo, created just now. Switch from the placeholder TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID names to the actual org secret names so the notify step works without any further repo-level secret setup. --- .github/workflows/nightly-e2e-datapath.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly-e2e-datapath.yml b/.github/workflows/nightly-e2e-datapath.yml index 8c52eb7..45f920e 100644 --- a/.github/workflows/nightly-e2e-datapath.yml +++ b/.github/workflows/nightly-e2e-datapath.yml @@ -112,19 +112,20 @@ jobs: path: artifacts/ - name: Notify Telegram on failure - # Reuses the org's existing bot pattern (see oci-free-tier-monitor). - # Skips cleanly (not a failed step) until TELEGRAM_BOT_TOKEN and - # TELEGRAM_CHAT_ID are added as repo secrets. + # Reuses the org-wide bot (org secrets SYSCODE_LABS_BOT_TOKEN / + # SYSCODE_LABS_BOT_CHATID -- same pattern as oci-free-tier-monitor's + # own bot). Skips cleanly (not a failed step) if the org secrets are + # ever unavailable to this repo. if: failure() && github.event_name != 'pull_request' env: - TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} - TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} + BOT_TOKEN: ${{ secrets.SYSCODE_LABS_BOT_TOKEN }} + CHAT_ID: ${{ secrets.SYSCODE_LABS_BOT_CHATID }} run: | - if [ -z "${TELEGRAM_BOT_TOKEN:-}" ] || [ -z "${TELEGRAM_CHAT_ID:-}" ]; then - echo "TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID not set, skipping notification" + if [ -z "${BOT_TOKEN:-}" ] || [ -z "${CHAT_ID:-}" ]; then + echo "SYSCODE_LABS_BOT_TOKEN/SYSCODE_LABS_BOT_CHATID not set, skipping notification" exit 0 fi run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - curl -fsS -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ - --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \ + curl -fsS -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ + --data-urlencode "chat_id=${CHAT_ID}" \ --data-urlencode "text=imp: Nightly E2E Datapath failed on ${GITHUB_REF_NAME} (${GITHUB_SHA:0:7}) -- ${run_url}"