feat: run benches in a libkrun microVM for predictable numbers

Makefile

@@ -240,16 +240,28 @@ ast-bench-self: ast-bench ## Compare the ast bench against itself — proves the
 #   BENCH     — `council` (default, ~5s) / `big-diff` (~30s) / `core` /
 #               `ast` / `all`.
 #   VERBOSE   — set to anything to stream cargo + bench output.
+#
+# microVM knobs (predictable numbers via a libkrun guest — see microvm-setup):
+#   ISOLATION — `local` (default) or `microvm`.
+#   VM_CPUS   — guest vCPUs (default 2).   VM_MEM — guest RAM MiB (default 2048).
+#   VM_IMAGE  — override the guest OCI image (default is per-language).
+
+REF       ?= origin/master
+BENCH     ?= council
+ISOLATION ?=
+VM_CPUS   ?=
+VM_MEM    ?=
+VM_IMAGE  ?=
 
-REF   ?= origin/master
-BENCH ?= council
+# Flags appended to `perf-vs` from the microVM knobs above (empty ⇒ host run).
+vmflags = $(if $(ISOLATION),--isolation $(ISOLATION),) $(if $(VM_CPUS),--vm-cpus $(VM_CPUS),) $(if $(VM_MEM),--vm-mem $(VM_MEM),) $(if $(VM_IMAGE),--vm-image $(VM_IMAGE),)
 
 .PHONY: perf-vs
 perf-vs: $(TMP) build-rust ## Local A/B perf-compare HEAD vs $(REF) via a sibling worktree. Knobs: REF, BENCH, VERBOSE.
 	$(call say,perf-vs)
 	$(AATXE_BIN) perf-vs --against $(REF) --bench $(BENCH) \
 	    --out-dir $(TMP)/perf-vs/$(BENCH) \
-	    $(if $(VERBOSE),--verbose,)
+	    $(if $(VERBOSE),--verbose,) $(vmflags)
 	@echo "    ✓ perf-vs: see $(TMP)/perf-vs/$(BENCH)/{head,base}.json + cmp.{json,md}"
 
 .PHONY: perf-vs-self
@@ -260,6 +272,41 @@ perf-vs-self: $(TMP) build-rust ## Smoke test: perf-vs against HEAD~1 (or origin
 	       --out-dir $(TMP)/perf-vs/self
 	@echo "    ✓ perf-vs-self: see $(TMP)/perf-vs/self/cmp.md"
 
+# ----------------------------------------------------------------------------
+# microVM benches (predictable numbers via a libkrun guest)
+# ----------------------------------------------------------------------------
+#
+# `--isolation microvm` runs the bench inside an ephemeral libkrun microVM
+# (pinned vCPUs, fixed RAM, clean rootfs) so numbers don't drift with host
+# load. Works on macOS (Hypervisor.framework) and Linux (KVM). One-time
+# `microvm-setup` installs `krunvm`; the first run pulls the guest image.
+
+.PHONY: perf-vs-vm
+perf-vs-vm: ## perf-vs inside a libkrun microVM. Knobs: REF, BENCH, VM_CPUS, VM_MEM, VM_IMAGE.
+	@$(MAKE) --no-print-directory perf-vs ISOLATION=microvm
+
+.PHONY: microvm-setup
+microvm-setup: ## One-time: install krunvm (the libkrun microVM runner).
+	$(call say,microvm-setup)
+	@if [ "$$(uname)" = "Darwin" ]; then \
+	    command -v brew >/dev/null 2>&1 || { echo "    ✗ Homebrew required — see https://brew.sh"; exit 1; }; \
+	    brew tap slp/krun && brew install krunvm && echo "    ✓ krunvm installed — run: make microvm-doctor"; \
+	 else \
+	    echo "    Linux: install krunvm from your distro (Fedora: dnf install krunvm) or"; \
+	    echo "    build libkrun + krunvm from https://github.com/containers/krunvm"; \
+	 fi
+
+.PHONY: microvm-doctor
+microvm-doctor: ## Check the microVM backend (krunvm present, host arch, version).
+	$(call say,microvm-doctor)
+	@echo "    host  : $$(uname -m) $$(uname -s)"
+	@if command -v krunvm >/dev/null 2>&1; then \
+	    echo "    krunvm: $$(krunvm --version 2>/dev/null | head -1)"; \
+	    echo "    ✓ ready — run: make perf-vs-vm   (or: aatxe perf-vs --isolation microvm …)"; \
+	 else \
+	    echo "    ✗ krunvm not found — run: make microvm-setup"; exit 1; \
+	 fi
+
 # ----------------------------------------------------------------------------
 # Local realtime dashboard
 # ----------------------------------------------------------------------------

README.md

@@ -175,6 +175,40 @@ aatxe run --lang go   --out /tmp/head.json
 aatxe run --lang rust --out /tmp/head.json
 ```
 
+#### Predictable numbers: run in a microVM
+
+Host noise — background load, another core waking, thermal throttling — is the
+enemy of a stable bench number. Pass `--isolation microvm` to run the bench
+inside an ephemeral [libkrun](https://github.com/containers/krunvm) microVM
+instead of on the host: pinned vCPUs, a fixed RAM budget, and a clean rootfs,
+so the same code benches the same way twice. One code path on macOS
+(Hypervisor.framework) and Linux (KVM).
+
+```bash
+make microvm-setup     # one-time: install krunvm (Homebrew on macOS)
+make microvm-doctor    # check it's ready
+
+# both bench entry points take the same knobs:
+aatxe run     --lang rust --isolation microvm
+aatxe perf-vs --against origin/master --isolation microvm
+make perf-vs-vm        # shorthand for: make perf-vs ISOLATION=microvm
+```
+
+Only the knobs that actually move the numbers are surfaced; everything else
+uses libkrun defaults:
+
+| flag (or env) | default | meaning |
+|---|---|---|
+| `--isolation` | `local` | `local` (host) or `microvm` |
+| `--vm-cpus` (`AATXE_VM_CPUS`) | `2` | guest vCPUs |
+| `--vm-mem` (`AATXE_VM_MEM`) | `2048` | guest RAM (MiB) |
+| `--vm-image` (`AATXE_VM_IMAGE`) | `rust:1` / `golang:1` / `node:22` | guest image (must ship the toolchain) |
+
+The working directory is mounted into the guest at the same path, and the
+`cargo` / `go` / `npm` caches persist under `~/.cache/aatxe/vm`, so only the
+first run pays the image-pull + cold-build cost. `perf-vs` runs *both* sides
+of the A/B in matching guests, keeping the comparison fair.
+
 ### 2½. Trial the gate locally — no CI wiring needed
 
 `aatxe baseline save` snapshots a report under `.aatxe/baselines/`

crates/aatxe/src/adapter/mod.rs

@@ -14,6 +14,7 @@
 //! so downstream consumers see identical statistics regardless of language.
 
 use crate::ast::ast_import_extractor::AstImportExtractor;
+use crate::sandbox::Isolation;
 use aatxe_core::affected::{resolve_affected, AffectedOptions};
 use aatxe_core::types::{AffectedScope as CoreScope, Language, RunReport};
 use anyhow::{Context, Result};
@@ -37,6 +38,8 @@ pub struct RunSpec {
     /// When `Some`, restrict the run to exactly these bench files (no discovery).
     pub bench_files: Option<Vec<PathBuf>>,
     pub verbose: bool,
+    /// Where the runner executes — host or microVM.
+    pub isolation: Isolation,
 }
 
 pub fn execute(lang: Language, spec: &RunSpec) -> Result<RunReport> {

crates/aatxe/src/adapter/ts_or_runner.rs

@@ -6,13 +6,27 @@
 //! * **TS**: `AATXE_TS_RUNNER`, default `npx --no-install aatxe-ts-runner`
 //! * **Go**: `AATXE_GO_RUNNER`, default `go test -bench=. -benchmem -json ./...`
 //! * **Rust**: `AATXE_RUST_RUNNER`, default `cargo run --release -q --bin aatxe-rust-runner -- --json`
+//!
+//! The command runs under [`crate::sandbox::Isolation`], so the same code
+//! path serves both a plain host run and an in-microVM run — the caller
+//! picks via `--isolation`. Either way we capture stdout (the RunReport) and
+//! let stderr stream through.
 
 use crate::adapter::RunSpec;
+use crate::sandbox;
 use aatxe_core::types::Language;
 use anyhow::{anyhow, Context, Result};
-use std::process::{Command, Stdio};
 
 pub fn run_runner(spec: &RunSpec, lang: Language) -> Result<String> {
+    // Streaming stdout and ingesting a RunReport off stdout are mutually
+    // exclusive regardless of isolation — bail early rather than run first.
+    if spec.verbose {
+        return Err(anyhow!(
+            "--verbose is incompatible with stdout-based RunReport ingestion; \
+             re-run without --verbose or pipe through the runner's --out flag"
+        ));
+    }
+
     let (env_key, default) = match lang {
         Language::Ts => ("AATXE_TS_RUNNER", "npx --no-install aatxe-ts-runner"),
         Language::Go => (
@@ -29,45 +43,32 @@ pub fn run_runner(spec: &RunSpec, lang: Language) -> Result<String> {
     let Some((program, args)) = tokens.split_first() else {
         return Err(anyhow!("empty runner command for {:?}", lang));
     };
-    let mut cmd = Command::new(program);
-    cmd.args(args);
-    cmd.current_dir(&spec.cwd);
+
+    let mut env: Vec<(String, String)> = Vec::new();
     if let Some(filter) = spec.filter.as_ref() {
-        cmd.env("AATXE_FILTER", filter);
+        env.push(("AATXE_FILTER".to_string(), filter.clone()));
     }
     if !spec.patterns.is_empty() {
-        cmd.env("AATXE_PATTERNS", spec.patterns.join(":"));
+        env.push(("AATXE_PATTERNS".to_string(), spec.patterns.join(":")));
     }
     if let Some(files) = spec.bench_files.as_ref() {
         let joined: Vec<String> = files.iter().map(|p| p.display().to_string()).collect();
-        cmd.env("AATXE_BENCH_FILES", joined.join(":"));
-    }
-    cmd.env("AATXE_SERVICE", &spec.service);
-    cmd.env("AATXE_REF", &spec.r#ref);
-    let verbose = spec.verbose;
-    cmd.stdin(Stdio::null());
-    if verbose {
-        cmd.stdout(Stdio::inherit());
-    } else {
-        cmd.stdout(Stdio::piped());
+        env.push(("AATXE_BENCH_FILES".to_string(), joined.join(":")));
     }
-    cmd.stderr(Stdio::inherit());
+    env.push(("AATXE_SERVICE".to_string(), spec.service.clone()));
+    env.push(("AATXE_REF".to_string(), spec.r#ref.clone()));
 
-    let output = cmd
-        .output()
-        .with_context(|| format!("spawning runner: {}", raw))?;
+    let script = sandbox::exec_line(program, args);
+    let output = spec
+        .isolation
+        .run_script(&spec.cwd, &script, &env)
+        .with_context(|| format!("running bench runner: {raw}"))?;
     if !output.status.success() {
         return Err(anyhow!(
             "runner exited with status {}",
             output.status.code().unwrap_or(-1)
         ));
     }
-    if verbose {
-        return Err(anyhow!(
-            "--verbose is incompatible with stdout-based RunReport ingestion; \
-             re-run without --verbose or pipe through the runner's --out flag"
-        ));
-    }
-    let raw = String::from_utf8(output.stdout).context("runner stdout was not valid UTF-8")?;
-    Ok(raw)
+    let raw_out = String::from_utf8(output.stdout).context("runner stdout was not valid UTF-8")?;
+    Ok(raw_out)
 }

crates/aatxe/src/cli.rs

@@ -62,6 +62,38 @@ pub enum Command {
     Ui(UiArgs),
 }
 
+/// Where benches execute — the backend for `--isolation`.
+#[derive(Copy, Clone, Debug, ValueEnum, PartialEq, Eq)]
+pub enum IsolationArg {
+    /// Run on the host, directly. Fast, but subject to host noise
+    /// (background load, thermal throttling, other processes).
+    Local,
+    /// Run inside an ephemeral libkrun microVM via `krunvm` — pinned
+    /// vCPUs, fixed RAM, clean rootfs — for reproducible numbers. Needs
+    /// `krunvm` installed (`make microvm-setup`); works on macOS and Linux.
+    Microvm,
+}
+
+/// Shared microVM knobs, flattened into every bench-running subcommand.
+/// Only the parameters that meaningfully change results are surfaced;
+/// everything else uses libkrun defaults.
+#[derive(clap::Args, Debug, Clone)]
+pub struct VmOpts {
+    /// Where to run the bench. `local` (default) or `microvm`.
+    #[arg(long, value_enum, default_value_t = IsolationArg::Local)]
+    pub isolation: IsolationArg,
+    /// microVM vCPUs (only with `--isolation microvm`).
+    #[arg(long, env = "AATXE_VM_CPUS", default_value_t = crate::sandbox::DEFAULT_VM_CPUS)]
+    pub vm_cpus: u32,
+    /// microVM RAM in MiB (only with `--isolation microvm`).
+    #[arg(long, env = "AATXE_VM_MEM", default_value_t = crate::sandbox::DEFAULT_VM_MEM_MIB)]
+    pub vm_mem: u32,
+    /// OCI image for the guest (must ship the language toolchain). Defaults
+    /// per language: `rust:1` / `golang:1` / `node:22`.
+    #[arg(long, env = "AATXE_VM_IMAGE")]
+    pub vm_image: Option<String>,
+}
+
 #[derive(Copy, Clone, Debug, ValueEnum, PartialEq, Eq)]
 pub enum PerfBenchArg {
     /// `examples/council-bench` — 9 micros covering the council's pure
@@ -129,6 +161,10 @@ pub struct PerfVsArgs {
     /// the final summary prints).
     #[arg(long)]
     pub verbose: bool,
+    /// microVM isolation knobs (`--isolation microvm` for predictable
+    /// numbers). Both sides of the A/B run in matching guests.
+    #[command(flatten)]
+    pub vm: VmOpts,
 }
 
 #[derive(Copy, Clone, Debug, ValueEnum, PartialEq, Eq)]
@@ -177,6 +213,10 @@ pub struct RunArgs {
     /// Print runner stdout to the caller as it produces it.
     #[arg(long)]
     pub verbose: bool,
+    /// microVM isolation knobs. `--isolation microvm` runs the language
+    /// runner inside a libkrun guest for predictable numbers.
+    #[command(flatten)]
+    pub vm: VmOpts,
     /// Bench discovery globs (passed to the adapter). Defaults are language-specific.
     pub patterns: Vec<String>,
 }