From ef645ab5f93496089661f7c0c925323c06d96ebb Mon Sep 17 00:00:00 2001 From: Eneko Sarasola <113593779+enekos@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:54:50 +0200 Subject: [PATCH] feat: run benches in a libkrun microVM for predictable numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Host noise (background load, thermal throttling, neighbouring processes) makes bench numbers drift run-to-run. Add an opt-in `--isolation microvm` that runs the bench inside an ephemeral libkrun microVM — pinned vCPUs, fixed RAM, clean rootfs — driven by the `krunvm` CLI. One code path on macOS (Hypervisor.framework) and Linux (KVM); no libkrun FFI, no new build-time dependency. - New `sandbox` module: Isolation::{Local,Microvm}, resolves VmOpts, boots an ephemeral guest, mounts the workdir 1:1 + a persistent build cache, execs through /bin/sh with a normalized PATH, and deletes the VM on drop. - Wired into both bench entry points via their single choke points: `perf-vs` (both A/B sides run in matching guests) and `aatxe run` (the shared run_runner all three language adapters funnel through). - Surfaces only the knobs that move numbers: --isolation / --vm-cpus / --vm-mem / --vm-image (env-backed, sane per-language image defaults). - Plug-and-play: `make microvm-setup` installs krunvm, `make microvm-doctor` checks readiness, `make perf-vs-vm` runs the A/B loop in a guest. Missing krunvm fails with an install hint, not a cryptic spawn error. Build caches (cargo/go/npm) persist under ~/.cache/aatxe/vm, isolated from the host's own target/ (different arch). Unit-tested at the argv/quoting level; the live boot path needs krunvm installed. --- Makefile | 53 ++- README.md | 34 ++ crates/aatxe/src/adapter/mod.rs | 3 + crates/aatxe/src/adapter/ts_or_runner.rs | 57 +-- crates/aatxe/src/cli.rs | 40 ++ crates/aatxe/src/commands/perf_vs.rs | 102 +++-- crates/aatxe/src/commands/run.rs | 3 + crates/aatxe/src/main.rs | 1 + crates/aatxe/src/sandbox.rs | 459 +++++++++++++++++++++++ 9 files changed, 696 insertions(+), 56 deletions(-) create mode 100644 crates/aatxe/src/sandbox.rs diff --git a/Makefile b/Makefile index fe0f22d..2e23a8f 100644 --- a/Makefile +++ b/Makefile @@ -240,16 +240,28 @@ ast-bench-self: ast-bench ## Compare the ast bench against itself — proves the # BENCH — `council` (default, ~5s) / `big-diff` (~30s) / `core` / # `ast` / `all`. # VERBOSE — set to anything to stream cargo + bench output. +# +# microVM knobs (predictable numbers via a libkrun guest — see microvm-setup): +# ISOLATION — `local` (default) or `microvm`. +# VM_CPUS — guest vCPUs (default 2). VM_MEM — guest RAM MiB (default 2048). +# VM_IMAGE — override the guest OCI image (default is per-language). + +REF ?= origin/master +BENCH ?= council +ISOLATION ?= +VM_CPUS ?= +VM_MEM ?= +VM_IMAGE ?= -REF ?= origin/master -BENCH ?= council +# Flags appended to `perf-vs` from the microVM knobs above (empty ⇒ host run). +vmflags = $(if $(ISOLATION),--isolation $(ISOLATION),) $(if $(VM_CPUS),--vm-cpus $(VM_CPUS),) $(if $(VM_MEM),--vm-mem $(VM_MEM),) $(if $(VM_IMAGE),--vm-image $(VM_IMAGE),) .PHONY: perf-vs perf-vs: $(TMP) build-rust ## Local A/B perf-compare HEAD vs $(REF) via a sibling worktree. Knobs: REF, BENCH, VERBOSE. $(call say,perf-vs) $(AATXE_BIN) perf-vs --against $(REF) --bench $(BENCH) \ --out-dir $(TMP)/perf-vs/$(BENCH) \ - $(if $(VERBOSE),--verbose,) + $(if $(VERBOSE),--verbose,) $(vmflags) @echo " ✓ perf-vs: see $(TMP)/perf-vs/$(BENCH)/{head,base}.json + cmp.{json,md}" .PHONY: perf-vs-self @@ -260,6 +272,41 @@ perf-vs-self: $(TMP) build-rust ## Smoke test: perf-vs against HEAD~1 (or origin --out-dir $(TMP)/perf-vs/self @echo " ✓ perf-vs-self: see $(TMP)/perf-vs/self/cmp.md" +# ---------------------------------------------------------------------------- +# microVM benches (predictable numbers via a libkrun guest) +# ---------------------------------------------------------------------------- +# +# `--isolation microvm` runs the bench inside an ephemeral libkrun microVM +# (pinned vCPUs, fixed RAM, clean rootfs) so numbers don't drift with host +# load. Works on macOS (Hypervisor.framework) and Linux (KVM). One-time +# `microvm-setup` installs `krunvm`; the first run pulls the guest image. + +.PHONY: perf-vs-vm +perf-vs-vm: ## perf-vs inside a libkrun microVM. Knobs: REF, BENCH, VM_CPUS, VM_MEM, VM_IMAGE. + @$(MAKE) --no-print-directory perf-vs ISOLATION=microvm + +.PHONY: microvm-setup +microvm-setup: ## One-time: install krunvm (the libkrun microVM runner). + $(call say,microvm-setup) + @if [ "$$(uname)" = "Darwin" ]; then \ + command -v brew >/dev/null 2>&1 || { echo " ✗ Homebrew required — see https://brew.sh"; exit 1; }; \ + brew tap slp/krun && brew install krunvm && echo " ✓ krunvm installed — run: make microvm-doctor"; \ + else \ + echo " Linux: install krunvm from your distro (Fedora: dnf install krunvm) or"; \ + echo " build libkrun + krunvm from https://github.com/containers/krunvm"; \ + fi + +.PHONY: microvm-doctor +microvm-doctor: ## Check the microVM backend (krunvm present, host arch, version). + $(call say,microvm-doctor) + @echo " host : $$(uname -m) $$(uname -s)" + @if command -v krunvm >/dev/null 2>&1; then \ + echo " krunvm: $$(krunvm --version 2>/dev/null | head -1)"; \ + echo " ✓ ready — run: make perf-vs-vm (or: aatxe perf-vs --isolation microvm …)"; \ + else \ + echo " ✗ krunvm not found — run: make microvm-setup"; exit 1; \ + fi + # ---------------------------------------------------------------------------- # Local realtime dashboard # ---------------------------------------------------------------------------- diff --git a/README.md b/README.md index d7941b9..c418c71 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,40 @@ aatxe run --lang go --out /tmp/head.json aatxe run --lang rust --out /tmp/head.json ``` +#### Predictable numbers: run in a microVM + +Host noise — background load, another core waking, thermal throttling — is the +enemy of a stable bench number. Pass `--isolation microvm` to run the bench +inside an ephemeral [libkrun](https://github.com/containers/krunvm) microVM +instead of on the host: pinned vCPUs, a fixed RAM budget, and a clean rootfs, +so the same code benches the same way twice. One code path on macOS +(Hypervisor.framework) and Linux (KVM). + +```bash +make microvm-setup # one-time: install krunvm (Homebrew on macOS) +make microvm-doctor # check it's ready + +# both bench entry points take the same knobs: +aatxe run --lang rust --isolation microvm +aatxe perf-vs --against origin/master --isolation microvm +make perf-vs-vm # shorthand for: make perf-vs ISOLATION=microvm +``` + +Only the knobs that actually move the numbers are surfaced; everything else +uses libkrun defaults: + +| flag (or env) | default | meaning | +|---|---|---| +| `--isolation` | `local` | `local` (host) or `microvm` | +| `--vm-cpus` (`AATXE_VM_CPUS`) | `2` | guest vCPUs | +| `--vm-mem` (`AATXE_VM_MEM`) | `2048` | guest RAM (MiB) | +| `--vm-image` (`AATXE_VM_IMAGE`) | `rust:1` / `golang:1` / `node:22` | guest image (must ship the toolchain) | + +The working directory is mounted into the guest at the same path, and the +`cargo` / `go` / `npm` caches persist under `~/.cache/aatxe/vm`, so only the +first run pays the image-pull + cold-build cost. `perf-vs` runs *both* sides +of the A/B in matching guests, keeping the comparison fair. + ### 2½. Trial the gate locally — no CI wiring needed `aatxe baseline save` snapshots a report under `.aatxe/baselines/` diff --git a/crates/aatxe/src/adapter/mod.rs b/crates/aatxe/src/adapter/mod.rs index 41f1a63..139cbae 100644 --- a/crates/aatxe/src/adapter/mod.rs +++ b/crates/aatxe/src/adapter/mod.rs @@ -14,6 +14,7 @@ //! so downstream consumers see identical statistics regardless of language. use crate::ast::ast_import_extractor::AstImportExtractor; +use crate::sandbox::Isolation; use aatxe_core::affected::{resolve_affected, AffectedOptions}; use aatxe_core::types::{AffectedScope as CoreScope, Language, RunReport}; use anyhow::{Context, Result}; @@ -37,6 +38,8 @@ pub struct RunSpec { /// When `Some`, restrict the run to exactly these bench files (no discovery). pub bench_files: Option>, pub verbose: bool, + /// Where the runner executes — host or microVM. + pub isolation: Isolation, } pub fn execute(lang: Language, spec: &RunSpec) -> Result { diff --git a/crates/aatxe/src/adapter/ts_or_runner.rs b/crates/aatxe/src/adapter/ts_or_runner.rs index f060b6d..37627b3 100644 --- a/crates/aatxe/src/adapter/ts_or_runner.rs +++ b/crates/aatxe/src/adapter/ts_or_runner.rs @@ -6,13 +6,27 @@ //! * **TS**: `AATXE_TS_RUNNER`, default `npx --no-install aatxe-ts-runner` //! * **Go**: `AATXE_GO_RUNNER`, default `go test -bench=. -benchmem -json ./...` //! * **Rust**: `AATXE_RUST_RUNNER`, default `cargo run --release -q --bin aatxe-rust-runner -- --json` +//! +//! The command runs under [`crate::sandbox::Isolation`], so the same code +//! path serves both a plain host run and an in-microVM run — the caller +//! picks via `--isolation`. Either way we capture stdout (the RunReport) and +//! let stderr stream through. use crate::adapter::RunSpec; +use crate::sandbox; use aatxe_core::types::Language; use anyhow::{anyhow, Context, Result}; -use std::process::{Command, Stdio}; pub fn run_runner(spec: &RunSpec, lang: Language) -> Result { + // Streaming stdout and ingesting a RunReport off stdout are mutually + // exclusive regardless of isolation — bail early rather than run first. + if spec.verbose { + return Err(anyhow!( + "--verbose is incompatible with stdout-based RunReport ingestion; \ + re-run without --verbose or pipe through the runner's --out flag" + )); + } + let (env_key, default) = match lang { Language::Ts => ("AATXE_TS_RUNNER", "npx --no-install aatxe-ts-runner"), Language::Go => ( @@ -29,45 +43,32 @@ pub fn run_runner(spec: &RunSpec, lang: Language) -> Result { let Some((program, args)) = tokens.split_first() else { return Err(anyhow!("empty runner command for {:?}", lang)); }; - let mut cmd = Command::new(program); - cmd.args(args); - cmd.current_dir(&spec.cwd); + + let mut env: Vec<(String, String)> = Vec::new(); if let Some(filter) = spec.filter.as_ref() { - cmd.env("AATXE_FILTER", filter); + env.push(("AATXE_FILTER".to_string(), filter.clone())); } if !spec.patterns.is_empty() { - cmd.env("AATXE_PATTERNS", spec.patterns.join(":")); + env.push(("AATXE_PATTERNS".to_string(), spec.patterns.join(":"))); } if let Some(files) = spec.bench_files.as_ref() { let joined: Vec = files.iter().map(|p| p.display().to_string()).collect(); - cmd.env("AATXE_BENCH_FILES", joined.join(":")); - } - cmd.env("AATXE_SERVICE", &spec.service); - cmd.env("AATXE_REF", &spec.r#ref); - let verbose = spec.verbose; - cmd.stdin(Stdio::null()); - if verbose { - cmd.stdout(Stdio::inherit()); - } else { - cmd.stdout(Stdio::piped()); + env.push(("AATXE_BENCH_FILES".to_string(), joined.join(":"))); } - cmd.stderr(Stdio::inherit()); + env.push(("AATXE_SERVICE".to_string(), spec.service.clone())); + env.push(("AATXE_REF".to_string(), spec.r#ref.clone())); - let output = cmd - .output() - .with_context(|| format!("spawning runner: {}", raw))?; + let script = sandbox::exec_line(program, args); + let output = spec + .isolation + .run_script(&spec.cwd, &script, &env) + .with_context(|| format!("running bench runner: {raw}"))?; if !output.status.success() { return Err(anyhow!( "runner exited with status {}", output.status.code().unwrap_or(-1) )); } - if verbose { - return Err(anyhow!( - "--verbose is incompatible with stdout-based RunReport ingestion; \ - re-run without --verbose or pipe through the runner's --out flag" - )); - } - let raw = String::from_utf8(output.stdout).context("runner stdout was not valid UTF-8")?; - Ok(raw) + let raw_out = String::from_utf8(output.stdout).context("runner stdout was not valid UTF-8")?; + Ok(raw_out) } diff --git a/crates/aatxe/src/cli.rs b/crates/aatxe/src/cli.rs index 3124df5..1b700fe 100644 --- a/crates/aatxe/src/cli.rs +++ b/crates/aatxe/src/cli.rs @@ -62,6 +62,38 @@ pub enum Command { Ui(UiArgs), } +/// Where benches execute — the backend for `--isolation`. +#[derive(Copy, Clone, Debug, ValueEnum, PartialEq, Eq)] +pub enum IsolationArg { + /// Run on the host, directly. Fast, but subject to host noise + /// (background load, thermal throttling, other processes). + Local, + /// Run inside an ephemeral libkrun microVM via `krunvm` — pinned + /// vCPUs, fixed RAM, clean rootfs — for reproducible numbers. Needs + /// `krunvm` installed (`make microvm-setup`); works on macOS and Linux. + Microvm, +} + +/// Shared microVM knobs, flattened into every bench-running subcommand. +/// Only the parameters that meaningfully change results are surfaced; +/// everything else uses libkrun defaults. +#[derive(clap::Args, Debug, Clone)] +pub struct VmOpts { + /// Where to run the bench. `local` (default) or `microvm`. + #[arg(long, value_enum, default_value_t = IsolationArg::Local)] + pub isolation: IsolationArg, + /// microVM vCPUs (only with `--isolation microvm`). + #[arg(long, env = "AATXE_VM_CPUS", default_value_t = crate::sandbox::DEFAULT_VM_CPUS)] + pub vm_cpus: u32, + /// microVM RAM in MiB (only with `--isolation microvm`). + #[arg(long, env = "AATXE_VM_MEM", default_value_t = crate::sandbox::DEFAULT_VM_MEM_MIB)] + pub vm_mem: u32, + /// OCI image for the guest (must ship the language toolchain). Defaults + /// per language: `rust:1` / `golang:1` / `node:22`. + #[arg(long, env = "AATXE_VM_IMAGE")] + pub vm_image: Option, +} + #[derive(Copy, Clone, Debug, ValueEnum, PartialEq, Eq)] pub enum PerfBenchArg { /// `examples/council-bench` — 9 micros covering the council's pure @@ -129,6 +161,10 @@ pub struct PerfVsArgs { /// the final summary prints). #[arg(long)] pub verbose: bool, + /// microVM isolation knobs (`--isolation microvm` for predictable + /// numbers). Both sides of the A/B run in matching guests. + #[command(flatten)] + pub vm: VmOpts, } #[derive(Copy, Clone, Debug, ValueEnum, PartialEq, Eq)] @@ -177,6 +213,10 @@ pub struct RunArgs { /// Print runner stdout to the caller as it produces it. #[arg(long)] pub verbose: bool, + /// microVM isolation knobs. `--isolation microvm` runs the language + /// runner inside a libkrun guest for predictable numbers. + #[command(flatten)] + pub vm: VmOpts, /// Bench discovery globs (passed to the adapter). Defaults are language-specific. pub patterns: Vec, } diff --git a/crates/aatxe/src/commands/perf_vs.rs b/crates/aatxe/src/commands/perf_vs.rs index bff712d..98c4217 100644 --- a/crates/aatxe/src/commands/perf_vs.rs +++ b/crates/aatxe/src/commands/perf_vs.rs @@ -15,6 +15,7 @@ use crate::cli::{PerfBenchArg, PerfVsArgs}; use crate::commands::Outcome; +use crate::sandbox::Isolation; use aatxe_core::types::{BenchRun, Language, RunReport, SCHEMA_VERSION}; use aatxe_core::{compare_reports, has_regressions, render_markdown, CompareOptions}; use anyhow::{anyhow, bail, Context, Result}; @@ -54,9 +55,14 @@ pub fn execute(args: PerfVsArgs) -> Result { ensure_worktree(&repo_root, &wt_path, &base_sha, args.verbose)?; + // Both sides run under the same isolation so the comparison stays fair. + // perf-vs only benches aatxe's own (Rust) binaries, so the guest image + // defaults to the Rust toolchain image. + let iso = Isolation::from_opts(&args.vm, Language::Rust)?; + let benches = expand_bench(args.bench); eprintln!( - "perf-vs: HEAD {} vs {} ({}) · bench: {} · worktree: {}", + "perf-vs: HEAD {} vs {} ({}) · bench: {} · isolation: {} · worktree: {}", short(&head_sha), args.against, short(&base_sha), @@ -65,11 +71,13 @@ pub fn execute(args: PerfVsArgs) -> Result { .map(|b| bench_slug(*b)) .collect::>() .join("+"), + if iso.is_microvm() { "microvm" } else { "local" }, wt_path.display() ); - let head_report = build_and_run_all(&repo_root, &benches, "head", &head_sha, args.verbose)?; - let base_report = build_and_run_all(&wt_path, &benches, "base", &base_sha, args.verbose)?; + let head_report = + build_and_run_all(&repo_root, &benches, "head", &head_sha, &iso, args.verbose)?; + let base_report = build_and_run_all(&wt_path, &benches, "base", &base_sha, &iso, args.verbose)?; let head_json = out_dir.join("head.json"); let base_json = out_dir.join("base.json"); @@ -239,11 +247,12 @@ fn build_and_run_all( benches: &[PerfBenchArg], side: &str, sha: &str, + iso: &Isolation, verbose: bool, ) -> Result { let mut merged: Option = None; for b in benches { - let report = build_and_run(cwd, *b, side, sha, verbose) + let report = build_and_run(cwd, *b, side, sha, iso, verbose) .with_context(|| format!("running {} on {}", bench_slug(*b), side))?; match merged.as_mut() { None => merged = Some(report), @@ -261,9 +270,48 @@ fn build_and_run( bench: PerfBenchArg, side: &str, sha: &str, + iso: &Isolation, verbose: bool, ) -> Result { let bin = bench_binary(bench); + let stdout_str = if iso.is_microvm() { + build_and_run_microvm(cwd, bin, bench, side, iso)? + } else { + build_and_run_host(cwd, bin, bench, side, verbose)? + }; + + // Big-diff-bench interleaves `METRIC k=v` lines (for the autoresearch + // loop) with the trailing RunReport JSON; council-bench emits the JSON + // directly. Strip any leading non-JSON lines before parsing so both + // protocols round-trip through the same code path. + let json_slice = strip_non_json_prefix(&stdout_str); + let mut report: RunReport = serde_json::from_str(json_slice).with_context(|| { + format!( + "parsing RunReport from {} stdout (first 200 bytes: {:?})", + bin, + &stdout_str[..stdout_str.len().min(200)] + ) + })?; + // Stamp the ref we're actually benching. The bench binary uses + // env-driven defaults; overriding here keeps the report honest if + // someone runs it outside this command. + report.r#ref = short(sha).to_string(); + if report.schema_version == 0 { + report.schema_version = SCHEMA_VERSION; + } + sanity_check_report(&report, bench)?; + Ok(report) +} + +/// Build + run the bench directly on the host (the default path). Returns +/// the runner's captured stdout. +fn build_and_run_host( + cwd: &Path, + bin: &str, + bench: PerfBenchArg, + side: &str, + verbose: bool, +) -> Result { eprintln!("perf-vs: building {} ({})", bin, side); let build_status = Command::new("cargo") .args(["build", "--release", "--bin", bin]) @@ -306,28 +354,32 @@ fn build_and_run( String::from_utf8_lossy(&output.stderr).trim() ); } - // Big-diff-bench interleaves `METRIC k=v` lines (for the autoresearch - // loop) with the trailing RunReport JSON; council-bench emits the JSON - // directly. Strip any leading non-JSON lines before parsing so both - // protocols round-trip through the same code path. - let stdout_str = String::from_utf8_lossy(&output.stdout); - let json_slice = strip_non_json_prefix(&stdout_str); - let mut report: RunReport = serde_json::from_str(json_slice).with_context(|| { - format!( - "parsing RunReport from {} stdout (first 200 bytes: {:?})", - bin, - &stdout_str[..stdout_str.len().min(200)] - ) - })?; - // Stamp the ref we're actually benching. The bench binary uses - // env-driven defaults; overriding here keeps the report honest if - // someone runs it outside this command. - report.r#ref = short(sha).to_string(); - if report.schema_version == 0 { - report.schema_version = SCHEMA_VERSION; + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +/// Build + run the bench inside a microVM. Both steps happen in one guest +/// invocation so the binary is compiled for the guest, not the host. The +/// sandbox points `CARGO_TARGET_DIR` at the persistent cache mount, so the +/// freshly built binary lives at `$CARGO_TARGET_DIR/release/`. +fn build_and_run_microvm( + cwd: &Path, + bin: &str, + bench: PerfBenchArg, + side: &str, + iso: &Isolation, +) -> Result { + eprintln!("perf-vs: building+running {} ({}) in microVM", bin, side); + // `bin` is a fixed identifier (see `bench_binary`), safe to interpolate. + let script = + format!("cargo build --release --bin {bin} && exec \"$CARGO_TARGET_DIR/release/{bin}\""); + let env = vec![("AATXE_SERVICE".to_string(), service_for(bench).to_string())]; + let output = iso + .run_script(cwd, &script, &env) + .with_context(|| format!("running {bin} in microVM ({side})"))?; + if !output.status.success() { + bail!("{} in microVM exited with status {}", bin, output.status); } - sanity_check_report(&report, bench)?; - Ok(report) + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } fn service_for(bench: PerfBenchArg) -> &'static str { diff --git a/crates/aatxe/src/commands/run.rs b/crates/aatxe/src/commands/run.rs index 22e77ef..44048b2 100644 --- a/crates/aatxe/src/commands/run.rs +++ b/crates/aatxe/src/commands/run.rs @@ -7,6 +7,7 @@ use crate::adapter::{self, RunSpec}; use crate::cli::RunArgs; +use crate::sandbox::Isolation; use aatxe_core::types::AffectedScope; use anyhow::{Context, Result}; use std::fs; @@ -46,6 +47,7 @@ pub fn execute(args: RunArgs) -> Result<()> { .as_ref() .map(|s| s.bench_files.iter().map(PathBuf::from).collect()); + let isolation = Isolation::from_opts(&args.vm, lang)?; let spec = RunSpec { cwd: cwd.clone(), service, @@ -54,6 +56,7 @@ pub fn execute(args: RunArgs) -> Result<()> { patterns: args.patterns.clone(), bench_files, verbose: args.verbose, + isolation, }; let mut report = adapter::execute(lang, &spec)?; diff --git a/crates/aatxe/src/main.rs b/crates/aatxe/src/main.rs index 1b11f48..29aa9e2 100644 --- a/crates/aatxe/src/main.rs +++ b/crates/aatxe/src/main.rs @@ -12,6 +12,7 @@ mod commands; mod curator; mod github; mod llm; +mod sandbox; use std::process::ExitCode; diff --git a/crates/aatxe/src/sandbox.rs b/crates/aatxe/src/sandbox.rs new file mode 100644 index 0000000..fc9191b --- /dev/null +++ b/crates/aatxe/src/sandbox.rs @@ -0,0 +1,459 @@ +//! Execution isolation for bench runs. +//! +//! A bench command runs in one of two places: +//! +//! * [`Isolation::Local`] — directly on the host. The default; today's +//! behaviour, unchanged. +//! * [`Isolation::Microvm`] — inside an ephemeral libkrun microVM driven by +//! the [`krunvm`](https://github.com/containers/krunvm) CLI. Pinned vCPUs, +//! a fixed RAM budget, and a clean rootfs with no host noise give far more +//! *predictable* bench numbers — which is the whole point of benching in a +//! VM. Works on macOS (Hypervisor.framework) and Linux (KVM) with the same +//! code path. +//! +//! We shell out to the `krunvm` binary rather than link `libkrun`, so aatxe +//! gains **no** new build-time dependency and no FFI. The tradeoff is that +//! `krunvm` must be installed (`make microvm-setup`); when it isn't, the +//! microVM path fails with an actionable install hint instead of a cryptic +//! spawn error. +//! +//! ## How a guest run is wired +//! +//! `krunvm create` bakes the VM's resources + volume mounts, then +//! `krunvm start` execs a command inside it. We: +//! +//! 1. mount the working directory at the *same* absolute path in the guest, +//! so paths in the command line resolve identically; +//! 2. mount a persistent host cache dir at [`GUEST_CACHE_MOUNT`] and point +//! `CARGO_TARGET_DIR` / `CARGO_HOME` / `GOCACHE` / … at it. This keeps +//! guest builds incremental across runs *and* off the host's own +//! `target/` (which is a different architecture — sharing it would +//! corrupt both); +//! 3. exec the command through `/bin/sh` with a normalised `PATH` so the +//! language toolchains in the stock `rust` / `golang` / `node` images are +//! always found. +//! +//! The VM is deleted on drop ([`VmGuard`]) so runs leave nothing behind. + +use crate::cli::{IsolationArg, VmOpts}; +use aatxe_core::types::Language; +use anyhow::{bail, Context, Result}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Default guest vCPU count when `--vm-cpus` / `AATXE_VM_CPUS` is unset. +pub const DEFAULT_VM_CPUS: u32 = 2; +/// Default guest RAM (MiB) when `--vm-mem` / `AATXE_VM_MEM` is unset. +pub const DEFAULT_VM_MEM_MIB: u32 = 2048; + +/// Where the persistent build-cache volume is mounted inside the guest. +pub const GUEST_CACHE_MOUNT: &str = "/aatxe-cache"; + +/// A `PATH` covering the toolchain locations of the stock language images +/// (`/usr/local/cargo/bin` for rust, `/usr/local/go/bin` for golang, …) +/// plus the standard dirs. Prepended to the image's own `$PATH` so a +/// custom `--vm-image` still works. +const GUEST_PATH: &str = + "/usr/local/cargo/bin:/usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// Where a bench command executes. +#[derive(Clone, Debug)] +pub enum Isolation { + /// Directly on the host. + Local, + /// Inside an ephemeral libkrun microVM. + Microvm(MicrovmConfig), +} + +/// Resolved microVM parameters. Constructed from [`VmOpts`] via +/// [`Isolation::from_opts`]. +#[derive(Clone, Debug)] +pub struct MicrovmConfig { + /// OCI image for the guest rootfs. Must carry the language toolchain. + pub image: String, + /// Guest vCPUs. + pub cpus: u32, + /// Guest RAM in MiB. + pub mem_mib: u32, + /// Host directory mounted at [`GUEST_CACHE_MOUNT`] to persist build + /// caches between runs. + pub cache_dir: PathBuf, +} + +/// The default OCI image for a language — a stock upstream image that ships +/// the toolchain (and a C compiler, needed by e.g. tree-sitter crates). +pub fn default_image(lang: Language) -> &'static str { + match lang { + Language::Rust => "docker.io/library/rust:1", + Language::Go => "docker.io/library/golang:1", + Language::Ts => "docker.io/library/node:22", + } +} + +impl Isolation { + /// Resolve CLI/env options into an [`Isolation`]. `lang` selects the + /// default guest image when `--vm-image` is not given. + pub fn from_opts(opts: &VmOpts, lang: Language) -> Result { + match opts.isolation { + IsolationArg::Local => Ok(Isolation::Local), + IsolationArg::Microvm => Ok(Isolation::Microvm(MicrovmConfig { + image: opts + .vm_image + .clone() + .unwrap_or_else(|| default_image(lang).to_string()), + cpus: opts.vm_cpus, + mem_mib: opts.vm_mem, + cache_dir: default_cache_dir()?, + })), + } + } + + pub fn is_microvm(&self) -> bool { + matches!(self, Isolation::Microvm(_)) + } + + /// Run `script` under `/bin/sh -c` in `cwd` with `env` set, capturing + /// stdout. `stderr` streams to the caller's terminal (build logs), so + /// the returned [`Output`] carries only stdout. + /// + /// * `Local` runs the script on the host shell — equivalent to spawning + /// the program directly, but lets both callers share one entry point. + /// * `Microvm` runs it inside a fresh guest. + pub fn run_script(&self, cwd: &Path, script: &str, env: &[(String, String)]) -> Result { + match self { + Isolation::Local => run_local(cwd, script, env), + Isolation::Microvm(cfg) => run_microvm(cfg, cwd, script, env), + } + } +} + +fn run_local(cwd: &Path, script: &str, env: &[(String, String)]) -> Result { + let mut cmd = Command::new("/bin/sh"); + cmd.arg("-c") + .arg(script) + .current_dir(cwd) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + for (k, v) in env { + cmd.env(k, v); + } + cmd.output().context("spawning `/bin/sh -c` (local)") +} + +fn run_microvm( + cfg: &MicrovmConfig, + cwd: &Path, + script: &str, + env: &[(String, String)], +) -> Result { + let krunvm = ensure_krunvm()?; + std::fs::create_dir_all(&cfg.cache_dir) + .with_context(|| format!("creating microVM cache dir {}", cfg.cache_dir.display()))?; + + let name = unique_vm_name(); + eprintln!( + "microvm: booting guest {name} · image {} · {} vCPU · {} MiB \ + (first run pulls the image — this can take a few minutes)", + cfg.image, cfg.cpus, cfg.mem_mib + ); + + let create = create_argv(cfg, &name, cwd); + let status = Command::new(&krunvm) + .args(&create) + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status() + .context("spawning `krunvm create`")?; + if !status.success() { + bail!( + "`krunvm create` failed ({status}) — is the image '{}' reachable and is \ + virtualization available? Try `make microvm-doctor`.", + cfg.image + ); + } + // From here on the VM exists; delete it no matter how we leave. + let _guard = VmGuard { + name: name.clone(), + krunvm: krunvm.clone(), + }; + + // Cache-redirect env wins nothing over the caller's explicit env + // (no keys overlap), but is listed first so caller env can override. + let mut full_env = cache_env(); + full_env.extend_from_slice(env); + + let guest = guest_argv(script, &full_env); + let output = Command::new(&krunvm) + .arg("start") + .arg(&name) + .args(&guest) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .output() + .context("spawning `krunvm start`")?; + Ok(output) +} + +/// Build the `krunvm create …` argument vector. Options first, image last. +fn create_argv(cfg: &MicrovmConfig, name: &str, cwd: &Path) -> Vec { + let cwd = cwd.display().to_string(); + vec![ + "create".to_string(), + "--name".to_string(), + name.to_string(), + "--cpus".to_string(), + cfg.cpus.to_string(), + "--mem".to_string(), + cfg.mem_mib.to_string(), + "--workdir".to_string(), + cwd.clone(), + // Repo/worktree at the same path so command lines resolve 1:1. + "--volume".to_string(), + format!("{cwd}:{cwd}"), + // Persistent build cache. + "--volume".to_string(), + format!("{}:{}", cfg.cache_dir.display(), GUEST_CACHE_MOUNT), + cfg.image.clone(), + ] +} + +/// The tokens that follow `krunvm start `: run the script through +/// `/usr/bin/env` (to set vars) and `/bin/sh -c` (to normalise `PATH`). +fn guest_argv(script: &str, env: &[(String, String)]) -> Vec { + let mut v = vec!["/usr/bin/env".to_string()]; + for (k, val) in env { + v.push(format!("{k}={val}")); + } + v.push("/bin/sh".to_string()); + v.push("-c".to_string()); + v.push(format!("export PATH=\"{GUEST_PATH}:$PATH\"\n{script}")); + v +} + +/// Guest environment redirecting each toolchain's cache into the persistent +/// mount. Harmless when a given toolchain isn't used. +fn cache_env() -> Vec<(String, String)> { + let c = GUEST_CACHE_MOUNT; + vec![ + ("CARGO_HOME".to_string(), format!("{c}/cargo")), + ("CARGO_TARGET_DIR".to_string(), format!("{c}/target")), + ("GOCACHE".to_string(), format!("{c}/go-build")), + ("GOMODCACHE".to_string(), format!("{c}/go-mod")), + ("npm_config_cache".to_string(), format!("{c}/npm")), + ] +} + +/// POSIX single-quote a token for safe interpolation into a shell string. +pub fn sh_quote(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + out.push('\''); + for c in s.chars() { + if c == '\'' { + out.push_str("'\\''"); + } else { + out.push(c); + } + } + out.push('\''); + out +} + +/// Join a program + args into a single `exec`-able shell command, each token +/// individually quoted. Used to lower a `program args…` runner invocation +/// into the script that [`Isolation::run_script`] expects. +pub fn exec_line(program: &str, args: &[&str]) -> String { + let mut line = String::from("exec "); + line.push_str(&sh_quote(program)); + for a in args { + line.push(' '); + line.push_str(&sh_quote(a)); + } + line +} + +fn default_cache_dir() -> Result { + let home = std::env::var_os("HOME") + .context("HOME is not set — cannot locate the microVM cache dir")?; + Ok(PathBuf::from(home).join(".cache/aatxe/vm")) +} + +/// Probe for the `krunvm` binary, returning an actionable error when it's +/// missing so the microVM path never dies on a bare "No such file". +fn ensure_krunvm() -> Result { + match Command::new("krunvm") + .arg("--version") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(_) => Ok(PathBuf::from("krunvm")), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => bail!( + "`krunvm` not found on PATH — the microVM isolation backend is not installed.\n\ + Install it once with:\n make microvm-setup\n\ + (brew tap slp/krun && brew install krunvm), then re-run with \ + `--isolation microvm`. Check readiness with `make microvm-doctor`." + ), + Err(e) => Err(anyhow::Error::new(e).context("probing for `krunvm`")), + } +} + +static VM_SEQ: AtomicU64 = AtomicU64::new(0); + +fn unique_vm_name() -> String { + format!( + "aatxe-{}-{}", + std::process::id(), + VM_SEQ.fetch_add(1, Ordering::Relaxed) + ) +} + +/// Deletes its microVM when dropped, so a run leaves no VMs behind even on +/// error or panic. +struct VmGuard { + name: String, + krunvm: PathBuf, +} + +impl Drop for VmGuard { + fn drop(&mut self) { + let _ = Command::new(&self.krunvm) + .args(["delete", &self.name]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cli::VmOpts; + + fn micro_opts() -> VmOpts { + VmOpts { + isolation: IsolationArg::Microvm, + vm_cpus: DEFAULT_VM_CPUS, + vm_mem: DEFAULT_VM_MEM_MIB, + vm_image: None, + } + } + + #[test] + fn from_opts_local_is_local() { + let opts = VmOpts { + isolation: IsolationArg::Local, + vm_cpus: 4, + vm_mem: 4096, + vm_image: Some("ignored".into()), + }; + assert!(!Isolation::from_opts(&opts, Language::Rust) + .unwrap() + .is_microvm()); + } + + #[test] + fn from_opts_microvm_defaults_image_per_language() { + let iso = Isolation::from_opts(µ_opts(), Language::Go).unwrap(); + match iso { + Isolation::Microvm(cfg) => { + assert_eq!(cfg.image, "docker.io/library/golang:1"); + assert_eq!(cfg.cpus, DEFAULT_VM_CPUS); + assert_eq!(cfg.mem_mib, DEFAULT_VM_MEM_MIB); + } + Isolation::Local => panic!("expected microvm"), + } + } + + #[test] + fn from_opts_microvm_honours_explicit_image_and_resources() { + let opts = VmOpts { + isolation: IsolationArg::Microvm, + vm_cpus: 8, + vm_mem: 8192, + vm_image: Some("my/custom:tag".into()), + }; + match Isolation::from_opts(&opts, Language::Rust).unwrap() { + Isolation::Microvm(cfg) => { + assert_eq!(cfg.image, "my/custom:tag"); + assert_eq!(cfg.cpus, 8); + assert_eq!(cfg.mem_mib, 8192); + } + Isolation::Local => panic!("expected microvm"), + } + } + + #[test] + fn default_image_covers_every_language() { + assert_eq!(default_image(Language::Rust), "docker.io/library/rust:1"); + assert_eq!(default_image(Language::Go), "docker.io/library/golang:1"); + assert_eq!(default_image(Language::Ts), "docker.io/library/node:22"); + } + + #[test] + fn create_argv_puts_options_first_image_last() { + let cfg = MicrovmConfig { + image: "docker.io/library/rust:1".into(), + cpus: 2, + mem_mib: 2048, + cache_dir: PathBuf::from("/home/u/.cache/aatxe/vm"), + }; + let argv = create_argv(&cfg, "aatxe-1-0", Path::new("/repo/aatxe")); + assert_eq!(argv[0], "create"); + assert_eq!(argv.last().unwrap(), "docker.io/library/rust:1"); + // Working dir mounted 1:1. + assert!(argv.contains(&"/repo/aatxe:/repo/aatxe".to_string())); + // Cache volume mounted at the well-known guest path. + assert!(argv.contains(&format!("/home/u/.cache/aatxe/vm:{GUEST_CACHE_MOUNT}"))); + assert!(argv.contains(&"--cpus".to_string())); + assert!(argv.contains(&"2".to_string())); + } + + #[test] + fn guest_argv_sets_env_then_shells_with_path_fix() { + let env = vec![("AATXE_SERVICE".to_string(), "svc".to_string())]; + let argv = guest_argv("exec ./bench", &env); + assert_eq!(argv[0], "/usr/bin/env"); + assert_eq!(argv[1], "AATXE_SERVICE=svc"); + assert_eq!(argv[2], "/bin/sh"); + assert_eq!(argv[3], "-c"); + assert!(argv[4].contains("export PATH=")); + assert!(argv[4].contains("/usr/local/cargo/bin")); + assert!(argv[4].trim_end().ends_with("exec ./bench")); + } + + #[test] + fn cache_env_redirects_all_toolchains_into_the_mount() { + let env = cache_env(); + for key in [ + "CARGO_HOME", + "CARGO_TARGET_DIR", + "GOCACHE", + "GOMODCACHE", + "npm_config_cache", + ] { + let (_, v) = env.iter().find(|(k, _)| k == key).expect("key present"); + assert!(v.starts_with(GUEST_CACHE_MOUNT), "{key} -> {v}"); + } + } + + #[test] + fn sh_quote_wraps_and_escapes_single_quotes() { + assert_eq!(sh_quote("plain"), "'plain'"); + assert_eq!(sh_quote("a b"), "'a b'"); + assert_eq!(sh_quote("it's"), "'it'\\''s'"); + } + + #[test] + fn exec_line_quotes_each_token() { + assert_eq!( + exec_line("cargo", &["run", "--release", "--bin", "x"]), + "exec 'cargo' 'run' '--release' '--bin' 'x'" + ); + } +}