define and enforce max threshold for agentd boot sequence in CI

.github/workflows/check.yml

@@ -302,3 +302,65 @@ jobs:
           export PATH="$HOME/.microsandbox/bin:$PATH"
           export LD_LIBRARY_PATH="${{ github.workspace }}/build:$HOME/.microsandbox/lib"
           npm test
+
+  # ---------------------------------------------------------------------------
+  # Boot timing regression gate (requires KVM)
+  # ---------------------------------------------------------------------------
+  boot-timing:
+    name: Boot Timing
+    needs: check
+    runs-on: self-hosted-ubuntu-2404-x64
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Install build deps
+        run: sudo apt-get update && sudo apt-get install -y musl-tools libcap-ng-dev gcc make flex bison libelf-dev bc python3-pyelftools libdbus-1-dev
+
+      - name: Build agentd
+        run: |
+          rustup target add x86_64-unknown-linux-musl
+          cargo build --release --manifest-path crates/agentd/Cargo.toml --target x86_64-unknown-linux-musl
+          mkdir -p build
+          cp crates/agentd/target/x86_64-unknown-linux-musl/release/agentd build/agentd
+
+      - name: Build libkrunfw
+        run: |
+          cd vendor/libkrunfw
+          make -j$(nproc)
+          cd ../..
+          mkdir -p build
+          cp vendor/libkrunfw/libkrunfw.so.${{ env.LIBKRUNFW_VERSION }} build/
+          cd build
+          ln -sf libkrunfw.so.${{ env.LIBKRUNFW_VERSION }} libkrunfw.so.${{ env.LIBKRUNFW_ABI }}
+          ln -sf libkrunfw.so.${{ env.LIBKRUNFW_ABI }} libkrunfw.so
+
+      - name: Build msb
+        run: |
+          cargo build --release --no-default-features --features net -p microsandbox-cli
+          mkdir -p build
+          cp target/release/msb build/msb
+
+      - name: Build boot timing benchmark
+        run: cargo build --release -p boot-timing-ci
+
+      - name: Check boot timing thresholds
+        run: |
+          export MSB_PATH="${{ github.workspace }}/build/msb"
+          export LD_LIBRARY_PATH="${{ github.workspace }}/build:$HOME/.microsandbox/lib"
+          python3 scripts/ci/check_boot_timings.py \
+            --binary target/release/boot-timing-ci \
+            --config scripts/ci/boot-timing-thresholds.json \
+            --output boot-timing-results.json
+
+      - name: Upload boot timing results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: boot-timing-results
+          path: boot-timing-results.json

Cargo.toml

@@ -11,6 +11,7 @@ members = [
     "crates/protocol",
     "crates/runtime",
     "crates/utils",
+    "examples/rust/boot-timing-ci",
     "examples/rust/fs-read-stream",
     "examples/rust/shell-attach",
     "examples/rust/metrics-stream",

crates/filesystem/build.rs

@@ -29,15 +29,14 @@ fn build_agentd(workspace_root: &Path, out_dir: &Path) {
             return;
         }
 
-        // In CI, prefer the locally-built agentd from workspace build/.
+       // In CI, prefer the locally-built agentd from workspace build/.
         if std::env::var_os("CI").is_some() {
             let local = workspace_root.join("build").join(AGENTD_BINARY);
             if local.is_file() {
                 std::fs::copy(&local, &dest).expect("failed to copy agentd from build/");
                 return;
             }
         }
-
         let _ = workspace_root;
         let arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap();
         let url = agentd_download_url(PREBUILT_VERSION, &arch);

crates/microsandbox/build.rs

@@ -60,6 +60,7 @@ fn main() {
         return;
     }
 
+
     let url = bundle_url();
     println!(
         "cargo:warning=downloading microsandbox runtime dependencies (v{PREBUILT_VERSION})..."
@@ -136,7 +137,6 @@ fn install_ci_local_bundle(
     if std::env::var_os("CI").is_none() {
         return Ok(false);
     }
-
     let Some(build_dir) = workspace_build_dir() else {
         return Ok(false);
     };
@@ -161,7 +161,7 @@ fn install_ci_local_bundle(
     }
 
     create_symlinks(lib_dir, libkrunfw_name);
-    println!("cargo:warning=installed microsandbox runtime dependencies from local CI build/");
+    println!("cargo:warning=installed microsandbox runtime dependencies from local build/");
     Ok(true)
 }
 

crates/microsandbox/lib/lib.rs

@@ -26,5 +26,5 @@ pub use microsandbox_utils::size;
 #[cfg(feature = "net")]
 pub use sandbox::NetworkPolicy;
 pub use sandbox::exec::{ExecEvent, ExecHandle};
-pub use sandbox::{ExecOutput, Sandbox, SandboxConfig};
+pub use sandbox::{BootTimings, ExecOutput, Sandbox, SandboxConfig};
 pub use volume::Volume;

crates/microsandbox/lib/sandbox/mod.rs

@@ -80,6 +80,33 @@ pub struct Sandbox {
     client: Arc<AgentClient>,
 }
 
+/// Boot timing data reported by the guest in `core.ready`.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct BootTimings {
+    /// Guest `CLOCK_BOOTTIME` at the start of `agentd::main()`.
+    ///
+    /// In practice this tracks the time from VM entry until the guest reaches
+    /// userspace and starts `agentd`.
+    pub enter_to_boot_ns: u64,
+    /// Nanoseconds spent in synchronous guest init before the async agent loop.
+    pub boot_to_init_ns: u64,
+    /// Nanoseconds from `agentd::main()` start until `core.ready` was sent.
+    pub boot_to_ready_ns: u64,
+    /// Total guest time from kernel boot until `core.ready`.
+    pub enter_to_ready_ns: u64,
+}
+
+impl BootTimings {
+    fn from_ready(ready: &microsandbox_protocol::core::Ready) -> Self {
+        Self {
+            enter_to_boot_ns: ready.boot_time_ns,
+            boot_to_init_ns: ready.init_time_ns,
+            boot_to_ready_ns: ready.ready_time_ns.saturating_sub(ready.boot_time_ns),
+            enter_to_ready_ns: ready.ready_time_ns,
+        }
+    }
+}
+
 //--------------------------------------------------------------------------------------------------
 // Methods: Static
 //--------------------------------------------------------------------------------------------------
@@ -422,6 +449,11 @@ impl Sandbox {
         fs::SandboxFs::new(&self.client)
     }
 
+    /// Return the cached guest boot timing snapshot captured during startup.
+    pub fn boot_timings(&self) -> BootTimings {
+        BootTimings::from_ready(self.client.ready())
+    }
+
     /// Stop the sandbox gracefully by sending `core.shutdown` to agentd.
     pub async fn stop(&self) -> MicrosandboxResult<()> {
         tracing::debug!(sandbox = %self.config.name, "stop: sending shutdown");
@@ -1706,11 +1738,12 @@ mod tests {
         image as image_entity, run as run_entity, sandbox_image as sandbox_image_entity,
     };
     use microsandbox_migration::{Migrator, MigratorTrait};
+    use microsandbox_protocol::core::Ready;
     use sea_orm::{ColumnTrait, ConnectOptions, Database, EntityTrait, QueryFilter, Set};
     use tempfile::tempdir;
 
     use super::{
-        RootfsSource, SandboxConfig, SandboxStatus, insert_sandbox_record,
+        BootTimings, RootfsSource, SandboxConfig, SandboxStatus, insert_sandbox_record,
         persist_oci_manifest_pin, prepare_create_target, reconcile_sandbox_runtime_state,
         remove_dir_if_exists, validate_rootfs_source,
     };
@@ -1800,6 +1833,20 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_boot_timings_from_ready() {
+        let timings = BootTimings::from_ready(&Ready {
+            boot_time_ns: 70_000_000,
+            init_time_ns: 15_000_000,
+            ready_time_ns: 92_000_000,
+        });
+
+        assert_eq!(timings.enter_to_boot_ns, 70_000_000);
+        assert_eq!(timings.boot_to_init_ns, 15_000_000);
+        assert_eq!(timings.boot_to_ready_ns, 22_000_000);
+        assert_eq!(timings.enter_to_ready_ns, 92_000_000);
+    }
+
     #[test]
     fn test_validate_rootfs_source_missing_bind_path() {
         let path = unique_temp_path("missing");

examples/rust/boot-timing-ci/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "boot-timing-ci"
+version = "0.1.0"
+edition = "2024"
+publish = false
+
+[[bin]]
+name = "boot-timing-ci"
+path = "bin/main.rs"
+
+[dependencies]
+microsandbox = { path = "../../../crates/microsandbox" }
+serde_json = "1.0"
+tokio = { version = "1.42", features = ["full"] }

examples/rust/boot-timing-ci/bin/main.rs

@@ -0,0 +1,102 @@
+use microsandbox::Sandbox;
+use serde_json::json;
+use std::path::{Path, PathBuf};
+
+const SANDBOX_NAME: &str = "boot-timing-ci";
+const ROOTFS_ENV_VAR: &str = "MICROSANDBOX_BOOT_TIMING_ROOTFS";
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let sandbox = Sandbox::builder(SANDBOX_NAME)
+        .image(rootfs_path())
+        .cpus(1)
+        .memory(256)
+        .quiet_logs()
+        .replace()
+        .create()
+        .await?;
+
+    let timings = sandbox.boot_timings();
+    sandbox.stop_and_wait().await?;
+    sandbox.remove_persisted().await?;
+
+    println!(
+        "{}",
+        serde_json::to_string(&json!({
+            "enter_to_boot_ns": timings.enter_to_boot_ns,
+            "boot_to_init_ns": timings.boot_to_init_ns,
+            "boot_to_ready_ns": timings.boot_to_ready_ns,
+            "enter_to_ready_ns": timings.enter_to_ready_ns,
+            "enter_to_boot_ms": ns_to_ms(timings.enter_to_boot_ns),
+            "boot_to_init_ms": ns_to_ms(timings.boot_to_init_ns),
+            "boot_to_ready_ms": ns_to_ms(timings.boot_to_ready_ns),
+            "enter_to_ready_ms": ns_to_ms(timings.enter_to_ready_ns),
+        }))?
+    );
+
+    Ok(())
+}
+
+fn ns_to_ms(value: u64) -> f64 {
+    value as f64 / 1_000_000.0
+}
+
+fn rootfs_path() -> String {
+    let arch = std::env::consts::ARCH;
+    let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let relative_rootfs = Path::new("examples/rust/root-bind/rootfs-alpine").join(arch);
+
+    if let Some(path) = std::env::var_os(ROOTFS_ENV_VAR) {
+        let candidate = PathBuf::from(path);
+        return resolve_rootfs_candidate(candidate)
+            .unwrap_or_else(|reason| {
+                panic!(
+                    "{ROOTFS_ENV_VAR} is set but unusable: {reason}. Set it to a populated {arch} rootfs directory."
+                )
+            })
+            .display()
+            .to_string();
+    }
+
+    let mut candidates = Vec::new();
+    candidates.push(manifest_dir.join("../root-bind/rootfs-alpine").join(arch));
+
+    for ancestor in manifest_dir.ancestors() {
+        candidates.push(ancestor.join(&relative_rootfs));
+        if let Some(parent) = ancestor.parent() {
+            candidates.push(parent.join("microsandbox").join(&relative_rootfs));
+        }
+    }
+
+    for candidate in candidates {
+        if resolve_rootfs_candidate(candidate.clone()).is_ok() {
+            return candidate.display().to_string();
+        }
+    }
+
+    panic!(
+        "unable to find a populated {arch} rootfs for {SANDBOX_NAME}. \
+expected the root-bind submodule at ../root-bind/rootfs-alpine/{arch}, \
+but it is missing in this checkout. Run `git submodule update --init --recursive` \
+or set {ROOTFS_ENV_VAR} to a populated {arch} rootfs directory."
+    );
+}
+
+fn resolve_rootfs_candidate(candidate: PathBuf) -> Result<PathBuf, String> {
+    if !candidate.exists() {
+        return Err(format!("path does not exist: {}", candidate.display()));
+    }
+
+    if !candidate.is_dir() {
+        return Err(format!("path is not a directory: {}", candidate.display()));
+    }
+
+    let mut entries = candidate
+        .read_dir()
+        .map_err(|err| format!("failed to read {}: {err}", candidate.display()))?;
+    if entries.next().is_none() {
+        return Err(format!("path is empty: {}", candidate.display()));
+    }
+
+    Ok(candidate)
+}

scripts/ci/boot-timing-thresholds.json

@@ -0,0 +1,16 @@
+{
+  "samples": 7,
+  "warmups": 1,
+  "baseline_ms": {
+    "enter_to_boot": 70.0,
+    "boot_to_init": 16.0
+  },
+  "max_regression_ms": {
+    "enter_to_boot": 20.0,
+    "boot_to_init": 8.0
+  },
+  "max_threshold_ms": {
+    "enter_to_boot": 100.0,
+    "boot_to_init": 30.0
+  }
+}