feat(skippy-cache): bounded host-RAM L2 tier prototype (#1651)

crates/skippy-bench/Cargo.toml

@@ -10,6 +10,7 @@ clap.workspace = true
 csv = "1.4.0"
 dirs = "6.0.0"
 libc = "0.2"
+skippy-cache = { path = "../skippy-cache" }
 skippy-protocol = { path = "../skippy-protocol" }
 skippy-runtime = { path = "../skippy-runtime" }
 skippy-topology = { path = "../skippy-topology" }

crates/skippy-bench/src/cli.rs

@@ -35,6 +35,8 @@ pub enum CommandKind {
     LocalSplitChainBinary(LocalSplitChainBinaryArgs),
     #[command(name = "verify-window-local")]
     VerifyWindowLocal(VerifyWindowLocalArgs),
+    #[command(name = "l2-tier")]
+    L2Tier(L2TierArgs),
     #[command(name = "chat-corpus")]
     ChatCorpus(ChatCorpusArgs),
     #[command(name = "token-lengths")]
@@ -45,6 +47,35 @@ pub enum CommandKind {
     Run(RunArgs),
 }
 
+#[derive(Parser)]
+pub struct L2TierArgs {
+    /// Working directory for the L3 store. Must not exist: the bench
+    /// creates, sentinel-marks, and (unless `--keep-store`) removes a
+    /// directory it owns, and refuses any pre-existing path instead of
+    /// deleting it.
+    #[arg(long, default_value = "/tmp/skippy-l2-tier-bench")]
+    pub store_root: PathBuf,
+    /// Number of timed L3-cold / L2-warm matched pairs after warmup.
+    #[arg(long, default_value_t = 50)]
+    pub pairs: usize,
+    /// Recorded prefix length in tokens (the synthetic conversation length).
+    #[arg(long, default_value_t = 1_893)]
+    pub tokens: usize,
+    /// Bytes of KV payload per token — sized to mimic a real dense model's
+    /// per-token KV footprint at the target dtype.
+    #[arg(long, default_value_t = 512)]
+    pub kv_bytes_per_token: usize,
+    /// L2 budget in MiB. Defaults to four times one entry.
+    #[arg(long)]
+    pub l2_budget_mib: Option<u64>,
+    /// Keep the L3 store directory after the run for inspection.
+    #[arg(long, default_value_t = false)]
+    pub keep_store: bool,
+    /// Model identity stamped into the L3 tier and L2 keys.
+    #[arg(long, default_value = "bench-model")]
+    pub model_identity: String,
+}
+
 #[derive(Parser)]
 pub struct EvalArgs {
     #[command(subcommand)]

crates/skippy-bench/src/l2_tier.rs

@@ -0,0 +1,309 @@
+//! `l2-tier` benchmark: cold L3 fill versus warm L2 lookup on identical
+//! packed entries (#1651).
+//!
+//! Builds a temporary L3 store, spills a synthetic multi-turn prompt at a
+//! recorded prefix length, then measures two restore paths in-process:
+//!
+//! - **L3 cold fill**: `L3Tier::fill_longest` — index probe + segment
+//!   assembly + digest verification from disk.
+//! - **L2 warm lookup**: `L2Tier::get` + `to_payload` + materialization —
+//!   the entry was admitted from an identical verified L3 fill, so the
+//!   lookup is a digest-keyed handle assembly (no re-hash: admission
+//!   verified the wire once; segments are immutable afterward).
+//!
+//! Both arms are timed through the same boundary: the moment their bytes
+//! are usable (`full_state_bytes_timed`). A multi-segment L2 entry still
+//! materializes its wire on read, so stopping the L2 timer at handle
+//! creation would understate the real cost; the equality gate compares the
+//! materialized bytes of both arms before the pair is counted. The
+//! handle-only lookup time is reported separately as
+//! `l2_handle_lookup_ns`.
+//!
+//! Admission hashing (the one-time wire BLAKE3) is measured separately
+//! and reported as its own metric, never inside the timed lookup.
+//!
+//! The store root is owned by the run: it must not exist beforehand (the
+//! bench refuses existing paths instead of deleting user data) and it is
+//! marked with an ownership sentinel so cleanup never touches a directory
+//! the bench did not create.
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+
+use anyhow::{Context, Result};
+
+use crate::cli::L2TierArgs;
+use skippy_cache::{
+    ExactStatePayload, ExactStatePayloadMirror, L2Origin, L2Tier, l2_cache_key, l3_prefix_key,
+};
+
+/// Marker file proving the bench created the store root itself; cleanup
+/// refuses to `remove_dir_all` a directory without it.
+const OWNERSHIP_SENTINEL: &str = ".skippy-l2-tier-bench-owned";
+
+/// Create a fresh, bench-owned store root. Existing paths are refused —
+/// the bench must never delete a user-supplied directory it did not
+/// create.
+fn prepare_store_root(requested: &Path) -> Result<PathBuf> {
+    if requested.symlink_metadata().is_ok() {
+        anyhow::bail!(
+            "refusing to use store root {}: the path already exists; the bench only runs \
+             in a root it created itself",
+            requested.display()
+        );
+    }
+    std::fs::create_dir_all(requested)
+        .with_context(|| format!("failed to create bench store root {}", requested.display()))?;
+    std::fs::write(
+        requested.join(OWNERSHIP_SENTINEL),
+        b"skippy-bench l2-tier store\n",
+    )
+    .with_context(|| format!("failed to mark {} as bench-owned", requested.display()))?;
+    Ok(requested.to_path_buf())
+}
+
+/// Remove a bench-owned store root. Refuses paths without the ownership
+/// sentinel so `remove_dir_all` can never hit arbitrary input.
+fn remove_owned_store_root(root: &Path) -> Result<()> {
+    if !root.join(OWNERSHIP_SENTINEL).is_file() {
+        anyhow::bail!(
+            "refusing to remove store root {}: missing bench ownership sentinel",
+            root.display()
+        );
+    }
+    std::fs::remove_dir_all(root)
+        .with_context(|| format!("failed to remove bench store root {}", root.display()))
+}
+
+fn percentile(samples_ns: &mut [u128], pct: f64) -> f64 {
+    samples_ns.sort_unstable();
+    let index = ((pct / 100.0) * (samples_ns.len() as f64 - 1.0)).round() as usize;
+    samples_ns[index.min(samples_ns.len() - 1)] as f64
+}
+
+pub fn l2_tier(args: L2TierArgs) -> Result<()> {
+    // Validation: reject degenerate configurations up front instead of
+    // dividing by zero or allocating nothing below.
+    if args.pairs == 0 {
+        anyhow::bail!("--pairs must be at least 1");
+    }
+    if args.tokens == 0 {
+        anyhow::bail!("--tokens must be at least 1");
+    }
+    if args.kv_bytes_per_token == 0 {
+        anyhow::bail!("--kv-bytes-per-token must be at least 1");
+    }
+
+    // Owned store root: refuse existing paths rather than deleting them,
+    // and mark the created directory so cleanup stays bounded to it.
+    let store_root = prepare_store_root(&args.store_root)?;
+
+    let namespace = "bench-namespace";
+    let state_identity = args.model_identity.clone();
+    let token_ids: Vec<i32> = (0..args.tokens).map(|i| (i % 128_000) as i32).collect();
+
+    // Deterministic synthetic KV payload: content matters only for digests,
+    // size matters for timing.
+    let payload_len = args
+        .tokens
+        .checked_mul(args.kv_bytes_per_token)
+        .context("--tokens * --kv-bytes-per-token overflows")?;
+    let payload_bytes: Vec<u8> = (0..payload_len).map(|i| (i % 251) as u8).collect();
+    let payload = ExactStatePayload::full_state(payload_bytes);
+
+    let tier = skippy_cache::L3Tier::open(
+        store_root.clone(),
+        (payload_len as u64) * 8,
+        state_identity.clone(),
+        64 * 1024,
+    )
+    .context("failed to open bench L3 tier")?;
+
+    // Spill once: this is the population path, not the measured path.
+    let manifest_key = tier
+        .spill(namespace, &token_ids, &payload, None, None)
+        .context("bench spill failed")?;
+    let _ = manifest_key;
+
+    // Locate once to learn the recorded prefix key/digest used by both paths.
+    let location = tier
+        .locate_longest(namespace, &token_ids, 8)
+        .context("bench locate failed")?
+        .context("bench spill was not locatable")?;
+    let manifest = tier.store().load_manifest(&location.manifest_key)?;
+    let payload_digest = manifest.payload_digest.clone();
+    let recorded_tokens = manifest.token_count;
+
+    let l2_budget_bytes = args
+        .l2_budget_mib
+        .map(|mib| {
+            mib.checked_mul(1024 * 1024)
+                .context("--l2-budget-mib overflows")
+        })
+        .transpose()?
+        .unwrap_or(payload_len as u64 * 4);
+    let l2 = L2Tier::new(l2_budget_bytes);
+
+    let cache_key = l2_cache_key(&args.model_identity, &state_identity, namespace, &token_ids);
+
+    // Warmup: one L3 fill, then admit it into L2 from the verified wire.
+    // The wire check inside `admit` is the one-time admission hash; it is
+    // timed separately below.
+    let warm_fill = tier
+        .fill_longest(namespace, &token_ids, 8)
+        .context("bench warmup L3 fill failed")?
+        .context("bench warmup L3 fill missed")?;
+    let (warm_wire, _) = warm_fill.payload.full_state_bytes_timed().context("wire")?;
+    let admission_started = Instant::now();
+    l2.admit(
+        cache_key.clone(),
+        warm_fill.token_count,
+        payload_digest.clone(),
+        &warm_wire,
+        ExactStatePayloadMirror::from_manifest(&manifest)
+            .map_err(|refusal| anyhow::anyhow!(refusal.reason()))?,
+        L2Origin::FromL3,
+    )
+    .map_err(|refusal| anyhow::anyhow!("bench warmup L2 admit refused: {}", refusal.reason()))?;
+    let admission_hash_ns = admission_started.elapsed().as_nanos();
+
+    let mut l3_samples: Vec<u128> = Vec::with_capacity(args.pairs);
+    let mut l2_samples: Vec<u128> = Vec::with_capacity(args.pairs);
+    let mut l2_handle_samples: Vec<u128> = Vec::with_capacity(args.pairs);
+
+    for pair in 0..args.pairs {
+        // Cold-ish L3 fill: the OS page cache will help after warmup, which
+        // matches the production comparison — both paths run on the same
+        // machine state, the delta is the tier delta.
+        let start = Instant::now();
+        let fill = tier
+            .fill_longest(namespace, &token_ids, 8)
+            .context("bench L3 fill failed")?
+            .context("bench L3 fill missed")?;
+        let l3_ns = start.elapsed().as_nanos();
+
+        // Timed through the same boundary as the L3 arm, starting before
+        // the lookup: the L3 timer covers index probe + assembly +
+        // verification, so the L2 timer covers lookup + handle assembly +
+        // materialization — both arms measure "nothing to usable bytes".
+        // The handle-only lookup time (this same `get`, inner timer) is
+        // reported separately as `l2_handle_lookup_ns`.
+        let start = Instant::now();
+        let handle_start = Instant::now();
+        let hit = l2.get(&cache_key);
+        let l2_handle_ns = handle_start.elapsed().as_nanos();
+        let l2_payload = hit.as_ref().map(|hit| hit.to_payload());
+        let (l2_bytes, _) = l2_payload
+            .as_ref()
+            .context("bench L2 payload missing")?
+            .full_state_bytes_timed()
+            .context("bench L2 bytes")?;
+        let l2_ns = start.elapsed().as_nanos();
+
+        // Correctness gate, outside the timer: L2 must return
+        // byte-identical state to the L3 fill, or the speedup is
+        // meaningless.
+        let hit = hit.context("bench L2 lookup missed")?;
+        anyhow::ensure!(hit.token_count == fill.token_count);
+        anyhow::ensure!(hit.payload_digest == payload_digest);
+        let (l3_bytes, _) = fill.payload.full_state_bytes_timed().context("l3 bytes")?;
+        anyhow::ensure!(
+            l3_bytes.as_ref() == l2_bytes.as_ref(),
+            "pair {pair}: L2 payload diverged from L3 fill"
+        );
+
+        l3_samples.push(l3_ns);
+        l2_samples.push(l2_ns);
+        l2_handle_samples.push(l2_handle_ns);
+    }
+
+    let stats = l2.stats();
+    let mut l3_sorted = l3_samples.clone();
+    let mut l2_sorted = l2_samples.clone();
+    let mut l2_handle_sorted = l2_handle_samples.clone();
+    let summary = serde_json::json!({
+        "bench": "l2-tier",
+        "pairs": args.pairs,
+        "tokens": recorded_tokens,
+        "payload_bytes": payload_len,
+        "l2_budget_bytes": l2_budget_bytes,
+        "model_identity": args.model_identity,
+        "l3_fill_ns": {
+            "p50": percentile(&mut l3_sorted, 50.0),
+            "p99": percentile(&mut l3_sorted, 99.0),
+        },
+        "l2_lookup_to_usable_bytes_ns": {
+            "p50": percentile(&mut l2_sorted, 50.0),
+            "p99": percentile(&mut l2_sorted, 99.0),
+        },
+        "l2_handle_lookup_ns": {
+            "p50": percentile(&mut l2_handle_sorted, 50.0),
+            "p99": percentile(&mut l2_handle_sorted, 99.0),
+        },
+        "l2_admission_hash_ns_one_time": admission_hash_ns,
+        "speedup_p50": percentile(&mut l3_sorted, 50.0) / percentile(&mut l2_sorted, 50.0).max(1.0),
+        "l2_stats": {
+            "hits": stats.hits,
+            "misses": stats.misses,
+            "evictions": stats.evictions,
+            "bytes": stats.bytes,
+            "segments": stats.segments,
+            "shared_bytes_admitted": stats.shared_bytes_admitted,
+        },
+        "l3_prefix_key": l3_prefix_key(namespace, &token_ids),
+    });
+    println!("{summary}");
+
+    if !args.keep_store {
+        remove_owned_store_root(&store_root)?;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn store_root_refuses_existing_paths() {
+        let dir =
+            std::env::temp_dir().join(format!("skippy-l2-bench-refuse-{}", std::process::id()));
+        let _ = std::fs::remove_dir_all(&dir);
+        std::fs::create_dir_all(&dir).expect("create pre-existing dir");
+        std::fs::write(dir.join("precious.txt"), b"user data").expect("seed user data");
+
+        let err = prepare_store_root(&dir).expect_err("existing path must be refused");
+        assert!(
+            err.to_string().contains("refusing to use store root"),
+            "unexpected error: {err}"
+        );
+        assert!(
+            dir.join("precious.txt").is_file(),
+            "pre-existing contents must survive the refusal"
+        );
+
+        // A root the bench created is removable; a lookalike without the
+        // sentinel is not. (Creation itself stays refused for any
+        // pre-existing path, owned or not.)
+        let _ = std::fs::remove_dir_all(&dir);
+
+        let owned =
+            std::env::temp_dir().join(format!("skippy-l2-bench-owned-{}", std::process::id()));
+        let _ = std::fs::remove_dir_all(&owned);
+        prepare_store_root(&owned).expect("fresh root is created and owned");
+        assert!(owned.join(OWNERSHIP_SENTINEL).is_file());
+        remove_owned_store_root(&owned).expect("owned root is removable");
+        assert!(!owned.exists());
+
+        // Unmarked directory: cleanup must refuse.
+        let unowned =
+            std::env::temp_dir().join(format!("skippy-l2-bench-unowned-{}", std::process::id()));
+        let _ = std::fs::remove_dir_all(&unowned);
+        std::fs::create_dir_all(&unowned).expect("create unowned dir");
+        std::fs::write(unowned.join("keep.txt"), b"user data").expect("seed");
+        let err =
+            remove_owned_store_root(&unowned).expect_err("unowned root removal must be refused");
+        assert!(err.to_string().contains("sentinel"), "unexpected: {err}");
+        assert!(unowned.join("keep.txt").is_file(), "contents survive");
+        let _ = std::fs::remove_dir_all(&unowned);
+    }
+}

crates/skippy-bench/src/main.rs

@@ -3,6 +3,7 @@ mod cli;
 mod direct_return_listener;
 mod distributed;
 mod evals;
+mod l2_tier;
 mod local_single;
 mod local_split;
 mod model_identity;
@@ -54,6 +55,7 @@ fn main() -> Result<()> {
         CommandKind::LocalSplitCompare(args) => local_split_compare(args),
         CommandKind::LocalSplitChainBinary(args) => local_split_chain_binary(args),
         CommandKind::VerifyWindowLocal(args) => verify_window_local(args),
+        CommandKind::L2Tier(args) => l2_tier::l2_tier(args),
         CommandKind::ChatCorpus(args) => chat_corpus(args),
         CommandKind::TokenLengths(args) => token_lengths(args),
         CommandKind::FocusedRuntime(args) => focused_runtime(args),