diff --git a/Cargo.lock b/Cargo.lock index 6f0f6650..1d6c00ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1204,6 +1204,7 @@ dependencies = [ "flate2", "futures", "hex", + "libc", "rand 0.10.2", "rcgen", "rustls", diff --git a/Cargo.toml b/Cargo.toml index 1b262e63..022beab7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ futures-lite = "2.6" hex = "0.4" hmac = "0.13" lending-stream = "1.0" +libc = "0.2" # glibc malloc tunables metrics = "0.24" metrics-exporter-prometheus = { version = "0.18.1", default-features = false, features = ["http-listener"] } opentelemetry = "0.32" diff --git a/book/src/guide/performance.md b/book/src/guide/performance.md index a20e85ac..be8d1a9e 100644 --- a/book/src/guide/performance.md +++ b/book/src/guide/performance.md @@ -65,6 +65,29 @@ If a process holds more than `ERYX_POOL_INSTANCES` sessions open at once, raise the limit or switch to `on-demand`. Note that the choice of allocator does not affect precompiled `.cwasm` artifacts; only compilation settings do. +## Host heap + +Wasmtime keeps each instance's metadata (the `VMContext`, about 450 KB for this +runtime's ~10k function references) on the ordinary host heap and frees it when +the instance is torn down. With glibc's default `malloc` settings that block is +usually the top of the heap, so freeing it hands the pages back to the kernel +and the next instantiation grows the heap and faults them in again: three `brk` +calls and roughly 100 page faults per execution, or about a quarter of a cold +`pass` once the allocator above is in place. + +On Linux with glibc, eryx therefore sets two `malloc` tunables once, when the +engine is created: trimming is disabled (`M_TRIM_THRESHOLD = -1`) and the heap +grows in 64 MiB steps (`M_TOP_PAD`). Both are process-wide; the cost is that +freed heap stays mapped in the process instead of being returned to the OS. + +| Variable | Default | Effect | +|----------|---------|--------| +| `ERYX_GLIBC_MALLOC_TUNING` | `1` | `0` leaves `malloc` untouched. | + +Processes that use another allocator (jemalloc, mimalloc, or musl's) are not +affected either way; check whether that allocator returns freed memory eagerly +if you see `brk`/`munmap` churn in `strace` around executions. + ## Measuring `crates/eryx/examples/profile_stateless.rs` times the stateless path and is a diff --git a/crates/eryx/Cargo.toml b/crates/eryx/Cargo.toml index 1f0bb7c0..672709b0 100644 --- a/crates/eryx/Cargo.toml +++ b/crates/eryx/Cargo.toml @@ -106,6 +106,8 @@ eryx-vfs = { workspace = true, optional = true } #unified flate2.workspace = true futures.workspace = true hex.workspace = true +# glibc malloc tunables (Linux only; see AllocatorSettings in wasm.rs) +libc.workspace = true # Secrets support rand.workspace = true # TLS networking diff --git a/crates/eryx/examples/profile_stateless.rs b/crates/eryx/examples/profile_stateless.rs index 0d732921..207eda2d 100644 --- a/crates/eryx/examples/profile_stateless.rs +++ b/crates/eryx/examples/profile_stateless.rs @@ -30,7 +30,12 @@ fn main() -> Result<(), Box> { rt.block_on(async { eprintln!("Creating sandbox..."); - let sandbox = Sandbox::embedded().build()?; + // Trace collection (sys.settrace) is on by default; `ERYX_PROFILE_TRACE=0` + // turns it off to measure without per-event trace overhead. + let collect_trace = !std::env::var("ERYX_PROFILE_TRACE").is_ok_and(|v| v.trim() == "0"); + let sandbox = Sandbox::embedded() + .with_trace_collection(collect_trace) + .build()?; let executor = sandbox.executor(); eprintln!("Warming up (10 iterations)..."); diff --git a/crates/eryx/src/wasm.rs b/crates/eryx/src/wasm.rs index 1c86eddf..3ae99566 100644 --- a/crates/eryx/src/wasm.rs +++ b/crates/eryx/src/wasm.rs @@ -512,12 +512,22 @@ impl std::str::FromStr for CpuFeatureLevel { /// pages with Linux's `PAGEMAP_SCAN` ioctl (6.7+) and reset only those; /// `0` resets the whole keep-resident budget with `memcpy` instead, which /// trades a larger copy for fewer page faults on the next execution. +/// - `ERYX_GLIBC_MALLOC_TUNING`: `1` (default) stops glibc's `malloc` from +/// returning freed heap to the kernel and grows the heap in 64 MiB steps +/// (`M_TRIM_THRESHOLD` / `M_TOP_PAD`). Wasmtime heap-allocates each +/// instance's `VMContext` (~450 KB for this runtime's ~10k function +/// references) and frees it on teardown; with glibc's defaults that +/// allocation sits at the top of the heap, so every execution trims it back +/// to the kernel and grows it again — three `brk` calls and ~100 page faults. +/// `0` leaves `malloc` alone. Only has an effect on Linux with glibc and the +/// `embedded` or `preinit` feature. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct AllocatorSettings { pooling: bool, pool_instances: u32, keep_resident_bytes: usize, pagemap_scan: bool, + glibc_malloc_tuning: bool, } impl Default for AllocatorSettings { @@ -527,10 +537,40 @@ impl Default for AllocatorSettings { pool_instances: 1000, keep_resident_bytes: 64 << 20, pagemap_scan: true, + glibc_malloc_tuning: true, } } } +/// Stop glibc returning freed heap to the kernel and grow it in 64 MiB steps. +/// +/// Process-wide, so it is applied at most once; see the +/// `ERYX_GLIBC_MALLOC_TUNING` entry on [`AllocatorSettings`] for why. +#[cfg(all( + target_os = "linux", + target_env = "gnu", + any(feature = "embedded", feature = "preinit") +))] +#[allow(unsafe_code)] +fn tune_glibc_malloc() { + static APPLIED: std::sync::Once = std::sync::Once::new(); + APPLIED.call_once(|| { + // SAFETY: `mallopt` only records allocator parameters; it has no + // preconditions beyond valid parameter constants, which these are. + unsafe { + libc::mallopt(libc::M_TRIM_THRESHOLD, -1); + libc::mallopt(libc::M_TOP_PAD, 64 << 20); + } + }); +} + +#[cfg(not(all( + target_os = "linux", + target_env = "gnu", + any(feature = "embedded", feature = "preinit") +)))] +fn tune_glibc_malloc() {} + impl AllocatorSettings { /// Upper bound on funcref table growth per instance. /// @@ -591,9 +631,27 @@ impl AllocatorSettings { None => {} } + match lookup("ERYX_GLIBC_MALLOC_TUNING") { + Some(v) if v.trim() == "0" => settings.glibc_malloc_tuning = false, + Some(v) if v.trim() == "1" => settings.glibc_malloc_tuning = true, + Some(v) => { + return Err(Error::WasmEngine(format!( + "ERYX_GLIBC_MALLOC_TUNING must be 0 or 1, got '{v}'" + ))); + } + None => {} + } + Ok(settings) } + /// Apply the process-wide `malloc` tunables, if enabled. + pub(crate) fn apply_malloc_tuning(&self) { + if self.glibc_malloc_tuning { + tune_glibc_malloc(); + } + } + /// The pooling configuration to use, or `None` for on-demand allocation. pub(crate) fn pooling_config(&self) -> Option { if !self.pooling { @@ -2269,6 +2327,7 @@ impl PythonExecutor { /// on either allocator, so that refusal only costs instantiation latency. fn build_engine(mut config: Config) -> std::result::Result { let allocator = AllocatorSettings::from_env()?; + allocator.apply_malloc_tuning(); let Some(pooling) = allocator.pooling_config() else { return Engine::new(&config).map_err(|e| Error::WasmEngine(e.to_string())); }; @@ -3152,6 +3211,17 @@ mod tests { assert!(matches!(pooling.get_pagemap_scan(), Enabled::No)); } + #[test] + fn malloc_tuning_is_on_by_default_and_can_be_disabled() { + assert!(allocator_settings(&[]).unwrap().glibc_malloc_tuning); + let settings = allocator_settings(&[("ERYX_GLIBC_MALLOC_TUNING", "0")]).unwrap(); + assert!(!settings.glibc_malloc_tuning); + // Applying is a no-op when disabled and idempotent when enabled. + settings.apply_malloc_tuning(); + AllocatorSettings::default().apply_malloc_tuning(); + AllocatorSettings::default().apply_malloc_tuning(); + } + #[test] fn allocator_rejects_invalid_values() { for vars in [ @@ -3160,6 +3230,7 @@ mod tests { [("ERYX_POOL_INSTANCES", "lots")], [("ERYX_POOL_KEEP_RESIDENT_MB", "-1")], [("ERYX_POOL_PAGEMAP_SCAN", "yes")], + [("ERYX_GLIBC_MALLOC_TUNING", "true")], ] { let err = allocator_settings(&vars).expect_err("invalid setting must be rejected"); assert!(matches!(err, Error::WasmEngine(_)), "{vars:?}: {err:?}");