From 948a40206693d37fdd9d4520582734e69afb87e2 Mon Sep 17 00:00:00 2001 From: Ben Sully Date: Fri, 11 Sep 2026 04:22:21 +0100 Subject: [PATCH] feat: make trace collection easy to disable (docs + ERYX_PROFILE_TRACE) `SandboxBuilder` defaults to `collect_trace: true`, which installs `sys.settrace` for every execution. The hook's cost scales with the amount of Python run: cold `pass` 1.28 ms vs 1.08 ms, a small json + string.Template render 3.7 ms vs 0.9 ms, and `sum(i * i for i in range(20_000))` 369 ms vs 3.8 ms (~96x). The knob to turn it off (`with_trace_collection(false)`, #270) existed but was not discoverable and the profiling harness always ran with it on. - Document the cost and the session/handler interaction on `SandboxBuilder::with_trace_collection`. - Add an `ERYX_PROFILE_TRACE=0` env switch to `profile_execution.rs`, `session_bench.rs` and the criterion bench's `create_sandbox()`. Default (unset) keeps tracing on so existing numbers stay comparable. - Explain in the pyeryx `Sandbox` docstring / stubs that Python sandboxes never collect traces (hardcoded off since #270 because `ExecuteResult` has no trace field), and comment the two call sites. - Add a "Trace Collection" subsection to the sandboxes guide. No defaults change and no new API is added. Co-Authored-By: Claude Fable 5.1 --- book/src/guide/sandboxes.md | 25 +++++++++++++++++++++++ crates/eryx-python/python/eryx/_eryx.pyi | 4 ++++ crates/eryx-python/src/preinit.rs | 3 ++- crates/eryx-python/src/sandbox.rs | 9 +++++++- crates/eryx/benches/execution.rs | 8 ++++++++ crates/eryx/examples/profile_execution.rs | 11 +++++++++- crates/eryx/examples/session_bench.rs | 6 +++++- crates/eryx/src/sandbox.rs | 24 +++++++++++++++++++--- 8 files changed, 83 insertions(+), 7 deletions(-) diff --git a/book/src/guide/sandboxes.md b/book/src/guide/sandboxes.md index c0c9a62b..da8dcec0 100644 --- a/book/src/guide/sandboxes.md +++ b/book/src/guide/sandboxes.md @@ -130,6 +130,31 @@ print(f"Callback invocations: {result.callback_invocations}") ``` +### Trace Collection + +In Rust, the sandbox also records line-level execution events in +`ExecuteResult::trace`. This is on by default and installs Python's +`sys.settrace` hook, whose cost scales with how much Python runs: `pass` +costs about 20% extra per execution, a small `json` + `string.Template` +render goes from 0.9 ms to 3.7 ms, and `sum(i * i for i in range(20_000))` +from 3.8 ms to 369 ms. Disable it unless you read the trace: + +```rust +# extern crate eryx; +use eryx::Sandbox; + +# fn main() -> Result<(), eryx::Error> { +let sandbox = Sandbox::embedded().with_trace_collection(false).build()?; +# Ok(()) +# } +``` + +A `TraceHandler` set with `with_trace_handler` still receives events when +collection is disabled. Sessions created from the sandbox inherit the setting. + +The Python bindings never collect traces (`ExecuteResult` does not expose +them), so no option is needed there. + ## Error Handling Sandbox execution can fail for various reasons. Eryx provides typed errors to help you handle them: diff --git a/crates/eryx-python/python/eryx/_eryx.pyi b/crates/eryx-python/python/eryx/_eryx.pyi index c35f599a..a1cbed07 100644 --- a/crates/eryx-python/python/eryx/_eryx.pyi +++ b/crates/eryx-python/python/eryx/_eryx.pyi @@ -441,6 +441,10 @@ class Sandbox: For sandboxes with custom packages, use `SandboxFactory` instead. + Execution trace collection (`sys.settrace`) is disabled for Python + sandboxes; `ExecuteResult` does not expose trace events, so there is no + per-line tracing overhead. + Example: # Basic sandbox (stdlib only) sandbox = Sandbox() diff --git a/crates/eryx-python/src/preinit.rs b/crates/eryx-python/src/preinit.rs index 45adf63c..6e736363 100644 --- a/crates/eryx-python/src/preinit.rs +++ b/crates/eryx-python/src/preinit.rs @@ -370,7 +370,8 @@ impl SandboxFactory { // Use provided site_packages or fall back to the one from initialization let site_packages_path = site_packages.or_else(|| self.site_packages_path.clone()); - // Build sandbox from precompiled bytes + // Build sandbox from precompiled bytes. Trace collection (sys.settrace) + // is always off for Python sandboxes; see `Sandbox::new`. // SAFETY: The precompiled bytes were created by PythonExecutor::precompile() // from a valid WASM component, so they are safe to deserialize. let mut builder = unsafe { diff --git a/crates/eryx-python/src/sandbox.rs b/crates/eryx-python/src/sandbox.rs index 89164e35..7418b7c2 100644 --- a/crates/eryx-python/src/sandbox.rs +++ b/crates/eryx-python/src/sandbox.rs @@ -89,6 +89,10 @@ impl Sandbox { /// /// For sandboxes with custom packages, use `SandboxFactory` instead. /// + /// Execution trace collection (`sys.settrace`) is disabled for Python + /// sandboxes; `ExecuteResult` does not expose trace events, so there is + /// no per-line tracing overhead. + /// /// Args: /// resource_limits: Optional resource limits for execution. /// network: Optional network configuration. If provided, enables networking. @@ -167,7 +171,10 @@ impl Sandbox { })?, ); - // Build the eryx sandbox with embedded runtime + // Build the eryx sandbox with embedded runtime. Trace collection + // (sys.settrace) is always off: the Python ExecuteResult does not + // expose trace events, and the hook makes instruction-heavy scripts + // orders of magnitude slower. let mut builder = eryx::Sandbox::embedded().with_trace_collection(false); // Apply resource limits if provided diff --git a/crates/eryx/benches/execution.rs b/crates/eryx/benches/execution.rs index 648afcd8..cf18492c 100644 --- a/crates/eryx/benches/execution.rs +++ b/crates/eryx/benches/execution.rs @@ -2,6 +2,10 @@ //! //! Run with: `cargo bench --package eryx --features embedded` //! +//! Trace collection (`sys.settrace`) is on by default, matching the library +//! default so historical numbers stay comparable. Set `ERYX_PROFILE_TRACE=0` +//! to benchmark with it disabled. +//! //! ## Benchmark Groups //! //! - **sandbox_creation**: Measures time to create a new sandbox @@ -121,7 +125,11 @@ impl TypedCallback for WorkCallback { // ============================================================================ fn create_sandbox() -> Sandbox { + // Trace collection (sys.settrace) is on by default; `ERYX_PROFILE_TRACE=0` + // turns it off to measure without per-event trace overhead. + let collect_trace = !std::env::var("ERYX_PROFILE_TRACE").is_ok_and(|v| v.trim() == "0"); Sandbox::embedded() + .with_trace_collection(collect_trace) .with_callback(NoopCallback) .with_callback(EchoCallback) .with_callback(WorkCallback) diff --git a/crates/eryx/examples/profile_execution.rs b/crates/eryx/examples/profile_execution.rs index f93d78d9..5ca723c0 100644 --- a/crates/eryx/examples/profile_execution.rs +++ b/crates/eryx/examples/profile_execution.rs @@ -6,6 +6,9 @@ //! //! Or with a specific iteration count: //! samply record ./target/release/examples/profile_execution 5000 +//! +//! Set `ERYX_PROFILE_TRACE=0` to disable trace collection (`sys.settrace`), +//! which is on by default and dominates anything heavier than `pass`. use std::time::Instant; @@ -23,7 +26,13 @@ fn main() -> Result<(), Box> { rt.block_on(async { eprintln!("Creating sandbox..."); - let sandbox = Sandbox::embedded().build()?; + // Trace collection (sys.settrace) is on by default; `ERYX_PROFILE_TRACE=0` + // turns it off to measure without per-event trace overhead. + let collect_trace = !std::env::var("ERYX_PROFILE_TRACE").is_ok_and(|v| v.trim() == "0"); + eprintln!(" trace collection: {collect_trace}"); + let sandbox = Sandbox::embedded() + .with_trace_collection(collect_trace) + .build()?; eprintln!("Creating session..."); let mut session = InProcessSession::new(&sandbox).await?; diff --git a/crates/eryx/examples/session_bench.rs b/crates/eryx/examples/session_bench.rs index 361e0594..3ef472a1 100644 --- a/crates/eryx/examples/session_bench.rs +++ b/crates/eryx/examples/session_bench.rs @@ -56,9 +56,13 @@ async fn main() -> Result<(), Box> { // Create sandbox using cache_dir (handles linking, pre-init, precompile, and mmap) println!("Creating sandbox (cold - linking + compiling + caching)..."); let start = Instant::now(); + // Trace collection (sys.settrace) is on by default; `ERYX_PROFILE_TRACE=0` + // turns it off to measure without per-event trace overhead. + let collect_trace = !std::env::var("ERYX_PROFILE_TRACE").is_ok_and(|v| v.trim() == "0"); + println!(" trace collection: {collect_trace}"); // Start with embedded() which provides runtime+stdlib, then late-linking // overrides the runtime when native extensions are added - let mut builder = Sandbox::embedded(); + let mut builder = Sandbox::embedded().with_trace_collection(collect_trace); for (name, bytes) in &extensions { builder = builder.with_native_extension(name.clone(), bytes.clone()); } diff --git a/crates/eryx/src/sandbox.rs b/crates/eryx/src/sandbox.rs index 56f7bc53..d8b55cd2 100644 --- a/crates/eryx/src/sandbox.rs +++ b/crates/eryx/src/sandbox.rs @@ -1887,9 +1887,27 @@ impl SandboxBuilder { /// Configure whether execution trace events are collected in the result. /// /// Trace collection is enabled by default for backward compatibility. It - /// installs Python's `sys.settrace` hook, which can be expensive for - /// instruction-heavy workloads. A configured [`TraceHandler`] always keeps - /// tracing enabled regardless of this setting. + /// installs Python's `sys.settrace` hook, which fires on every line, call + /// and return in the guest, so its cost scales with the amount of Python + /// executed rather than with sandbox setup. Measured on a Ryzen 9 7950X + /// (wasmtime 48, fresh instance per call), enabled vs disabled: + /// + /// | Workload | On | Off | + /// |---|---|---| + /// | `pass` | 1.28 ms | 1.08 ms | + /// | `json.loads` + `string.Template(...).substitute(...)` | 3.7 ms | 0.9 ms | + /// | `sum(i * i for i in range(20_000))` | 369 ms | 3.8 ms | + /// + /// Disable it unless you read [`ExecuteResult::trace`]: + /// + /// ```rust,ignore + /// let sandbox = Sandbox::embedded().with_trace_collection(false).build()?; + /// ``` + /// + /// This also applies to [`InProcessSession`](crate::session::InProcessSession)s + /// created from the sandbox. A configured [`TraceHandler`] always keeps + /// tracing enabled regardless of this setting; only the retention of + /// events in the result is controlled here. #[must_use] pub const fn with_trace_collection(mut self, enabled: bool) -> Self { self.collect_trace = enabled;