From 16dd8a0a34577039f180f61e2fb91ef76331ef48 Mon Sep 17 00:00:00 2001 From: intentos-dev Date: Sun, 21 Jun 2026 15:14:14 +0200 Subject: [PATCH] feat(core): multi-sample runs + variance, folded into confidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the long-reserved `Scenario.samples` field (plus a `--samples` override) so a scenario can be measured N times on a non-deterministic backend. The per-sample `total_cu` values become a `SampleStats` distribution (count/min/median/max/variance/std-dev/CV) on the measurement, and the coefficient of variation folds into the confidence score (CV >=2% -> Medium, >=10% -> Low) — implementing the spec §12 "sample variance" factor. Crucially this is honestly gated: `ExecutionBackend::is_deterministic()` (true for the recorded backend) makes the profiler run a deterministic backend exactly once, so it never fabricates a run-to-run spread it did not observe. The field is now settable from `[scenario.x] samples` and overridable with `--samples`. `SampleStats` serializes only when present (Option, skip-if-none), so existing single-sample JSON reports are byte-identical. Docs: reference §12/§15 updated; CHANGELOG. Gate: grade A. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 6 + crates/cu-profiler-cli/src/args.rs | 5 + crates/cu-profiler-cli/src/commands/mod.rs | 9 +- crates/cu-profiler-core/src/backend/mod.rs | 11 ++ .../cu-profiler-core/src/backend/recorded.rs | 5 + crates/cu-profiler-core/src/confidence.rs | 43 +++++++ crates/cu-profiler-core/src/config.rs | 6 +- crates/cu-profiler-core/src/model.rs | 107 +++++++++++++++++ crates/cu-profiler-core/src/profiler.rs | 108 +++++++++++++++++- docs/reference.md | 5 + 10 files changed, 297 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ede0477..d2d236b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). `--input report.md`, `--pr`, `--repo`, `--marker`, and `--dry-run` (no network). `init --workflow` now scaffolds a render-and-comment step with `permissions: pull-requests: write`. +- **Multi-sample runs + variance.** `Scenario.samples` (and a `--samples` override) + now run a scenario N times on non-deterministic backends and record a `SampleStats` + distribution (count/min/median/max/variance/std-dev/CV) in the measurement. Variance + folds into the confidence score (CV ≥2% → Medium, ≥10% → Low), implementing the + spec §12 "sample variance" factor. The deterministic recorded backend ignores + `samples` and never fabricates a spread (`ExecutionBackend::is_deterministic`). ## [0.1.2] - 2026-06-20 diff --git a/crates/cu-profiler-cli/src/args.rs b/crates/cu-profiler-cli/src/args.rs index 017f422..4c08f9e 100644 --- a/crates/cu-profiler-cli/src/args.rs +++ b/crates/cu-profiler-cli/src/args.rs @@ -68,6 +68,11 @@ pub struct CommonRun { /// Only run scenarios carrying these tags (repeatable). #[arg(long = "tag")] pub tags: Vec, + + /// Override the per-scenario sample count (number of measurement runs). + /// Only affects non-deterministic backends; the recorded backend ignores it. + #[arg(long)] + pub samples: Option, } /// `cu-profiler run` / `cu-profiler ci`. diff --git a/crates/cu-profiler-cli/src/commands/mod.rs b/crates/cu-profiler-cli/src/commands/mod.rs index 0efe3ae..4a92f12 100644 --- a/crates/cu-profiler-cli/src/commands/mod.rs +++ b/crates/cu-profiler-cli/src/commands/mod.rs @@ -177,13 +177,20 @@ fn warn_if_live_mode(config: &Config, quiet: bool) { eprintln!(" or `cu-profiler import ` to profile a real transaction's logs."); } -/// Filter the configured scenarios by `--scenario` / `--tag`. +/// Filter the configured scenarios by `--scenario` / `--tag`, applying the +/// `--samples` override when given. fn select_scenarios(config: &Config, common: &CommonRun) -> Vec { config .scenarios() .into_iter() .filter(|s| common.scenarios.is_empty() || common.scenarios.contains(&s.name)) .filter(|s| common.tags.is_empty() || common.tags.iter().any(|t| s.has_tag(t))) + .map(|mut s| { + if let Some(n) = common.samples { + s.samples = n.max(1); + } + s + }) .collect() } diff --git a/crates/cu-profiler-core/src/backend/mod.rs b/crates/cu-profiler-core/src/backend/mod.rs index ad45227..74f8971 100644 --- a/crates/cu-profiler-core/src/backend/mod.rs +++ b/crates/cu-profiler-core/src/backend/mod.rs @@ -49,4 +49,15 @@ pub trait ExecutionBackend { /// Returns [`crate::Error::Simulation`] on execution failure, or /// [`crate::Error::BackendUnimplemented`] for skeleton backends. fn run(&self, scenario: &Scenario) -> Result; + + /// Whether re-running the same scenario always yields identical logs. + /// + /// Deterministic backends (e.g. recorded-log replay) gain nothing from + /// multi-sampling — `Scenario::samples` is ignored for them, so the profiler + /// never reports fabricated run-to-run variance. Live execution backends that + /// can vary between runs override this to `false` (the default) so sampling + /// applies. See [`crate::profiler::Profiler::run`]. + fn is_deterministic(&self) -> bool { + false + } } diff --git a/crates/cu-profiler-core/src/backend/recorded.rs b/crates/cu-profiler-core/src/backend/recorded.rs index e9e20d3..cc84199 100644 --- a/crates/cu-profiler-core/src/backend/recorded.rs +++ b/crates/cu-profiler-core/src/backend/recorded.rs @@ -43,6 +43,11 @@ impl ExecutionBackend for RecordedLogsBackend { BackendKind::Recorded } + /// Replaying a fixed log is deterministic — multi-sampling is a no-op here. + fn is_deterministic(&self) -> bool { + true + } + fn run(&self, scenario: &Scenario) -> Result { self.by_scenario .get(&scenario.name) diff --git a/crates/cu-profiler-core/src/confidence.rs b/crates/cu-profiler-core/src/confidence.rs index 2a4cbc9..279e14e 100644 --- a/crates/cu-profiler-core/src/confidence.rs +++ b/crates/cu-profiler-core/src/confidence.rs @@ -80,6 +80,9 @@ pub struct ConfidenceFactors { pub scope_markers: usize, /// Whether runtime/version metadata was available. pub metadata_available: bool, + /// Coefficient of variation of `total_cu` across samples, when multi-sampled. + /// `None` for a single sample / deterministic backend. + pub sample_cv: Option, } impl Default for ConfidenceFactors { @@ -92,6 +95,7 @@ impl Default for ConfidenceFactors { unattributed_pct: 0.0, scope_markers: 0, metadata_available: false, + sample_cv: None, } } } @@ -135,6 +139,17 @@ pub fn score(factors: &ConfidenceFactors) -> Confidence { if factors.scope_markers > 0 { reasons.push(format!("{} scope markers detected", factors.scope_markers)); } + // Run-to-run variance across samples: a wide spread means the headline number + // is less trustworthy. Thresholds are on the coefficient of variation. + if let Some(cv) = factors.sample_cv { + if cv >= 0.10 { + level = level.min(ConfidenceLevel::Low); + reasons.push(format!("high run-to-run variance (CV {:.1}%)", cv * 100.0)); + } else if cv >= 0.02 { + level = level.min(ConfidenceLevel::Medium); + reasons.push(format!("run-to-run variance (CV {:.1}%)", cv * 100.0)); + } + } if !factors.metadata_available { level = level.min(ConfidenceLevel::Medium); reasons.push("runtime/version metadata unavailable".to_string()); @@ -178,6 +193,34 @@ mod tests { assert!(c.reasons.iter().any(|r| r.contains("22% unattributed"))); } + #[test] + fn sample_variance_demotes_confidence() { + // Low spread → Medium; high spread → Low. + let medium = ConfidenceFactors { + metadata_available: true, + sample_cv: Some(0.05), + ..Default::default() + }; + let c = score(&medium); + assert_eq!(c.level, ConfidenceLevel::Medium); + assert!(c.reasons.iter().any(|r| r.contains("variance"))); + + let low = ConfidenceFactors { + metadata_available: true, + sample_cv: Some(0.25), + ..Default::default() + }; + assert_eq!(score(&low).level, ConfidenceLevel::Low); + + // A tiny spread is within tolerance and stays High. + let high = ConfidenceFactors { + metadata_available: true, + sample_cv: Some(0.005), + ..Default::default() + }; + assert_eq!(score(&high).level, ConfidenceLevel::High); + } + #[test] fn levels_order_high_above_low() { assert!(ConfidenceLevel::High > ConfidenceLevel::Low); diff --git a/crates/cu-profiler-core/src/config.rs b/crates/cu-profiler-core/src/config.rs index b049d63..36c9e5e 100644 --- a/crates/cu-profiler-core/src/config.rs +++ b/crates/cu-profiler-core/src/config.rs @@ -144,6 +144,10 @@ pub struct ScenarioConfig { pub tags: Vec, /// Description. pub description: String, + /// How many times to measure (>= 1). Only meaningful on non-deterministic + /// backends; the recorded backend ignores it. Defaults to 1. + #[serde(skip_serializing_if = "Option::is_none")] + pub samples: Option, } impl Config { @@ -227,7 +231,7 @@ impl Config { owner: None, expected: crate::scenario::ExpectedResult::Success, budget: self.effective_policy(name), - samples: 1, + samples: sc.samples.unwrap_or(1).max(1), }) .collect() } diff --git a/crates/cu-profiler-core/src/model.rs b/crates/cu-profiler-core/src/model.rs index ac6caa4..1553613 100644 --- a/crates/cu-profiler-core/src/model.rs +++ b/crates/cu-profiler-core/src/model.rs @@ -64,6 +64,75 @@ pub struct InstructionMeasurement { pub consumed: Option, } +/// Distribution of `total_cu` across multiple measurement samples. +/// +/// Only present when a scenario was run more than once on a *non-deterministic* +/// backend (`Scenario::samples > 1`); the recorded backend is deterministic, so it +/// never produces this — the tool never fabricates a spread it did not observe. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct SampleStats { + /// Number of samples taken. + pub count: u32, + /// Smallest observed `total_cu`. + pub min: u64, + /// Median observed `total_cu`. + pub median: u64, + /// Largest observed `total_cu`. + pub max: u64, + /// Population variance of `total_cu`. + pub variance: f64, + /// Standard deviation of `total_cu` (`sqrt(variance)`). + pub std_dev: f64, + /// Coefficient of variation (`std_dev / mean`); `0.0` when the mean is `0`. + pub cv: f64, +} + +impl SampleStats { + /// Compute statistics over the per-sample `total_cu` values, or `None` for + /// fewer than two samples (a single run has no distribution). + #[must_use] + pub fn from_samples(totals: &[u64]) -> Option { + if totals.len() < 2 { + return None; + } + let count = totals.len() as u32; + let min = *totals.iter().min()?; + let max = *totals.iter().max()?; + + let mut sorted = totals.to_vec(); + sorted.sort_unstable(); + let mid = sorted.len() / 2; + let median = if sorted.len() % 2 == 1 { + sorted[mid] + } else { + (sorted[mid - 1] + sorted[mid]) / 2 + }; + + let n = totals.len() as f64; + let mean = totals.iter().map(|&x| x as f64).sum::() / n; + let variance = totals + .iter() + .map(|&x| { + let d = x as f64 - mean; + d * d + }) + .sum::() + / n; + let std_dev = variance.sqrt(); + let cv = if mean == 0.0 { 0.0 } else { std_dev / mean }; + + Some(Self { + count, + min, + median, + max, + variance, + std_dev, + cv, + }) + } +} + /// The quantitative core of a scenario result. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Measurement { @@ -89,6 +158,10 @@ pub struct Measurement { /// Per-instruction breakdown. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub per_instruction: Vec, + /// Distribution of `total_cu` across samples, when multi-sampled (>1 run on a + /// non-deterministic backend). Absent for single-sample / recorded runs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sample_stats: Option, /// Whether the simulation completed successfully. pub simulation_success: bool, } @@ -108,6 +181,7 @@ impl Measurement { unattributed_pct: 0.0, instrumentation_overhead_pct: None, per_instruction: Vec::new(), + sample_stats: None, simulation_success: true, } } @@ -212,6 +286,39 @@ mod tests { use crate::confidence::Confidence; use crate::metadata::RunMetadata; + #[test] + fn sample_stats_needs_at_least_two_samples() { + assert!(SampleStats::from_samples(&[]).is_none()); + assert!(SampleStats::from_samples(&[42]).is_none()); + } + + #[test] + fn sample_stats_computes_spread() { + let s = SampleStats::from_samples(&[100, 120, 110]).expect("two+ samples"); + assert_eq!(s.count, 3); + assert_eq!(s.min, 100); + assert_eq!(s.max, 120); + assert_eq!(s.median, 110); + // mean 110, population variance ((100)+(100)+0)/3 = 66.67, std ~8.16. + assert!((s.variance - 66.6667).abs() < 0.01); + assert!((s.std_dev - 8.165).abs() < 0.01); + assert!((s.cv - 0.0742).abs() < 0.001); + } + + #[test] + fn sample_stats_identical_samples_have_zero_variance() { + let s = SampleStats::from_samples(&[500, 500, 500, 500]).unwrap(); + assert_eq!(s.variance, 0.0); + assert_eq!(s.cv, 0.0); + assert_eq!(s.median, 500); + } + + #[test] + fn sample_stats_even_count_medians_the_middle_pair() { + let s = SampleStats::from_samples(&[10, 20, 30, 40]).unwrap(); + assert_eq!(s.median, 25); // (20 + 30) / 2 + } + fn scenario(name: &str, status: Status, cu: u64) -> ScenarioReport { ScenarioReport { name: name.into(), diff --git a/crates/cu-profiler-core/src/profiler.rs b/crates/cu-profiler-core/src/profiler.rs index 1f47470..63b5161 100644 --- a/crates/cu-profiler-core/src/profiler.rs +++ b/crates/cu-profiler-core/src/profiler.rs @@ -94,13 +94,39 @@ impl Profiler { scenario: &Scenario, baseline: Option<&BaselineStore>, ) -> ScenarioReport { - match backend.run(scenario) { - Ok(output) => { - let analysis = parser::analyze(&output.logs, &self.registry); - self.assemble(scenario, analysis, output.success, output.logs, baseline) + // The first run drives the structural report (call tree, per-instruction, + // scopes). On a non-deterministic backend, extra samples add only their + // `total_cu` to a distribution; a deterministic backend (recorded replay) + // is run once — multi-sampling it would fabricate a spread of zero. + let output = match backend.run(scenario) { + Ok(output) => output, + Err(e) => return self.simulation_error_report(scenario, &e.to_string()), + }; + let analysis = parser::analyze(&output.logs, &self.registry); + + let extra = if backend.is_deterministic() { + 0 + } else { + u64::from(scenario.samples.saturating_sub(1)) + }; + let mut totals = vec![analysis.total_cu]; + for _ in 0..extra { + // Best-effort: a sample that fails to execute or parse is skipped, not + // fatal — the surviving samples still yield an honest distribution. + if let Ok(o) = backend.run(scenario) { + totals.push(parser::analyze(&o.logs, &self.registry).total_cu); } - Err(e) => self.simulation_error_report(scenario, &e.to_string()), } + let sample_stats = crate::model::SampleStats::from_samples(&totals); + + self.assemble( + scenario, + analysis, + output.success, + output.logs, + baseline, + sample_stats, + ) } fn assemble( @@ -110,6 +136,7 @@ impl Profiler { sim_success: bool, logs: Vec, baseline: Option<&BaselineStore>, + sample_stats: Option, ) -> ScenarioReport { // Each top-level (depth-1) invocation is one transaction instruction. let per_instruction: Vec = analysis @@ -135,6 +162,7 @@ impl Profiler { unattributed_pct: analysis.unattributed_pct, instrumentation_overhead_pct: None, per_instruction, + sample_stats, simulation_success: sim_success && analysis.simulation_success, }; @@ -160,7 +188,11 @@ impl Profiler { budget::evaluate(&measurement, &scenario.budget, baseline_units); // Confidence. - let confidence = self.score_confidence(&analysis, comparison.as_ref()); + let confidence = self.score_confidence( + &analysis, + comparison.as_ref(), + measurement.sample_stats.map(|s| s.cv), + ); // Status. let status = self.derive_status(&measurement, &policy_results, scenario.expected); @@ -198,6 +230,7 @@ impl Profiler { &self, analysis: &ParseAnalysis, comparison: Option<&BaselineComparison>, + sample_cv: Option, ) -> Confidence { // Unattributed CU only counts against confidence when the user opted // into scope attribution; otherwise it is just normal program work. @@ -214,6 +247,7 @@ impl Profiler { unattributed_pct, scope_markers: analysis.scope_marker_count, metadata_available: true, + sample_cv, }; confidence::score(&factors) } @@ -326,6 +360,68 @@ mod tests { ); } + /// A non-deterministic backend that returns a different CU figure per call, + /// cycling through `cus` — used to exercise multi-sampling and variance. + struct VaryingBackend { + calls: std::cell::Cell, + cus: Vec, + } + + impl crate::backend::ExecutionBackend for VaryingBackend { + fn kind(&self) -> crate::metadata::BackendKind { + crate::metadata::BackendKind::Mollusk + } + fn run(&self, _scenario: &Scenario) -> crate::Result { + let i = self.calls.get(); + self.calls.set(i + 1); + let cu = self.cus[i % self.cus.len()]; + Ok(crate::backend::SimulationOutput::success(vec![ + "Program P invoke [1]".to_string(), + format!("Program P consumed {cu} of 200000 compute units"), + "Program P success".to_string(), + ])) + } + } + + #[test] + fn multi_sample_records_variance_and_demotes_confidence() { + let backend = VaryingBackend { + calls: std::cell::Cell::new(0), + cus: vec![100_000, 120_000, 110_000], + }; + let mut s = swap_scenario(200_000); + s.samples = 3; + let report = Profiler::new().run(&backend, &[s], None, RunMetadata::recorded("0.1.0")); + + let stats = report.scenarios[0] + .measurement + .sample_stats + .expect("multi-sample stats present"); + assert_eq!(stats.count, 3); + assert_eq!(stats.min, 100_000); + assert_eq!(stats.max, 120_000); + assert!(stats.variance > 0.0); + // ~7.4% CV demotes confidence below High with a variance reason. + assert!(report.scenarios[0].confidence.level < confidence::ConfidenceLevel::High); + assert!( + report.scenarios[0] + .confidence + .reasons + .iter() + .any(|r| r.contains("variance")) + ); + } + + #[test] + fn deterministic_backend_ignores_samples() { + // The recorded backend is deterministic: samples > 1 must not fabricate a + // distribution — there is no real run-to-run spread to report. + let mut s = swap_scenario(200_000); + s.samples = 5; + let report = Profiler::new().run(&backend(), &[s], None, RunMetadata::recorded("0.1.0")); + assert!(report.scenarios[0].measurement.sample_stats.is_none()); + } + #[test] fn missing_scenario_yields_unknown() { let report = Profiler::new().run( diff --git a/docs/reference.md b/docs/reference.md index 600dc14..5c14fb5 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -438,6 +438,11 @@ scope marker quality runtime/version metadata availability ``` +**Sample variance** folds in when a scenario is multi-sampled (`samples > 1`) on a +non-deterministic backend: a high coefficient of variation across the runs demotes +confidence (≥2% → Medium, ≥10% → Low) with a reason. The deterministic recorded +backend ignores `samples`, so it never reports variance it did not observe. + Always report **why** confidence is not `High`. Example: ```