feat(core): multi-sample runs + variance, folded into confidence

CHANGELOG.md

@@ -14,6 +14,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
   `--input report.md`, `--pr`, `--repo`, `--marker`, and `--dry-run` (no network).
   `init --workflow` now scaffolds a render-and-comment step with
   `permissions: pull-requests: write`.
+- **Multi-sample runs + variance.** `Scenario.samples` (and a `--samples` override)
+  now run a scenario N times on non-deterministic backends and record a `SampleStats`
+  distribution (count/min/median/max/variance/std-dev/CV) in the measurement. Variance
+  folds into the confidence score (CV ≥2% → Medium, ≥10% → Low), implementing the
+  spec §12 "sample variance" factor. The deterministic recorded backend ignores
+  `samples` and never fabricates a spread (`ExecutionBackend::is_deterministic`).
 
 ## [0.1.2] - 2026-06-20
 

crates/cu-profiler-cli/src/args.rs

@@ -68,6 +68,11 @@ pub struct CommonRun {
     /// Only run scenarios carrying these tags (repeatable).
     #[arg(long = "tag")]
     pub tags: Vec<String>,
+
+    /// Override the per-scenario sample count (number of measurement runs).
+    /// Only affects non-deterministic backends; the recorded backend ignores it.
+    #[arg(long)]
+    pub samples: Option<u32>,
 }
 
 /// `cu-profiler run` / `cu-profiler ci`.

crates/cu-profiler-cli/src/commands/mod.rs

@@ -177,13 +177,20 @@ fn warn_if_live_mode(config: &Config, quiet: bool) {
     eprintln!("  or `cu-profiler import <tx.json>` to profile a real transaction's logs.");
 }
 
-/// Filter the configured scenarios by `--scenario` / `--tag`.
+/// Filter the configured scenarios by `--scenario` / `--tag`, applying the
+/// `--samples` override when given.
 fn select_scenarios(config: &Config, common: &CommonRun) -> Vec<Scenario> {
     config
         .scenarios()
         .into_iter()
         .filter(|s| common.scenarios.is_empty() || common.scenarios.contains(&s.name))
         .filter(|s| common.tags.is_empty() || common.tags.iter().any(|t| s.has_tag(t)))
+        .map(|mut s| {
+            if let Some(n) = common.samples {
+                s.samples = n.max(1);
+            }
+            s
+        })
         .collect()
 }
 

crates/cu-profiler-core/src/backend/mod.rs

@@ -49,4 +49,15 @@ pub trait ExecutionBackend {
     /// Returns [`crate::Error::Simulation`] on execution failure, or
     /// [`crate::Error::BackendUnimplemented`] for skeleton backends.
     fn run(&self, scenario: &Scenario) -> Result<SimulationOutput>;
+
+    /// Whether re-running the same scenario always yields identical logs.
+    ///
+    /// Deterministic backends (e.g. recorded-log replay) gain nothing from
+    /// multi-sampling — `Scenario::samples` is ignored for them, so the profiler
+    /// never reports fabricated run-to-run variance. Live execution backends that
+    /// can vary between runs override this to `false` (the default) so sampling
+    /// applies. See [`crate::profiler::Profiler::run`].
+    fn is_deterministic(&self) -> bool {
+        false
+    }
 }

crates/cu-profiler-core/src/backend/recorded.rs

@@ -43,6 +43,11 @@ impl ExecutionBackend for RecordedLogsBackend {
         BackendKind::Recorded
     }
 
+    /// Replaying a fixed log is deterministic — multi-sampling is a no-op here.
+    fn is_deterministic(&self) -> bool {
+        true
+    }
+
     fn run(&self, scenario: &Scenario) -> Result<SimulationOutput> {
         self.by_scenario
             .get(&scenario.name)

crates/cu-profiler-core/src/confidence.rs

@@ -80,6 +80,9 @@ pub struct ConfidenceFactors {
     pub scope_markers: usize,
     /// Whether runtime/version metadata was available.
     pub metadata_available: bool,
+    /// Coefficient of variation of `total_cu` across samples, when multi-sampled.
+    /// `None` for a single sample / deterministic backend.
+    pub sample_cv: Option<f64>,
 }
 
 impl Default for ConfidenceFactors {
@@ -92,6 +95,7 @@ impl Default for ConfidenceFactors {
             unattributed_pct: 0.0,
             scope_markers: 0,
             metadata_available: false,
+            sample_cv: None,
         }
     }
 }
@@ -135,6 +139,17 @@ pub fn score(factors: &ConfidenceFactors) -> Confidence {
     if factors.scope_markers > 0 {
         reasons.push(format!("{} scope markers detected", factors.scope_markers));
     }
+    // Run-to-run variance across samples: a wide spread means the headline number
+    // is less trustworthy. Thresholds are on the coefficient of variation.
+    if let Some(cv) = factors.sample_cv {
+        if cv >= 0.10 {
+            level = level.min(ConfidenceLevel::Low);
+            reasons.push(format!("high run-to-run variance (CV {:.1}%)", cv * 100.0));
+        } else if cv >= 0.02 {
+            level = level.min(ConfidenceLevel::Medium);
+            reasons.push(format!("run-to-run variance (CV {:.1}%)", cv * 100.0));
+        }
+    }
     if !factors.metadata_available {
         level = level.min(ConfidenceLevel::Medium);
         reasons.push("runtime/version metadata unavailable".to_string());
@@ -178,6 +193,34 @@ mod tests {
         assert!(c.reasons.iter().any(|r| r.contains("22% unattributed")));
     }
 
+    #[test]
+    fn sample_variance_demotes_confidence() {
+        // Low spread → Medium; high spread → Low.
+        let medium = ConfidenceFactors {
+            metadata_available: true,
+            sample_cv: Some(0.05),
+            ..Default::default()
+        };
+        let c = score(&medium);
+        assert_eq!(c.level, ConfidenceLevel::Medium);
+        assert!(c.reasons.iter().any(|r| r.contains("variance")));
+
+        let low = ConfidenceFactors {
+            metadata_available: true,
+            sample_cv: Some(0.25),
+            ..Default::default()
+        };
+        assert_eq!(score(&low).level, ConfidenceLevel::Low);
+
+        // A tiny spread is within tolerance and stays High.
+        let high = ConfidenceFactors {
+            metadata_available: true,
+            sample_cv: Some(0.005),
+            ..Default::default()
+        };
+        assert_eq!(score(&high).level, ConfidenceLevel::High);
+    }
+
     #[test]
     fn levels_order_high_above_low() {
         assert!(ConfidenceLevel::High > ConfidenceLevel::Low);

crates/cu-profiler-core/src/config.rs

@@ -144,6 +144,10 @@ pub struct ScenarioConfig {
     pub tags: Vec<String>,
     /// Description.
     pub description: String,
+    /// How many times to measure (>= 1). Only meaningful on non-deterministic
+    /// backends; the recorded backend ignores it. Defaults to 1.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub samples: Option<u32>,
 }
 
 impl Config {
@@ -227,7 +231,7 @@ impl Config {
                 owner: None,
                 expected: crate::scenario::ExpectedResult::Success,
                 budget: self.effective_policy(name),
-                samples: 1,
+                samples: sc.samples.unwrap_or(1).max(1),
             })
             .collect()
     }

crates/cu-profiler-core/src/model.rs

@@ -64,6 +64,75 @@ pub struct InstructionMeasurement {
     pub consumed: Option<u64>,
 }
 
+/// Distribution of `total_cu` across multiple measurement samples.
+///
+/// Only present when a scenario was run more than once on a *non-deterministic*
+/// backend (`Scenario::samples > 1`); the recorded backend is deterministic, so it
+/// never produces this — the tool never fabricates a spread it did not observe.
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
+pub struct SampleStats {
+    /// Number of samples taken.
+    pub count: u32,
+    /// Smallest observed `total_cu`.
+    pub min: u64,
+    /// Median observed `total_cu`.
+    pub median: u64,
+    /// Largest observed `total_cu`.
+    pub max: u64,
+    /// Population variance of `total_cu`.
+    pub variance: f64,
+    /// Standard deviation of `total_cu` (`sqrt(variance)`).
+    pub std_dev: f64,
+    /// Coefficient of variation (`std_dev / mean`); `0.0` when the mean is `0`.
+    pub cv: f64,
+}
+
+impl SampleStats {
+    /// Compute statistics over the per-sample `total_cu` values, or `None` for
+    /// fewer than two samples (a single run has no distribution).
+    #[must_use]
+    pub fn from_samples(totals: &[u64]) -> Option<Self> {
+        if totals.len() < 2 {
+            return None;
+        }
+        let count = totals.len() as u32;
+        let min = *totals.iter().min()?;
+        let max = *totals.iter().max()?;
+
+        let mut sorted = totals.to_vec();
+        sorted.sort_unstable();
+        let mid = sorted.len() / 2;
+        let median = if sorted.len() % 2 == 1 {
+            sorted[mid]
+        } else {
+            (sorted[mid - 1] + sorted[mid]) / 2
+        };
+
+        let n = totals.len() as f64;
+        let mean = totals.iter().map(|&x| x as f64).sum::<f64>() / n;
+        let variance = totals
+            .iter()
+            .map(|&x| {
+                let d = x as f64 - mean;
+                d * d
+            })
+            .sum::<f64>()
+            / n;
+        let std_dev = variance.sqrt();
+        let cv = if mean == 0.0 { 0.0 } else { std_dev / mean };
+
+        Some(Self {
+            count,
+            min,
+            median,
+            max,
+            variance,
+            std_dev,
+            cv,
+        })
+    }
+}
+
 /// The quantitative core of a scenario result.
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct Measurement {
@@ -89,6 +158,10 @@ pub struct Measurement {
     /// Per-instruction breakdown.
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub per_instruction: Vec<InstructionMeasurement>,
+    /// Distribution of `total_cu` across samples, when multi-sampled (>1 run on a
+    /// non-deterministic backend). Absent for single-sample / recorded runs.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub sample_stats: Option<SampleStats>,
     /// Whether the simulation completed successfully.
     pub simulation_success: bool,
 }
@@ -108,6 +181,7 @@ impl Measurement {
             unattributed_pct: 0.0,
             instrumentation_overhead_pct: None,
             per_instruction: Vec::new(),
+            sample_stats: None,
             simulation_success: true,
         }
     }
@@ -212,6 +286,39 @@ mod tests {
     use crate::confidence::Confidence;
     use crate::metadata::RunMetadata;
 
+    #[test]
+    fn sample_stats_needs_at_least_two_samples() {
+        assert!(SampleStats::from_samples(&[]).is_none());
+        assert!(SampleStats::from_samples(&[42]).is_none());
+    }
+
+    #[test]
+    fn sample_stats_computes_spread() {
+        let s = SampleStats::from_samples(&[100, 120, 110]).expect("two+ samples");
+        assert_eq!(s.count, 3);
+        assert_eq!(s.min, 100);
+        assert_eq!(s.max, 120);
+        assert_eq!(s.median, 110);
+        // mean 110, population variance ((100)+(100)+0)/3 = 66.67, std ~8.16.
+        assert!((s.variance - 66.6667).abs() < 0.01);
+        assert!((s.std_dev - 8.165).abs() < 0.01);
+        assert!((s.cv - 0.0742).abs() < 0.001);
+    }
+
+    #[test]
+    fn sample_stats_identical_samples_have_zero_variance() {
+        let s = SampleStats::from_samples(&[500, 500, 500, 500]).unwrap();
+        assert_eq!(s.variance, 0.0);
+        assert_eq!(s.cv, 0.0);
+        assert_eq!(s.median, 500);
+    }
+
+    #[test]
+    fn sample_stats_even_count_medians_the_middle_pair() {
+        let s = SampleStats::from_samples(&[10, 20, 30, 40]).unwrap();
+        assert_eq!(s.median, 25); // (20 + 30) / 2
+    }
+
     fn scenario(name: &str, status: Status, cu: u64) -> ScenarioReport {
         ScenarioReport {
             name: name.into(),