From b3237d60e3aadb14a49882fb95cc3e94e0420254 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sun, 1 Nov 2026 09:43:13 -0700 Subject: [PATCH 01/14] Day metrics and integration tools need reusable calculation state --- ALGORITHMS.md | 127 +- CHANGELOG.md | 202 +- README.md | 19 +- docs/COMBINED_BUMP_STATUS.md | 235 ++ docs/INCREMENTAL_INTEGRATION_PLAN.md | 168 + docs/INCREMENTAL_MATH.md | 513 +++ docs/INCREMENTAL_USAGE.md | 90 + lib/onehz.dart | 6 +- lib/src/onehz/incremental.dart | 27 + lib/src/onehz/incremental_enmo.dart | 198 + lib/src/onehz/incremental_hrv.dart | 266 ++ lib/src/onehz/incremental_minutes.dart | 376 ++ lib/src/onehz/incremental_state.dart | 50 + lib/src/onehz/registry/group_registry.dart | 417 +++ lib/src/onehz/workout/calories.dart | 468 ++- test/aggregate/adopted_aggregators_test.dart | 228 ++ test/aggregate/persistence_test.dart | 986 +++++ test/algo_version_test.dart | 20 + test/incremental/day_curves_test.dart | 192 + .../incremental/calculation_cache_test.dart | 190 + .../checkpoint_ownership_test.dart | 51 + test/onehz/incremental/drift_test.dart | 123 + test/onehz/incremental/enmo_series_test.dart | 123 + test/onehz/incremental/hrv_time_test.dart | 158 + .../incremental/minute_metrics_test.dart | 251 ++ .../incremental/minute_summary_test.dart | 281 ++ test/onehz/minute_energy_test.dart | 360 ++ test/onehz/nonfinite_beat_times_test.dart | 135 + test/onehz/phase0_parity_test.dart | 1047 ++++++ test/onehz/support/energy_day.dart | 56 + test/onehz/support/incremental_compare.dart | 68 + test/properties/incremental_laws_test.dart | 3172 +++++++++++++++++ test/registry_closure_test.dart | 414 +++ tool/bench_aggregate.dart | 215 ++ tool/bench_night_bins.dart | 173 + tool/bench_night_nn.dart | 350 ++ tool/bench_phase0.dart | 296 ++ tool/combined_bump_probe.dart | 189 + tool/incremental/bench_all.dart | 283 ++ tool/incremental/size_probe.dart | 27 + tool/incremental_benchmark.dart | 142 + 41 files changed, 12530 insertions(+), 162 deletions(-) create mode 100644 docs/COMBINED_BUMP_STATUS.md create mode 100644 docs/INCREMENTAL_INTEGRATION_PLAN.md create mode 100644 docs/INCREMENTAL_MATH.md create mode 100644 docs/INCREMENTAL_USAGE.md create mode 100644 lib/src/onehz/incremental.dart create mode 100644 lib/src/onehz/incremental_enmo.dart create mode 100644 lib/src/onehz/incremental_hrv.dart create mode 100644 lib/src/onehz/incremental_minutes.dart create mode 100644 lib/src/onehz/incremental_state.dart create mode 100644 lib/src/onehz/registry/group_registry.dart create mode 100644 test/aggregate/adopted_aggregators_test.dart create mode 100644 test/aggregate/persistence_test.dart create mode 100644 test/algo_version_test.dart create mode 100644 test/incremental/day_curves_test.dart create mode 100644 test/onehz/incremental/calculation_cache_test.dart create mode 100644 test/onehz/incremental/checkpoint_ownership_test.dart create mode 100644 test/onehz/incremental/drift_test.dart create mode 100644 test/onehz/incremental/enmo_series_test.dart create mode 100644 test/onehz/incremental/hrv_time_test.dart create mode 100644 test/onehz/incremental/minute_metrics_test.dart create mode 100644 test/onehz/incremental/minute_summary_test.dart create mode 100644 test/onehz/minute_energy_test.dart create mode 100644 test/onehz/nonfinite_beat_times_test.dart create mode 100644 test/onehz/phase0_parity_test.dart create mode 100644 test/onehz/support/energy_day.dart create mode 100644 test/properties/incremental_laws_test.dart create mode 100644 test/registry_closure_test.dart create mode 100644 tool/bench_aggregate.dart create mode 100644 tool/bench_night_bins.dart create mode 100644 tool/bench_night_nn.dart create mode 100644 tool/bench_phase0.dart create mode 100644 tool/combined_bump_probe.dart create mode 100644 tool/incremental/bench_all.dart create mode 100644 tool/incremental/size_probe.dart create mode 100644 tool/incremental_benchmark.dart diff --git a/ALGORITHMS.md b/ALGORITHMS.md index efbd80f2..7de6b20c 100644 --- a/ALGORITHMS.md +++ b/ALGORITHMS.md @@ -1,13 +1,14 @@ # OpenStrap Analytics — Algorithms -Every metric here is a **published, peer-reviewed method**, computed by a **pure -deterministic function**, running on the 1 Hz substrate a WHOOP 4.0 actually exposes: +This index includes published methods and internal rules. Unpublished rules are +identified below as ESTIMATE with low confidence. Computations are pure and +deterministic, running on the 1 Hz substrate a WHOOP 4.0 actually exposes: beat-to-beat RR (pulse-derived — this is **PRV, not ECG-HRV**, and that distinction matters more than it sounds like it should), 1 Hz heart rate, 1 Hz tri-axial accel, and relative-only ADC channels (skin temp, SpO2, ambient light). This file is the index — what's implemented, where it lives, and who published it. If a -row here doesn't have a real citation next to it, that's a bug in this file, go fix it. +row describes an internal rule, its component citations support only those components. See `docs/ALGORITHM_CATALOG_1HZ.md` for the underlying literature review and what's deliberately not shipped. @@ -30,8 +31,8 @@ A few multi-day/list outputs (`illnessCusum`, `multivariateAnomaly`, confidence/tier that applies across a whole list. **Tiers**: `AUTH` = directly measured/definitional. `HIGH` = strong literature support on -this substrate. `ESTIMATE` = published, but estimate-grade once run on a wrist 1 Hz -signal instead of a lab setup. `RELATIVE` = meaningful only as a deviation from the +this substrate. `ESTIMATE` = estimate-grade on a wrist 1 Hz signal, or an internal rule without +published validation. `RELATIVE` = meaningful only as a deviation from the user's own baseline (skin temp, SpO2 — never an absolute unit). **Absent input → `null` + confidence `0`, always.** A baseline-gated metric that doesn't @@ -47,33 +48,33 @@ Grouped by family (subdirectory under `lib/src/onehz/`). File paths are relative ### `foundations/` — shared math every family builds on | Function | File | Method | |---|---|---| -| `correctRr` | `foundations/rr_correction.dart` | Lipponen & Tarvainen 2019 RR artifact correction (dRR/mRR/sRR beat classification, Kubios-style) | +| `correctRr` | `foundations/rr_correction.dart` | Lipponen & Tarvainen 2019 RR artifact correction (dRR/mRR/sRR classification); Peltola 2012 correction policy; unpublished artifact-run cap is an internal rule, ESTIMATE with low confidence | | `Baselines` (Winsorized-EWMA) | `foundations/ewma_baselines.dart` | Winsorized exponentially-weighted moving baseline — the rolling personal reference most other metrics compare against | | inverse-variance fusion | `foundations/fusion.dart` | Standard inverse-variance weighting for combining multiple noisy estimates of the same quantity | ### `clinical/` — Tier-1 cardiac/autonomic metrics | Function | File | Method | Citation | |---|---|---|---| -| `hrvTime` | `clinical/hrv_time.dart` | RMSSD/SDNN/pNNx | standard time-domain HRV | +| `hrvTime` | `clinical/hrv_time.dart` | RMSSD/SDNN/pNNx | Task Force 1996 | | `nocturnalRmssd` | `clinical/hrv_time.dart` | median-of-5-min-window nightly RMSSD | — | | `sleepSessionWindowedRmssd` / `sleepSessionRmssdDetail` | `clinical/hrv_time.dart` | mean-of-5-min-window RMSSD; windows need ≥ 20 clean successive differences (ultra-short RMSSD literature); Malik 20 % ectopic rejection | Malik et al.; Munoz et al. 2015; Baek et al. 2015 | | `rrCoverage` | `clinical/hrv_time.dart` | Σ plausible RR ÷ wall-clock span; > 1.10 means duplicated or interleaved beats, and `hrvTime` / `nocturnalRmssd` / `nightHrvShape` / the sleep-session headline refuse RMSSD; the headline also drops any 5-min window holding more beat-time than its length | — (integrity check; contiguous runs measure 0.963–1.001) | | `hrvFreq` | `clinical/hrv_freq.dart` | LF/HF via Lomb-Scargle periodogram on native (unevenly-sampled) beat times | Laguna, Moody & Mark 1998; Bigger 1992 | | `decelerationCapacity` / `accelerationCapacity` | `clinical/prsa.dart` | Phase-rectified signal averaging (DC/AC) | Bauer et al. 2006 | -| `nocturnalRhr` / `hrDip` | `clinical/nocturnal.dart` | nocturnal resting HR + dip classification | — | +| `nocturnalRhr` / `hrDip` | `clinical/nocturnal.dart` | nocturnal resting HR + internal HR-dip classification rule, ESTIMATE with low confidence | Avram 2019; Dial 2025 support nocturnal RHR only; HR-dip bands are unvalidated | | `illnessCusum` | `clinical/illness_cusum.dart` | Online CUSUM state machine (green/yellow/red) over RHR — "NightSignal" | Alavi et al. 2022; Mishra et al. 2020 | | `readinessLnRmssd` | `clinical/readiness_lnrmssd.dart` | ln(RMSSD) z-scored against a rolling prior-nights baseline | Plews et al. 2013 | | `cosinor` | `clinical/cosinor.dart` | Cosinor rhythmometry (MESOR/amplitude/acrophase) | Halberg & Nelson 1979 | -| `banisterTrimp` | `clinical/load_trimp.dart` | Training impulse from HR-reserve | Banister 1991 | +| `banisterTrimp` | `clinical/load_trimp.dart` | Training impulse from HR-reserve | Morton, Fitz-Clarke & Banister 1990; sex-specific coefficients retained from Banister 1991 | | `dailyQuietWakingHrr` | `clinical/load_trimp.dart` | one day's quiet-waking level: median per-minute %HRR over waking minutes; null above 40 % HRR | Karvonen 1957; ACSM moderate-intensity floor | | `personalQuietWakingHrr` | `clinical/load_trimp.dart` | THIS user's quiet-waking level: median of the trailing 28 prior days' `dailyQuietWakingHrr`; abstains (need_baseline) below 3 days, "calibrating" below 7 | — | | `netTrimpAboveQuiet` | `clinical/load_trimp.dart` | Banister TRIMP above the quiet level, split at 40 % HRR so quiet minutes never offset exercise minutes (living block floored at 0) | Banister 1991; Morton 1990; ACSM 40 % HRR | | `strainScoreFromSeries` | `clinical/load_trimp.dart` | **the headline 0-21 strain** from per-minute wake HR: `netTrimpAboveQuiet` log-mapped onto 0-21 | — | | `strainCurveFromSeries` | `clinical/load_trimp.dart` | cumulative 0-21 strain per minute, same arithmetic; last point == headline | — | | `strainScoreMetric` | `clinical/load_trimp.dart` | LUMP form (TRIMP minus wake-minutes × quiet cost) log-squashed onto 0-21; can debit exercise, kept only for rescaling a stored TRIMP with no per-minute series | — | -| `ctlAtlTsb` | `clinical/load_trimp.dart` | Fitness-Fatigue-Form: EWMA CTL (42d) / ATL (7d) / TSB = CTL-ATL | Banister impulse-response model | +| `ctlAtlTsb` | `clinical/load_trimp.dart` | Fitness-Fatigue-Form: EWMA CTL (42d) / ATL (7d) / TSB = CTL-ATL | Banister et al. 1975; Morton et al. 1990; 42/7-day defaults and priming are implementation choices | | `baevskyStressIndex` | `clinical/stress_si.dart` | Baevsky Stress Index, 5-min sliding RR-histogram | Baevsky & Berseneva 2008 | -| `irregularBeatScreen` | `clinical/irregular_rhythm.dart` | Poincaré SD1/SD2 + pNN70 irregular-rhythm screen | — (screen, not a diagnosis; no ECG) | +| `irregularBeatScreen` | `clinical/irregular_rhythm.dart` | Poincaré SD1/SD2 + pNN70 irregular-rhythm screen | Internal rule, ESTIMATE with low confidence; Task Force 1996 supports successive-difference components only; SD1/SD2 thresholds and sustained-window voting are unvalidated | | `cardiacCoherence` | `clinical/cardiac_coherence.dart` | peak-band vs. remaining spectral power ratio, live RR during guided breathing | McCraty & Zayas 2014; McCraty et al. 2009 | ### `sleep/` — staging, structure, circadian @@ -84,10 +85,12 @@ Grouped by family (subdirectory under `lib/src/onehz/`). File paths are relative | `detectNaps` | `sleep/nap.dart` | **the only nap source.** z-angle immobility bouts on the complement of the main sleep window, corroborated by an HR dip vs the user's *awake daytime* baseline. Reports TST and TIB separately; makes **no stage claim**. Deliberately NOT `AdvancedSleepStager`, whose `minSleepMin=60` + 90-min daytime guard exist to reject naps | van Hees et al. | | `segmentSleep` (`SleepSegmentation`) | `sleep/segment.dart` | **single source of truth** for sleep windowing; delegates staging to `cardioStager` | | `cardioStager` | `sleep/cardio_stager.dart` | Webster/Cole-Kripke actigraphy + HRV fusion — explicitly replaces Walch 2019 (documented WAKE over-call bias) | +| `CausalStager.observe` | `sleep/causal_stager.dart` | **online** wake/NREM/REM for live decisions: `cardioStager`'s features, REM weights and cutoffs on past-only (trailing) windows and expanding baselines; returns stage + confidence + evidence age + a serialisable next state, and **abstains with a reason** (stale/off-wrist/missing HR or accel/thin coverage/warm-up) rather than guess. No look-ahead, deterministic, replacement-tolerant. Not PSG, not a replacement for `cardioStager` — see `docs/CAUSAL_STAGER.md` | `cardioStager`'s basis (linear-discriminant cardiorespiratory staging: Long 2016; Fonseca 2015/2018; weights measured on DREAMT); the causal adaptation is an unpublished internal rule, ESTIMATE with low confidence | +| `stageMinuteRanges` | `sleep/stage_ranges.dart` | per-stage [lo, hi] minutes from `cardioStager`'s `epochEvidence`: uncovered epochs could be any stage, scores within `kStageScoreMargin` of a cutoff count toward both neighbours. A bound, not a confidence interval | Internal margin/bound calibration rule, ESTIMATE with low confidence; underlying staging: Long 2016; Fonseca 2015/2018 | | `AdvancedSleepStager` | `sleep/advanced_stager.dart` | AASM-style 4-class staging + hypnogram metrics (TIB/TST/SOL/WASO/REM-latency); `StagingMethod.cardio` is the wired production default, v1/v2 kept for regression coverage only | | `sleepAccounting` | `sleep/accounting.dart` | TIB/TST/WASO/efficiency accounting | — | | `sleepCyclesMetric` | `sleep/cycles.dart` | fractal ultradian sleep-cycle detection | Rosenblum et al. 2024 | -| `circadianNonparametric` | `sleep/circadian_np.dart` | IS/IV/RA/L5/M10 nonparametric circadian indices | — | +| `circadianNonparametric` | `sleep/circadian_np.dart` | IS/IV/RA/L5/M10 nonparametric circadian indices | Witting 1990; van Someren 1999 | | `phillipsSri` | `sleep/sri.dart` | true Phillips epoch-agreement Sleep Regularity Index | Phillips et al. 2017 | ### `respiration/` @@ -95,7 +98,7 @@ Grouped by family (subdirectory under `lib/src/onehz/`). File paths are relative |---|---|---|---| | `rsaRespRate` | `respiration/resp_rate.dart` | respiratory sinus arrhythmia — HF spectral peak of the RR series: Lomb–Scargle on native beat times in 300 s Welch sub-windows (≥ 80 % covered by clean beats), median across them with an agreement gate; beat-rate Nyquist from the median NN | Welch 1967; Lomb 1976; Scargle 1982; Press & Rybicki 1989; DeBoer, Karemaker & Strackee 1984 | | `riivRespRate` / `fuseRespRate` | `respiration/resp_rate.dart` | respiration-induced intensity variation, fused with the RSA estimate | Pimentel et al. (multi-grid RIIV fusion) | -| `cvhrApnea` / `cvhrApneaScreen` | `respiration/cvhr_apnea.dart` | cyclic-variation-in-HR apnea screening | — | +| `cvhrApnea` / `cvhrApneaScreen` | `respiration/cvhr_apnea.dart` | cyclic-variation-in-HR apnea screening | Hayano 2011 | | `relativeOdi` | `respiration/relative_odi.dart` | ratio-of-ratios relative desaturation index — **never an absolute SpO2 claim** | — | | `breathingRateVariability` | `respiration/brv_trend.dart` | within-user breathing-rate variability trend | — | @@ -114,6 +117,7 @@ Grouped by family (subdirectory under `lib/src/onehz/`). File paths are relative | `hrRecovery` | `workout/hr_recovery.dart` | HRR — HR drop N seconds post-peak | Cole/Lauer 1999-style HRR | | `Calories.dailyEnergy` / `estimateBoutCalories` | `workout/calories.dart` | Keytel HR→kcal regression + Harris-Benedict/Mifflin BMR | Keytel et al. 2005 | | `Calories.metFromCadenceSpm` | `workout/calories.dart` | walking METs from measured cadence, for sub-flex-gate minutes (100/110/120/130 spm ↔ 3/4/5/6 METs, linear between, clamped) | Tudor-Locke et al. 2019 (CADENCE-Adults) | +| `Calories.minuteEnergy` / `hourlyRollup` | `workout/calories.dart` | the `dailyEnergy` computation exposed per minute (same gate, same billing; folds back to the daily `active`/`walking` bit-for-bit) plus an hourly rollup with per-hour coverage. A minute with no usable HR and no billable cadence abstains (null + reason); nothing is filled or interpolated | Keytel et al. 2005; Tudor-Locke et al. 2019 | ### `wellness/` | Function | File | Method | Citation | @@ -136,6 +140,45 @@ Grouped by family (subdirectory under `lib/src/onehz/`). File paths are relative | `sleepNeed` / `strainTarget` / `recommendedBedtime` / `recommendedWake` / `sleepPerformance` | `human/coaching.dart` | deterministic coaching layer over the metrics above (naps come from `sleep/nap.dart`, not here) | — | | `journalCorrelations` | `human/coaching.dart` | per-tag mean-difference correlation vs. logged outcomes, on-device, personal | — | +### Mergeable and incremental forms + +Paths below are relative to `lib/src/onehz/`. Structural combinators, codecs, +checkpoints and caches add no physiological estimator. Tolerances in the adjacent +code comments describe parity tests, not universal numerical guarantees. + +| Function | File | Method | Citation | +|---|---|---|---| +| `cosinor / cosinorAggregator` | `clinical/cosinor_aggregator.dart` | Centered least-squares sufficient statistics; relative singularity gate | Halberg & Nelson 1979 | +| `hrvTimeAggregator` | `clinical/hrv_time_aggregator.dart` | Mergeable time-domain statistics; timestamp behavior requires gapAware:true | Task Force 1996; internal jitter rule is ESTIMATE with low confidence; Welch 1967; Hirsch & Bishop 1981 | +| `RMSSD jitter gate / JitterEvidenceAggregator` | `clinical/rmssd_gate.dart; clinical/hrv_jitter_evidence.dart` | Shared spectral evidence; internal rejection/exemption threshold rule, ESTIMATE with low confidence | Welch 1967; Hirsch & Bishop 1981 | +| `banisterTrimpAggregator` | `clinical/load_trimp_aggregator.dart` | Mergeable per-minute Banister TRIMP sum | Morton, Fitz-Clarke & Banister 1990; sex-specific coefficients retained from Banister 1991 | +| `ctlAtlTsbAggregator` | `clinical/load_trimp_aggregator.dart` | Bounded-prefix priming and affine EWMA composition | Banister et al. 1975; Morton et al. 1990; 42/7-day defaults and priming are implementation choices | +| `prsaAggregator` | `clinical/prsa_aggregator.dart` | Mergeable PRSA anchor/profile sums | Bauer et al. 2006 | +| `readinessLnRmssdFromSuffix / readinessLnRmssdAggregator` | `clinical/readiness_lnrmssd.dart; clinical/readiness_lnrmssd_aggregator.dart` | Shared readiness arithmetic over the required suffix; positional aggregator history | Plews et al. 2013 | +| `circadianNpAggregator` | `sleep/circadian_np_aggregator.dart` | Mergeable IS/IV/RA/L5/M10; near-tie start epochs can differ | Witting 1990; van Someren 1999 | +| `bestCircularWindow` | `sleep/circadian_window.dart` | Shared circular L5/M10 search | Witting 1990; van Someren 1999 | +| `sriAggregator` | `sleep/sri_aggregator.dart` | Integer adjacent-day epoch agreement counts | Phillips et al. 2017 | +| `HypnogramAggregator / AdvancedSleepStager.hypnogramMetrics` | `sleep/hypnogram_aggregator.dart` | TST sums sleep stages; SOL from session start; efficiency = TST/session duration, capped at 1. Session bounds substitute for lights-off/on; inferred stages, explicit wake only between first and final sleep, no gap imputation or segment clipping | Berry et al., AASM Manual; application boundary conventions differ as stated | +| `correctRr / RrCapWindow / RrCorrector` | `foundations/rr_correction.dart; foundations/rr_correction_kernel.dart; foundations/rr_correction_stream.dart` | Shared capped batch/stream correction; internal artifact-run cap rule is ESTIMATE with low confidence | Lipponen & Tarvainen 2019; Peltola 2012 | +| `RrSandwich / NnColumn` | `foundations/rr_sandwich.dart` | Re-corrects raw seams, preserves source timestamps; uncapped long-run limitation | Lipponen & Tarvainen 2019; Peltola 2012; inner method attribution applies | +| `EpochRows / EpochRowsMemo / CardioRowCache / cardioEpochFeatures / classifyCardioEpochs` | `sleep/epoch_rows.dart; sleep/cardio_row_cache.dart; sleep/cardio_stager.dart` | Session-anchored feature rows and dependency-checked reuse; internal fitted weights and calibration rules are ESTIMATE with low confidence | van Hees 2013; Long 2016; Fonseca 2015/2018 | +| `HrvFreqStream` | `clinical/hrv_freq_stream.dart` | Append-only batch-equivalent Welch band processing | Laguna, Moody & Mark 1998; Bigger 1992; Welch 1967 | +| `RsaRespStream` | `respiration/resp_rate.dart` | Append-only RSA windows and exact median-NN ceiling | Welch 1967; Lomb 1976; Scargle 1982; Press & Rybicki 1989; DeBoer, Karemaker & Strackee 1984 | +| `CvhrStream` | `respiration/cvhr_apnea.dart` | Shared batch/stream CVHR screening | Hayano 2011 | +| `irregularBeatScreenDetailed / irregularWindowVerdict / IrregularScreenState` | `clinical/irregular_rhythm.dart; clinical/irregular_window.dart; clinical/irregular_rhythm_state.dart; clinical/irregular_diagnostics.dart` | Internal SD1/SD2, pNNx and sustained-window rule, ESTIMATE with low confidence | Task Force 1996 supports successive-difference HRV components only, not classifier calibration | +| `RunningMoments` | `incremental_core.dart` | Welford updates and algebraic removal with nonnegative M2 clamp; partition merge | Welford 1962; Chan, Golub & LeVeque 1983 | +| `IntHistogram` | `incremental_core.dart` | Exact expanded-list percentileSorted order statistics, Hyndman-Fan type 7 | Hyndman & Fan 1996 | +| `IncrementalLombScargle` | `incremental_core.dart` | Fixed-grid sufficient sums for Lomb-Scargle | Press & Rybicki 1989 | +| `IncrementalHrvTime` | `incremental_hrv.dart` | Incremental HRV with batch jitter fallback | Task Force 1996; internal jitter rule is ESTIMATE with low confidence; Welch 1967; Hirsch & Bishop 1981 | +| `IncrementalEnmoSeries` | `incremental_enmo.dart` | Explicit-reference motion updates; automatic calibration uses batch | van Hees 2013; Vähä-Ypyä 2015; internal dynAmp adaptation | +| `IncrementalMinuteMetrics` | `incremental_minutes.dart` | Keyed batch TRIMP and energy contributions | Banister 1991; Morton 1990; Keytel 2005; Mifflin-St Jeor 1990; Tudor-Locke 2019 | +| `HeartRateZones.timeInZone / timeInZoneAggregator` | `workout/time_in_zone_aggregator.dart` | Internal cadence-validation, median-capped interval and tail-credit rule, ESTIMATE with low confidence; exact gap histograms preserve it | Karvonen 1957 supports the HR-reserve zone option only | +| `hrDipFromDayTotals` | `clinical/nocturnal.dart` | Internal HR-dip band and minimum-sample rule, ESTIMATE with low confidence; sample-order day totals and all-valid-sample night mean | Avram 2019; Dial 2025 support nocturnal RHR only, not HR-dip classification | +| `Calories.minuteEnergyPricer / assembleMinuteEnergySeries` | `workout/calories.dart` | Shared dailyEnergy minute billing and ordered assembly | Keytel et al. 2005; Mifflin-St Jeor 1990; Tudor-Locke et al. 2019 | +| `slidingMedians` | `util.dart` | Same clipped-window median via maintained sorted values; percentileSorted uses type 7, zero-based rank (n - 1) * p / 100 | Hyndman & Fan 1996 | +| `lombScargle` | `util.dart` | First-sample time origin reduces argument error; mathematically time-shift invariant | Press & Rybicki 1989 | +| `Sum / SumOf / SumSq / Moments / Ewma` | `../aggregate/primitives.dart` | Compensated Sum/SumOf and ordinary SumSq; centered moments; affine EWMA composition, algebraically equal to the recurrence: f_y(f_x(c)) = a*a*c + a*lambda*x + lambda*y, a = 1-lambda | Welford 1962 and Chan, Golub & LeVeque 1983 for Moments only; sums are arithmetic, EWMA composition follows algebra | + --- ## What this deliberately doesn't do @@ -159,56 +202,12 @@ al. 2017 · Pimentel et al. · Brage et al. 2004 · Keytel et al. 2005 · Tudor- Mahalanobis 1936 · Killick et al. 2012 · Wittmann & Roenneberg 2006 · Pietilä et al. 2018 · Malik et al. - - -| Batch change | Method and limits | -|---|---| -| Leave long RR artifact runs as gaps | Lipponen & Tarvainen 2019; Peltola 2012; The 32-beat cap is an internal ESTIMATE rule with low confidence, not a published threshold. Longer runs split the series; each side is corrected independently. The synthetic 33-beat case keeps 200 NN beats and drops 33. | - - - -| Batch change | Method and limits | -|---|---| -| Reduce spectral error on epoch timestamps | Lomb 1976; Scargle 1982; Press & Rybicki 1989; Evaluate trigonometric arguments relative to the first sample. The mathematical spectrum is time-shift invariant; floating-point band powers and threshold decisions can change for all Lomb-Scargle callers. | - - - -| Batch change | Method and limits | -|---|---| -| Abstain when a beat has no finite time | Task Force 1996 HRV; Welch 1967 spectra; input-validity guard is an implementation rule; HRV frequency/time, nocturnal and session RMSSD, RSA and CVHR return absent metrics for non-finite beat times. This prevents endless segment walks and invalid integer conversions. Finite-input formulas are unchanged. | - - - -| Batch change | Method and limits | -|---|---| -| Refuse unidentifiable cosinor fits | Nelson et al. 1979, Methods for cosinor-rhythmometry; Welford 1962 centered co-moments; Use centered least squares and a relative determinant gate of 1e-12 of the diagonal product. This is a numerical policy. Nearly collinear phase columns abstain, and ordinary fit parameters can move in their last digits. Upstream zero-amplitude significance gating is retained. | - - - -| Batch change | Method and limits | -|---|---| -| Retain small TRIMP contributions in the total | Morton, Fitz-Clarke & Banister 1990; Neumaier 1974 compensated summation; Compensate the per-minute sum. The reserve and sex-specific weighting are unchanged; the synthetic total moves in its last digits. | - - - -| Batch change | Method and limits | -|---|---| -| Evaluate training-load decay with affine arithmetic | Banister et al. 1975; Morton et al. 1990; Neumaier 1974 prefix summation; Evaluate the same EWMA recurrence as scale times seed plus offset. This matches the arithmetic used by the later incremental implementation. Last digits change; 42/7-day defaults and bounded-prefix priming remain implementation choices. | - - - -| Batch change | Method and limits | -|---|---| -| Count repeated zone intervals before summing | Karvonen 1957 supports reserve zones only; interval credit is an internal ESTIMATE rule; Group equal duration credits by zone. Cadence validation, median capping and tail credit are retained. Grouped multiplication changes the last digits of zone seconds. | - - - -| Batch change | Method and limits | -|---|---| -| Compute circadian variance with centered moments | Witting 1990; van Someren 1999; Welford 1962 centered variance; IS and IV may move in their last digits. Batch circular profile sums and first-winner tie selection are unchanged on the rebased head. Chunked merges can reorder profile sums and move near-tie L5/M10 start epochs; that stream-specific behavior belongs with the aggregators. | - +References added for the methods above: -| Batch change | Method and limits | -|---|---| -| Compute HRV moments without subtracting a large mean | Task Force 1996; Welford 1962 centered sums and products; Use origin-shifted moments for NN levels and five-minute bins, and centered lag co-moments for ACF. Last digits and decisions within rounding of jitter gates can change. Existing respiratory-line exemptions, coverage gates and absent-value rules remain. | +- Welford, B. P. (1962). Note on a method for calculating corrected sums of squares and products. Technometrics 4(3):419-420. +- Chan, T. F., Golub, G. H., LeVeque, R. J. (1983). Algorithms for computing the sample variance: analysis and recommendations. The American Statistician 37(3):242-247. +- Hyndman, R. J., Fan, Y. (1996). Sample quantiles in statistical packages. The American Statistician 50(4):361-365. +- Banister, E. W., Calvert, T. W., Savage, M. V., Bach, T. (1975). A systems model of training for athletic performance. Australian Journal of Sports Medicine 7:57-61. +- Morton, R. H., Fitz-Clarke, J. R., Banister, E. W. (1990). Modeling human performance in running. Journal of Applied Physiology 69(3):1171-1177. +- Berry, R. B., et al. The AASM Manual for the Scoring of Sleep and Associated Events: Rules, Terminology and Technical Specifications. American Academy of Sleep Medicine. Used for duration terminology; application conventions are listed in the hypnogram row. diff --git a/CHANGELOG.md b/CHANGELOG.md index 79caddc1..703d8302 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,60 +1,146 @@ # Changelog -## Split #99 output changes - -Edge must bump `kAlgoVersion` when adopting these changed batch outputs. - -### Leave long RR artifact runs as gaps - -The 32-beat cap is an internal ESTIMATE rule with low confidence, not a published threshold. Longer runs split the series; each side is corrected independently. The synthetic 33-beat case keeps 200 NN beats and drops 33. - -Method: Lipponen & Tarvainen 2019; Peltola 2012. Reproduce with `dart run tool/reproduce_outputs.dart`. - -### Reduce spectral error on epoch timestamps - -Evaluate trigonometric arguments relative to the first sample. The mathematical spectrum is time-shift invariant; floating-point band powers and threshold decisions can change for all Lomb-Scargle callers. - -Method: Lomb 1976; Scargle 1982; Press & Rybicki 1989. Reproduce with `dart run tool/reproduce_outputs.dart`. - -### Abstain when a beat has no finite time - -HRV frequency/time, nocturnal and session RMSSD, RSA and CVHR return absent metrics for non-finite beat times. This prevents endless segment walks and invalid integer conversions. Finite-input formulas are unchanged. - -Method: Task Force 1996 HRV; Welch 1967 spectra; input-validity guard is an implementation rule. Reproduce with `dart run tool/reproduce_outputs.dart`. - -### Refuse unidentifiable cosinor fits - -Use centered least squares and a relative determinant gate of 1e-12 of the diagonal product. This is a numerical policy. Nearly collinear phase columns abstain, and ordinary fit parameters can move in their last digits. Upstream zero-amplitude significance gating is retained. - -Method: Nelson et al. 1979, Methods for cosinor-rhythmometry; Welford 1962 centered co-moments. Reproduce with `dart run tool/reproduce_outputs.dart`. - -### Retain small TRIMP contributions in the total - -Compensate the per-minute sum. The reserve and sex-specific weighting are unchanged; the synthetic total moves in its last digits. - -Method: Morton, Fitz-Clarke & Banister 1990; Neumaier 1974 compensated summation. Reproduce with `dart run tool/reproduce_outputs.dart`. - -### Evaluate training-load decay with affine arithmetic - -Evaluate the same EWMA recurrence as scale times seed plus offset. This matches the arithmetic used by the later incremental implementation. Last digits change; 42/7-day defaults and bounded-prefix priming remain implementation choices. - -Method: Banister et al. 1975; Morton et al. 1990; Neumaier 1974 prefix summation. Reproduce with `dart run tool/reproduce_outputs.dart`. - -### Count repeated zone intervals before summing - -Group equal duration credits by zone. Cadence validation, median capping and tail credit are retained. Grouped multiplication changes the last digits of zone seconds. - -Method: Karvonen 1957 supports reserve zones only; interval credit is an internal ESTIMATE rule. Reproduce with `dart run tool/reproduce_outputs.dart`. - -### Compute circadian variance with centered moments - -IS and IV may move in their last digits. Batch circular profile sums and first-winner tie selection are unchanged on the rebased head. Chunked merges can reorder profile sums and move near-tie L5/M10 start epochs; that stream-specific behavior belongs with the aggregators. - -Method: Witting 1990; van Someren 1999; Welford 1962 centered variance. Reproduce with `dart run tool/reproduce_outputs.dart`. - -### Compute HRV moments without subtracting a large mean - -Use origin-shifted moments for NN levels and five-minute bins, and centered lag co-moments for ACF. Last digits and decisions within rounding of jitter gates can change. Existing respiratory-line exemptions, coverage gates and absent-value rules remain. - -Method: Task Force 1996; Welford 1962 centered sums and products. Reproduce with `dart run tool/reproduce_outputs.dart`. - +## Unreleased + +### Added +- `stageMinuteRanges`: a [lo, hi] range of minutes per sleep stage (wake, + light, deep, REM, and NREM = light + deep). `lo` counts epochs that can only + be that stage: HR-covered, labelled so, and every deciding score at least + `kStageScoreMargin` (0.1) from its cutoff. `hi` adds epochs that could still + be that stage: uncovered or unlabelled epochs (any stage), and epochs with a + score inside the margin (REM score: REM and its NREM substage; deep score: + light and deep). An honest bound, not a confidence interval: the stager has + no per-epoch probabilities. Null for no epochs. +- `CardioStagerResult.epochEvidence` (with `StageEpochEvidence`): per final + epoch, the label, deep flag, HR coverage and the REM/deep scores the classify + loop computed. `CardioStagerResult` carries `remScores`, `deepScores` and + `hrCovered` for it. Stage labels are unchanged. +- `irregularBeatScreenDetailed` and `IrregularScreenState.evaluateDetailed`: the + irregular-rhythm screen's verdict plus its evidence (`IrregularDiagnostics`) — + beats in / kept after the [300, 2000] ms filter, the corrector's raw / + corrected / dropped counts (passed in as `RrCleaningCounts`), artifact + fraction, window counts (total, voting, flagged, the final OPEN window and + what became of it, sustained share seen vs required) and the thresholds — for + a screen that ran and for one that abstained (with the reason). Detailed + diagnostics preserve the wrapper's result for the same screen calculation. + `IrregularScreenState` checkpoints are version 3, carrying diagnostic counters + and the previous kept beat's time. Versions 1 and 2, and incomplete version 3 + states, are refused. A NaN beat no longer makes the checkpoint unencodable. + `IrregularDiagnostics.artifactFraction` is null (absent) when the corrector was + handed no beats: a share of nothing has no value, and the `1 - cleanFraction` + of an empty series (1.0) must not read as "every beat is an artifact". + Output change: timestamp-gap pairs no longer contribute successive differences, + so Poincaré statistics, pNNx, window votes and screen verdicts can change. +- Incremental states for centered moments, fixed-grid Lomb–Scargle, time-domain + HRV, motion with a measured gravity reference, and keyed minute TRIMP/energy. + Appends reuse retained contributions; edits, removals and changed dependencies + update or rebuild them. Batch APIs remain available. `phillipsSri` uses + `sriAggregator` for normal inputs, retaining the legacy short-validity-mask + path; `AdvancedSleepStager.computeMetrics` uses `HypnogramAggregator`. + The batch functions listed under Changed also run their aggregators. Other + batch implementations remain separate, sometimes sharing kernels/scoring + helpers with incremental forms. +- Bounded dependency-aware `CalculationCache` and explicit `CalculationMode`. + Only `periodicAwake` permits reuse. Sleep, heavy and forced runs use full math. +- Minute summary mode avoids constructing per-minute energy records when only + totals are needed. See `docs/INCREMENTAL_USAGE.md` for contracts and limitations. +- Parameterized incremental parity tests, checked-in capture fixtures, checkpoint + tests, work counters and a deterministic timing comparison tool. +- `hrDipFromDayTotals`: `hrDip` from the day side's running count and sum, for + callers that keep the waking day as totals. `hrDip` shares its scoring, so + both give the same result for the same samples. +- `IncrementalMinuteMetrics.sync` takes `dayMinutes` as an `int`, as + `Calories.dailyEnergy` does; a fractional day can no longer be passed. +- Long-run drift tests: 30 000 Lomb–Scargle appends and 100 000-step sliding + moments stay within 1e-9 of the batch results. +- `IntHistogram`: exact, mergeable order statistics for integer-valued series + (heart rate), kept as a sorted `value -> count` list. `percentile` and `median` + return the same doubles as `percentileSorted` on the expanded sorted list, + and null when empty. Not wired into any caller yet; see + `docs/INCREMENTAL_USAGE.md` for which batch callers could adopt it. +- Pinned Nix devshell with Flutter 3.41.6 / Dart 3.11.4 and native build tooling. +- `Calories.minuteEnergy` — per-minute energy (basal, active, walking, total, + and an abstention reason) from the same computation `Calories.dailyEnergy` + does. A minute with no usable HR and no billable cadence abstains: all values + null, `abstained: MinuteEnergyAbstain.noHr`. Nothing is filled or interpolated. + `MinuteEnergySeries.active` / `.walking` are `==` `dailyEnergy`'s, and + `basalKcalPerMin * dayMinutes` is its `basal`. +- `Calories.hourlyRollup` — hourly sums of covered minutes with per-hour + coverage; an hour with no covered minute is null, not zero. + +### Changed +- **Output change (combined bump; edge must bump kAlgoVersion when pinning + this): capped RR artifact runs become gaps; downstream metrics may change.** + `correctRr(maxArtifactRun:)` and `RrCorrector(maxArtifactRun:)` (default + `kMaxArtifactRun` = 32; null = uncapped) split over-cap runs and re-correct + each side. Without a cap split the result equals uncapped correction. + Staging retains upstream main's session-onset 30 s grid, shared night + baselines across usable runs, per-run HR coverage gate and gap-bounded + continuity, except for separately documented RR or spectral changes. + `AdvancedSleepStager` keeps that batch path, with dependency-checked row + caching; it does not stage through `EpochRows`. + RSA streams and research accumulators retain upstream's lower-band floor + guard and include floor-pinned votes in consensus. `RsaRespStream` + checkpoints are version 3; versions 1 and 2 are refused because settled + peaks in those versions had no floor guard. After an accelerometer gap + exceeding the 60 s carry-forward limit, the next usable run re-anchors at + its first sample, as before. `EpochRows` accumulates these feature rows; + `EpochRowsMemo` caches rows by grid phase and recomputes an unavailable phase. + `RrSandwich` / `RawRr` / `NnColumn` feed capped corrected NN into aggregators, + retaining source-beat absolute timestamps via `nnSourceIndex`. + Registry counters `found.rr_correction` and `sleep.stage` remain at 2. +- **Output change: aggregator adoption (7483030).** `hrvTime`, `banisterTrimp`, + `ctlAtlTsb`, `cosinor`, `HeartRateZones.timeInZone` and + `circadianNonparametric` now return their aggregators' one-slice run. + Ordinary finite-input outputs move only in the last digits within tested + tolerances: HRV fields 1e-9 absolute plus 1e-9 relative; TRIMP 1e-12 relative; + CTL/ATL/TSB 1e-9 absolute and relative; zone seconds 1e-9 absolute / 1e-12 + relative; circadian indices 1e-12 relative on integer fixtures and 1e-9 on + real fixtures. Circadian near-ties can select different L5/M10 start epochs. + Cosinor mesor/amplitude use 1e-9 relative, with a near-zero amplitude floor + of 1e-12 * max(1, |mesor|); near-zero mesor has a 1e-12 data-scale floor. + Circular phase error uses 1e-9 * max(1, |phaseRad|) in radians and + 1e-9 * periodHours in hours; phase is not compared for rounding-noise + amplitudes. The aggregator also preserves upstream's zero-amplitude F-test, + null acrophase when p ≥ 0.05, and confidence from adjusted R². + R², adjusted R² and confidence use 1e-7 absolute. These are checked + parity tolerances, not universal bounds. + SDANN/SDNN-index pins moved to 1e-9 relative. +- **Output change: near-singular cosinor designs.** The relative determinant + test replaces the old batch solver's absolute 1e-12 pivot test. Insufficient + phase coverage returns a null-valued `Metric` with the singular-design note; + the two tests can disagree about whether a fit is identifiable. +- Registry counters `found.core`, `hrv.freq`, `night.rhythm` and `activity.load` + go to 2 for the spectral origin, HRV, rhythm and load changes. Dependent + groups re-derive through `deps`; see `docs/COMBINED_BUMP_STATUS.md`. +- `IrregularScreenState` checkpoints are version 3: they carry the previous kept + beat's time, because a pair across a time hole is no longer successive. + Versions 1 and 2, and a version 3 checkpoint without `prevT`, are refused. +- `IncrementalHrvTime` follows `hrvTime`'s jitter gate under the ACF floor + (respiratory-line exemption, breathing-line fallback): when either could + apply it hands the night to the batch, so the incremental output equals + `hrvTime` where it used to refuse RMSSD. Checkpoint format unchanged. +- `hrvTimeAggregator` follows `hrvTime`'s jitter gate under the ACF floor + (respiratory-line exemption, breathing-line fallback) from mergeable + evidence (O(5-min windows), never the beats). Its checkpoint state is now + `[core, evidence]`; a checkpoint without the evidence is refused whole. +- Output change for invalid timestamps: `hrvTime`, `nocturnalRmssd`, + `sleepSessionRmssdDetail`, `rsaRespRate` and `cvhrApneaScreen` return absent + metrics for non-finite beat times. Streaming folds refuse invalid times + before mutation; snapshots with invalid provisional times abstain. + Finite-input formulas are unchanged by these guards. +- **Output change: all `lombScargle` callers can move; edge must bump + kAlgoVersion when pinning this.** Time is now measured from the first sample. + The old error reached approximately 5e-6 relative per frequency; integrated + metric changes depend on the input and were approximately 1e-14 on the + checked night. Threshold decisions can therefore change. Callers include + `hrvFreq`, `cardiacCoherence`, respiration RSA/peak rate and the cardio + stager's LF/HF. `IncrementalLombScargle` already used first-sample time and + is unchanged; it now agrees with batch on epoch seconds within 1e-9 in the + precision test, which saw 9e-6 before the change. + +### Unchanged +- `Calories.dailyEnergy` outputs are bit-for-bit identical. Its loop body moved + into a helper shared with `minuteEnergy`; golden tests in + `test/onehz/minute_energy_test.dart` pin the pre-change values with `==`. + No `kAlgoVersion` bump is needed for this change. diff --git a/README.md b/README.md index bb564c6b..4d1fcc77 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,10 @@ if (hrv.value != null) { Every metric function in the package follows this shape: plain `List` (or a small typed input class for the composite ones) in, `Metric` out. +For repeated updates, [incremental calculation states](docs/INCREMENTAL_USAGE.md) +retain contributions while checking the complete current inputs for revisions. +The [math audit](docs/INCREMENTAL_MATH.md) explains the algebra and rebuild rules. + ## What's actually in here Eight families, each its own subdirectory with its own sub-barrel, built on two shared @@ -100,9 +104,9 @@ foundation layers: only strain scale), a Poincaré irregular-rhythm screen, cosinor circadian fitting, and real-time cardiac coherence for guided breathing sessions. - **`sleep/`** — van Hees z-angle segmentation feeding a cardiac/motion stager (the - single source of truth for sleep staging), AASM hypnogram metrics, - fractal sleep-cycle detection, circadian non-parametric indices - (IS/IV/RA/L5/M10). + single source of truth for sleep staging), a causal (online, abstaining) wake/NREM/REM + stager for live decisions, AASM hypnogram metrics, fractal + sleep-cycle detection, circadian non-parametric indices (IS/IV/RA/L5/M10). - **`respiration/`** — RSA-derived respiratory rate fused with motion-modulated RIIV, CVHR-based apnea screening, a relative (never absolute) oxygen-desaturation ratio. - **`motion/`** — ENMO/MAD activity metrics, a 100 Hz live pedometer that finds gait @@ -139,8 +143,9 @@ assertion out. ## Validation -`tool/` has four harnesses that score shipped detectors against labelled corpora, not -synthetic fixtures — run one before touching the logic it covers: +`tool/` has five harnesses that score shipped detectors against labelled corpora (or, for +the causal stager, the one recorded night in-tree) rather than synthetic fixtures — run one +before touching the logic it covers: - `dart run tool/oxwalk_validate.dart ` — the pedometer against OxWalk (Oxford, CC BY): 39 subjects, camera-annotated heel strikes. @@ -148,6 +153,10 @@ synthetic fixtures — run one before touching the logic it covers: against a PSG-labelled corpus (e.g. DREAMT), reporting Cohen's kappa. - `dart run tool/nap_harness.dart ` — the nap detector against hand-labelled days. +- `dart run tool/causal_stager_validate.dart` — replays the in-tree real night through the + causal (online) stager and scores agreement, false- and late-trigger behaviour for the + "wake during REM" use case against the offline stager. There is no PSG-labelled night + in-tree, so this is agreement, not accuracy; see `docs/CAUSAL_STAGER.md`. - `dart run tool/whoop_proportions.dart ` — sweeps sleep-stage cutoffs against normative stage proportions on real device captures. diff --git a/docs/COMBINED_BUMP_STATUS.md b/docs/COMBINED_BUMP_STATUS.md new file mode 100644 index 00000000..b17cdf65 --- /dev/null +++ b/docs/COMBINED_BUMP_STATUS.md @@ -0,0 +1,235 @@ +The RR cap, the epoch accumulator, the RR sandwich and the onset-anchored grid +(option 2, see "Grid") are implemented. + +Implemented APIs: + +- `correctRr(..., maxArtifactRun: K)` defaults to 32. `null` retains the + uncapped implementation. A hard artifact run over K is cut before its + neighbours are classified. Longer detected soft runs also split the input. + Each side has separate correction windows; dropped gap beats keep an input + class. Runs under the cap use the original batch implementation verbatim. +- `RrCorrector(maxArtifactRun:)` persists K and uses a bounded raw window for + the capped path. The uncapped path retains the original incremental kernel. + Settled output plus the snapshot equals the capped batch in the cap tests, + including one-beat folds, JSON restarts, and 10,000 junk beats. +- `NnColumn(inner)` delegates fold, combine, finish, and persistence to a + `DoubleColumn` aggregator over `Beats.nnMs`. +- `EpochRows` implements the anchored grid (below), fold, + final/provisional/all rows, `newestMs`, JSON restore (checkpoint version 4), + and `stageWindow`. It keeps final features and a + raw time tail. The provisional watermark is four minutes behind the newest + sample. Gravity references use a sliding sorted time window. Partial HR + coverage is retained so sparse cardiac evidence cannot appear complete + merely because each epoch has a mean. Carry-only epochs also remain in the + classifier's coverage denominator. +- The default advanced stager delegates to these rows. Feature + context extends beyond the requested session. The existing bounded accel + coverage and unobserved-run semantics remain in use. +- Registry counters `found.core`, `found.rr_correction`, `sleep.stage`, + `hrv.freq`, `night.rhythm` and `activity.load` are 2. A group's own algorithm + changes, including rounding and abstention, require a bump; dependency-only + changes propagate through `deps`. One bump per group covers this PR: + - `found.core`: Lomb-Scargle now subtracts the first timestamp. + - `found.rr_correction`: the artifact-run cap, already bumped. + - `sleep.stage`: retain the existing bump; its direct staging path also + computes adopted HRV statistics. Session-onset/post-gap anchoring is retained. + - `hrv.freq`: adopted HRV rounding and non-finite-time abstention. + - `night.rhythm`: adopted cosinor/circadian statistics, the cosinor singularity + gate, timestamp-gap irregular-screen pairs and invalid-time refusals. + - `activity.load`: adopted TRIMP, CTL/ATL and time-in-zone rounding. + Other counters remain 1. `found.energy` preserves ordered minute billing; + `found.staging_helpers`, `found.baseline_ewma` and `found.calibration` have no + direct output changes. `day.blocks`, `readiness.stress_illness`, + `body.spo2_temp_coverage` and `human.insights` have no direct algorithm change; + changed upstream inputs propagate through dependencies. In particular, + importing the cosinor file in a source closure does not mean an entry point + calls it: `nightlySkinTemp` does not call `tempCircadian`/`cosinor`. + Hypnogram integer accounting, SRI counts, readiness suffix scoring and + sample-order HR-dip totals preserve their existing arithmetic. + +The corrector checkpoint is version 2. Version 1 is refused, including a +pre-cap checkpoint whose already-settled output could conflict with a later +cap decision. K, including null, is explicit. The capped path's conservative +settle delay is **4H + 2K + 4 = 248 beats**, plus up to a short artifact run. +This exceeds the brief's approximately 124-beat target. It needs further work +before the latency requirement can be called satisfied. + +The LF/HF grid is unchanged. Complete, identical per-row beat windows do not +need a fixed spectral grid to give exact chunk parity. Probing fixed lower +frequencies of 1/180 Hz and 0.04 Hz did not change this night's stage agreement. + +Sandwich (owner decision 2026-10-22: absolute timestamps). `RrSandwich` +hands its inner aggregator `Beats(nnMs, timeMs)` where each NN's time is the +ABSOLUTE `rrTsMs` of the input beat it came from (the beat itself, or for a +spline-corrected beat its own timestamp), not `correctRr`'s first-beat-relative +clock. `RrCorrectionResult.nnSourceIndex` (new, optional, default empty on a +hand-built result) says which input index each NN came from; no existing field +changed. With absolute times a seam needs no clock translation, which removes +the counterexample that blocked the first attempt (an inner aggregator keyed on +one fixed time). + +State: a chunk of m beats keeps the first and last W = 2 * reach raw beats +(reach = 4H + 3K + 4 = 280 at the defaults; H = windowBeats ~/ 2, K = the cap) +and the inner state over the NN of beats [reach, m - reach), which no beat +outside the chunk can change. `combine` re-corrects only `left.tail ++ +right.head` (via `correctRr`, `nnSourceIndex` selecting the seam beats) and +feeds the newly settled NN, so state is head + tail + whatever the inner keeps. +`finish` corrects the head and tail windows (clipped exactly as the whole +series is) and combines them around a copy of the settled middle. +`addSlice` is `combine(state, fold(slice))`. A one-beat append therefore costs +a ~1,100-beat correction; use larger chunks when streaming. + +The reach is the streaming corrector's proven settle horizon (4H + 2K + 4) plus +one K for a run straddling an edge. It is not a proof for every adversarial +input: an artifact pattern alternating normal and compensatory-pair beats could +in principle carry the previous-class dependency further than one run. Tests do +not detect a reach down to about 2H (they fail at reach 6), so the margin is +conservative and unmeasured. With `maxArtifactRun: null` the reach is sized as +for the default cap and a run longer than that is not bounded-state exact. + +Grid (owner decision 2026-10-23, option 2: onset-anchored; replaces the +gap-anchored grid of 2026-10-22). The stager puts its 30 s epochs at the +detected session onset (phase 0), as before the bump. The onset is the first +valid accelerometer sample at or after the session start. A run that follows a +dropout longer than the carry (more than 61 s between valid samples) re-anchors +at its own first sample, as the pre-bump stager did. The stager reads nothing +outside [start, end): it clips samples to the session, so its labels equal the +frozen pre-bump stager (`test/onehz/support/pre_bump_stager.dart`). + +- `EpochRows({anchorMs})`: the first run's grid passes through `anchorMs`. Rows + start at the anchor, or at the epoch holding the first sample when the anchor + lies before it. `EpochRows.phaseMs` is the anchor mod 30 s. Without + `anchorMs` the first sample anchors, as before. +- `EpochRowsMemo({maxPhases = 2})`: rows keyed by phase. `rowsFor(anchorMs, ...)` + takes cumulative input. A held phase folds only samples it has not consumed. + An unheld phase, or an anchor earlier than the held rows were built for, is + built from scratch (exact). Least recently used phases beyond `maxPhases` are + dropped. `toJson` / `fromJson` carry each phase's rows and consumed counts. +- Carry-forward matches the pre-bump dense array: for features, `EpochRows` + repeats the last valid vector each second for up to 60 s (until the next real + sample). Epochs that start after a run's last real sample are classified but + hidden from `allRows` / `finalRows`. `fold(throughMs:)` states that data is + complete through a time; the stager passes the session end. +- `stageWindow` classifies whole epochs only; a sub-epoch remainder joins the + last one, as before the bump. +- Checkpoint version 4 (versions 1 to 3 are refused; 81b9b4b changed row + semantics, so version 3 rows are wrong). Version 3 added `phase` and the + pending `anchor`. A phase its rows contradict is refused. Accel is kept 61 s + longer so carry from a dropped sample still reaches a live window. +- The memo does not reuse interior rows of a larger window for a smaller + session. The stager builds rows from the clipped session, which is exact and + costs what the pre-bump stager cost. A caller that wants reuse must hand the + memo input already clipped to the session. + +Measured: frozen-night agreement is 1.0 (window starts at the first sample). +The advanced-stager still-night efficiency guard (> 0.85) passes again (phase 0 +gave 0.8588 in the earlier control). + +Tests changed: + +- `test/onehz/incremental/rr_corrector_test.dart`: expect checkpoint v2 and + reject v1; extend the capped settle-horizon bounds to the implemented delay. +- `test/properties/rr_irregular_laws_test.dart`: run the existing tests of the + uncapped pipeline layout, stored first difference, and exact 2H class delay + with `maxArtifactRun: null`. Update invalid checkpoint versions for v2. + New capped behavior remains covered by `rr_artifact_cap_test.dart`. +- `test/onehz/cardio_row_cache_test.dart`: assert the actual union of cache + keys. Wall-clock rows shared by adjoining windows now deduplicate at a seam. +- `test/onehz/sleep_honesty_test.dart`: back to the original expectations (600 + s wake, 15,000 s longest sleep run); the wall-clock grid had moved them and + the gap-anchored grid does not. +- `test/onehz/epoch_rows_test.dart`: B1 rewritten for the gap-anchored grid + (anchor at the first sample, finality of the anchor, re-anchor after a gap + of 61 s but not 60 s, gap-only clipping with an unaligned gap, the seeded + `continuingAnchorMs`); B3 boundaries are checked against the run's grid, plus + a session that starts mid-run and a hole that re-anchors. B2 is unchanged + except `whole.length >= 100` (3,000 s of data is exactly 100 epochs from the + first sample; the wall-clock grid had 101). +- `test/onehz/wallclock_agreement_test.dart`: the agreement loop steps 30 s + from the first sample (the frozen hypnogram's own grid) instead of from the + next wall-clock boundary; the boundary test checks the run grid, with run + anchors recomputed independently from the valid accel timestamps. The 0.97 + guard is unchanged. +- `test/aggregate/rr_sandwich_test.dart`: the oracle is `correctRr(..., + maxArtifactRun: K)`'s NN paired with the absolute timestamps of its + `nnSourceIndex` beats (was `nnTimesMs`); the A3 HRV/PRSA references and the + HRV origin use the same absolute times. Added: series longer than the + windows (settled middle in play; random cuts, many even chunk sizes around + W, one-beat chunks, a dense-artifact property), a bounded-size check across + series length, a longer law case, and long A3 cases. The insertion of a + definite dropout into the seam fixture stays. +- `test/onehz/rr_artifact_cap_test.dart`: `nnSourceIndex` tests (length, + order, value, dropped count, over-cap runs, tiny inputs, spline-corrected beat + is its own source). +- `test/onehz/rr_artifact_cap_test.dart`: add a regression that folds and + restores through a settled gap into a longer clean tail. It caught a lost + segment boundary after compaction; the state now persists that boundary and + whether the active gap is physiological or detected by the soft classifier. + +The existing advanced-stager efficiency assertion and the 0.97 agreement +assertion have not been weakened. With option 2 the former passes again. + +The capped latency is above target, the still-night efficiency guard fails +(see Grid), and the sandwich reach and the adversarial soft-run influence +bounds still need a proof beyond the passing prefix/oracle cases. These are +open limitations, not completed acceptance criteria. + +Benchmarks below used the supplied Dart 3.11.4 executable in JIT mode. They +are local measurements, not device timings. The new probe is reproducible via +`dart run tool/combined_bump_probe.dart [bench|rr-bench]`. + +| Workload | Accumulated/current | Batch/reference | +| --- | ---: | ---: | +| Real night, EpochRows fold + allRows versus cardioEpochFeatures | 1,937.511 ms | 1,954.044 ms | +| Same night, all 15-minute prefixes, total feature work | 2,446.084 ms | 38,535.495 ms | +| Real night, warmed capped versus uncapped correctRr | 37.871 ms | 37.873 ms | +| bench_phase0 cardioStager, current versus frozen reference | 1,735.504 ms | 1,762.683 ms | +| bench_night_nn seed 1, streaming RR versus repeated batch, 32 passes | 61.78 ms | 547.87 ms | +| Same tool, one whole-day corrector fold + snapshot versus batch | 210.42 ms | 174.75 ms | + +The one-shot corrector is about 20% slower than batch on that cold whole-day +measurement, so the blanket no-performance-regression requirement is not +satisfied, although repeated night passes are faster. + +The phase-0 benchmark reports exact frozen-reference parity. The night-NN +benchmark reports bit identity for all 32 passes. The real-night EpochRows +checkpoint is 188,121 JSON characters. The 96,712-beat corrector test measured +an 8,114-character maximum checkpoint. The 10,000-junk-beat bound passes. +No RrSandwich benchmark was taken. The benchmark rows above were measured +before the grid change (EpochRows has the same per-fold work, plus a run +lookup). + +Final full-suite summary: + +```text ++2596 ~7 -1: Some tests failed. +``` + +That is 2,596 passed, 7 skipped, and 1 failed: the epoch_rows_test B3 test +"staged on ITS OWN start", whose `checked > 2` count cannot hold for window ++777..+2111 s (the pre-bump labels there have only two boundaries). The +sandwich tests, the frozen-night agreement guard (1.0 against 0.97), the RR cap, +checkpoint regression, batch oracles, epoch-row parity, cache behavior, +registry closure, and counter tests pass. `dart analyze` reports no issues. +No commits, checkouts, or stashes were run. + +Files changed: + +- `lib/src/onehz/foundations/rr_correction.dart` +- `lib/src/onehz/foundations/rr_correction_kernel.dart` +- `lib/src/onehz/foundations/rr_correction_stream.dart` +- `lib/src/onehz/foundations/rr_sandwich.dart` +- `lib/src/onehz/sleep/advanced_stager.dart` +- `lib/src/onehz/sleep/epoch_rows.dart` +- `lib/src/onehz/registry/group_registry.dart` +- `CHANGELOG.md` +- `test/aggregate/rr_sandwich_test.dart` +- `test/onehz/cardio_row_cache_test.dart` +- `test/onehz/epoch_rows_test.dart` +- `test/onehz/incremental/rr_corrector_test.dart` +- `test/onehz/rr_artifact_cap_test.dart` +- `test/onehz/sleep_honesty_test.dart` (back to its original content) +- `test/onehz/wallclock_agreement_test.dart` +- `test/properties/rr_irregular_laws_test.dart` +- `tool/combined_bump_probe.dart` +- `docs/COMBINED_BUMP_STATUS.md` diff --git a/docs/INCREMENTAL_INTEGRATION_PLAN.md b/docs/INCREMENTAL_INTEGRATION_PLAN.md new file mode 100644 index 00000000..434bbdac --- /dev/null +++ b/docs/INCREMENTAL_INTEGRATION_PLAN.md @@ -0,0 +1,168 @@ +# Incremental calculations: integration plan + +Hand-off for getting the incremental work into the app (`edge`). Read +[INCREMENTAL_MATH.md](INCREMENTAL_MATH.md) for the derivations and +[INCREMENTAL_USAGE.md](INCREMENTAL_USAGE.md) for the API. + +## Where the work is + +| Repo | Branch | Location | State | +|---|---|---|---| +| analytics | `feat/minute-energy` | this checkout | `4bd76f7` committed; later changes below uncommitted | +| edge | `feat/incremental-analytics` | `../edge.worktrees/incremental-analytics` (git worktree of `../edge`) | all incremental work uncommitted on top of `687b5bd0` | + +`687b5bd0` is a snapshot of the scheduler work that was uncommitted in +`../edge` when this started. That work has since been committed there as +`561f2838` on `ecg-taps-one-clock` (about 100 lines differ), and that branch +has moved on to `f88d230c`. `../edge` itself also has uncommitted perf tests +(`test/perf/p3_*`). Leave that checkout alone. + +The edge worktree builds against this checkout through a gitignored +`pubspec_overrides.yaml`. `pubspec.lock` is modified only by that override: +never commit it. + +Run everything through the devshell in this repo: + +```sh +nix develop /path/to/openstrap-analytics -c flutter test # in the edge worktree +nix develop -c dart test # here +``` + +Run edge tests with `TZ=UTC`. The two-device golden fails under any other +offset. + +## What is in place + +Every incremental result is tested against the existing batch function as +its oracle. In this repo, exact paths must agree within a relative error of +1e-8 (`test/onehz/support/incremental_compare.dart`). In edge, the helper is +`test/support/incremental_compare.dart`: `kExactRelTol = 1e-9` for exact +reuse, and `kApproxRelTol = 0.02` reserved for a future method that is +approximate by design. No such method exists yet. + +**Used by the app** + +- `CalculationCache`, through `DayCalculationState.evaluate`. It holds the + finished night's results (about 25 entries) and the completed windows of + the rolling HRV and breathing-rate curves. These all hit on every periodic + awake pass. +- `IncrementalMinuteMetrics`: TRIMP, strain, and daily calories, with each + minute priced once. +- `IncrementalEnmoSeries`: motion minutes. Full passes calibrate the gravity + reference exactly as `enmoSeries` does. Awake passes keep that reference. + Every field the app reads is still the batch value; only + `MotionMinute.enmo` uses the kept reference, and nothing in edge reads it. +- Edge-side running summaries (`lib/compute/day_activity_state.dart`): wake + active minutes, the activity curve, wear runs, per-minute wake HR, the + day's HR stats (smoothed max/min, mean), and the day side of HR dip + (`hrDipFromDayTotals` here). Each one adds in sample order, so it is + bit-identical to the batch reader it replaces. +- Mode selection: `PeriodicCalculationPolicy` allows reuse only while the + phone is unplugged and the causal stager reports fresh, confident `wake`. + Sleep, heavy, forced, charging and missing or stale evidence all run in + full. + +Benchmark (`INCREMENTAL_BENCH=1 flutter test test/incremental_cache_benchmark_test.dart`, +16 h awake day plus an 8 h night, six 5-minute appends): 86.2 s recomputing +everything against 2.2 s with reuse. A full pass is also no slower than +before the work: about 350 ms compute per derive in +`sleep_override_blanks_night_test`, against 445 ms at baseline. + +**Implemented here, not used by the app** + +| Incremental | Why it is unused | Where it would pay | +|---|---|---| +| `IncrementalLombScargle` | Its callers (`hrvFreq`, `cardiacCoherence`) run on the finished night or on short sessions; the night is cached whole. | A live coherence/breathing session that recomputes its spectrum as beats arrive. | +| `IntHistogram` | No existing incremental state recomputes an HR median per pass. | A rolling or trailing HR median/percentile over a long window (see the caller table in INCREMENTAL_USAGE.md). | +| `RunningMoments` | No caller recomputes a long mean/SD per pass. Night SDNN is cached whole. | Rolling windows over the awake day (daytime HRV, rolling HR variability) or baselines kept as running state. | +| `IncrementalHrvTime` | Removed from the app: the night's beats do not change while awake, so the cached result already covers it. | Daytime HRV (`_daytimeHrv`) if it becomes expensive. | + +## Steps + +1. **Land this repo's changes.** Uncommitted: + - `hrDipFromDayTotals`, which `hrDip` now shares its scoring with + - `IncrementalMinuteMetrics.sync` taking `int dayMinutes` (as + `Calories.dailyEnergy` does) + - day-duration tests + - `drift_test.dart` + - `IntHistogram` + - the first-sample time shift in `lombScargle` and its precision test + - this plan + + Commit them, open a PR, and note the new commit SHA. The incremental work + changes no algorithm output, but the `lombScargle` shift does (about 1e-6 + relative, worst case), so edge must bump `kAlgoVersion` when it pins a commit + that contains it. + +2. **Move the edge work onto the current edge branch.** Commit the worktree's + changes on `feat/incremental-analytics`, then rebase onto + `ecg-taps-one-clock`, dropping `687b5bd0` (its content is already in + `561f2838`). Expect conflicts in: + - `lib/compute/derivation_engine.dart`: `run()` gained + `calculationMode`, `_derivePreparedDay` now publishes state after + persistence, and the activity helpers were changed. + - `lib/state/app_state.dart`: `_deriveRun` asks the policy before each + light pass. `f88d230c` changed about 230 lines here. + - `lib/compute/onehz_pipeline.dart`: memoized night components and the HR + summary. + + Keep both sides' behaviour. Keep `changedOnly` from forcing a full mode: + the automatic light pass uses `changedOnly`, so mapping it to `forced` + silently disables all reuse. `test/incremental_engine_run_test.dart` + covers this path end to end. + +3. **Pin the analytics dependency.** Replace the local override with the new + analytics SHA in `pubspec.yaml`, run `flutter pub get`, and update the SHA + that `db_serve_version_and_reads_test.dart` checks. That test already + fails on this branch because the pins and `kAlgoVersion` disagree. + +4. **Verify.** In edge, the incremental tests are + `test/incremental_*_test.dart`, 8 files including the opt-in benchmark. + Then run the full suite with `TZ=UTC`. Known failures that also fail + without this work: + - `db_serve_version_and_reads_test` (pins) + - three Today-tab tests in `health/health_h2_tabs_test.dart` + - `setUpAll` of the four proof-view tests + (`proof/affected_views`, `proof/chart_key_views`, `proof/phase8_views`, + `sources/proof_views`) + - the two-device golden comparison + + `sleep_blank_review_test` and `sleep_override_blanks_night_test` can hit + the 30 s timeout under full-suite load. They pass alone. + +5. **Device check.** On a phone, during the day and unplugged, the derive log + should show light passes with low compute times. Plugged in, or overnight, + should show full passes. `DerivationEngine.debugCalculationState(day)` + exposes hits and work counters. + +## Follow-ups + +- **Medians from compacted histograms (done).** `IntHistogram` is an exact + `value -> count` summary with `percentile` and `median` bit-identical to + `percentileSorted` on the expanded list, tested against the real-night HR and + randomized data. HR has about 45 distinct values in a real night + (`test/onehz/fixtures/real_night_2026_07_onehz.csv`). No existing incremental + state recomputed an HR median per pass, so nothing was switched; candidate + callers are listed in INCREMENTAL_USAGE.md. The largest medians in a day are + the two inside `calibrateGRef`, over every valid accelerometer magnitude + (up to 86,400 values, 30,848 distinct of 32,041 in the same capture), so a + histogram does not help there. Those are off the awake path because the + reference is kept. If they ever need to be incremental, use an + order-statistic structure, not a histogram. +- **Observed max HR (pending, edge-side).** The all-time ceiling is already a + cached comparison: `LocalDb.observedHrCeiling` takes the max over one stored + value per day. The per-day value comes from `sessionHrCeiling` over each + session's samples. A finished session's ceiling never changes, so cache it + per session and sample revision. Only an open session needs its hold-window + state carried forward. +- **Batch Lomb–Scargle precision (done here; edge must bump `kAlgoVersion`).** + `lombScargle` now shifts times by the first sample. On raw epoch seconds the + trig arguments reached about 1e10 radians and the periodogram was off by up + to 4.6e-6 relative against an 80-bit reference. This changes `hrvFreq`, + `cardiacCoherence`, the respiration RSA rate and the cardio stager's LF/HF at + the 1e-6 level worst case, so edge must bump `kAlgoVersion` when it pins this + commit (and the pin must contain it; see AGENTS.md invariants 4 and 5). +- Remaining batch calculations and their constraints are listed in + INCREMENTAL_MATH.md. Several (sleep staging, naps, cycles, rank-based + statistics) cannot be made exactly incremental, because new data revises + old decisions. diff --git a/docs/INCREMENTAL_MATH.md b/docs/INCREMENTAL_MATH.md new file mode 100644 index 00000000..a72a94ef --- /dev/null +++ b/docs/INCREMENTAL_MATH.md @@ -0,0 +1,513 @@ +# Incremental calculation audit + +Audited on 2026-10-03 against analytics `7334289` and the local sibling +`edge` checkout `a613d8d8`. This is a math audit and implementation design. +It does not change calculation code or scheduling. + +The best first target is the awake-day pipeline: retain minute summaries, +replace changed minutes, and accumulate their contributions. Most headline +daytime calculations then cost work proportional to new or changed data. +Nighttime processing, heavy passes, and explicit recalculation should keep +running the complete pipeline and rebuild the incremental state afterward. + +## What runs today + +This package owns deterministic math. The app owns reads, scheduling, +isolate execution, and persistence. These sibling source links assume the +local checkout layout used for this audit. + +- [`DeriveScheduler`](../../edge/lib/compute/derive_scheduler.dart) queues light + work after stored data and heavy work after capture settles. It serializes + jobs and holds them during capture, workouts, and iOS background execution. +- [`DerivationEngine.run`](../../edge/lib/compute/derivation_engine.dart) selects + one freshness-critical day for a light pass and the pending span for a heavy + pass. Both use the same per-day derivation, followed by baseline refresh and + the cross-day pipeline when a day completes. “Light” currently limits days, + not calculations within a day. +- [`onehz_pipeline.dart`](../../edge/lib/compute/onehz_pipeline.dart) recomputes + sleep RR correction, HRV, resting HR, respiration, stress, whole-day rhythm + screening, readiness, awake-minute TRIMP, calories, curves, and other detail. + The coordinator's second isolate computes additional day blocks. +- [`crossday_pipeline.dart`](../../edge/lib/compute/crossday_pipeline.dart) builds + history-dependent results. A daytime update can therefore trigger work on + unchanged nightly history too. + +The current input stream is not append-only. The app stores canonical readings +with replacement by timestamp. History can arrive late, sources can change, +and sleep bounds can be edited. A timestamp cursor alone cannot prove that +cached contributions are still correct. + +## Calculation inventory + +These tables group computational entry points and their shared math. Data +classes, serialization, and barrel exports are excluded. “Incremental” means +the same mathematical result, subject to floating-point differences and the +same validity gates. It does not promise bit-for-bit identical accumulation. + +### Foundations and shared statistics + +| Calculation and source | Current math | Incremental design | +|---|---|---| +| Mean, SD, population SD, ordinary z; [`util.dart`](../lib/src/onehz/util.dart) | `mean = sum/n`; sample variance `sum((x-mean)^2)/(n-1)`; population variance divides by `n`; `z=(x-mean)/SD` | Count, mean, centered sum of squares. Append, remove, and merge summaries. Preserve each divisor and null case. | +| Percentiles, median, MAD, robust z; [`util.dart`](../lib/src/onehz/util.dart), [`baseline.dart`](../lib/src/onehz/foundations/baseline.dart) | Sort; interpolate rank `p*(n-1)/100`; scaled MAD `1.4826*median(abs(x-median(x)))`; MDC `1.96*sqrt(2)*typicalError` | Exact ordered values or histograms. MAD changes when the median moves. Recompute small daily baselines only when their membership changes. | +| Sampling cadence; [`util.dart`](../lib/src/onehz/util.dart) | Median positive timestamp gap, maximum supported cadence, and fraction of gaps within a factor of two of the median | Ordered gap counts can update; a changed cadence or eligibility can invalidate all duration-dependent results. | +| OLS slope/fit; [`util.dart`](../lib/src/onehz/util.dart) | `slope=sum((x-mx)*(y-my))/sum((x-mx)^2)`; `intercept=my-slope*mx` | Bivariate centered moments, or additive sums with careful numerical handling. | +| Theil–Sen slope; [`util.dart`](../lib/src/onehz/util.dart) | Median of all valid pairwise slopes | One new point adds up to `n` slopes. Exact storage grows quadratically; keep batch evaluation on small aggregate series. | +| Lomb–Scargle; [`util.dart`](../lib/src/onehz/util.dart) | Mean-subtracted sinusoidal projection at every grid frequency, phase rotation, one-sided PSD normalization, then band integration | Exact additive trigonometric sums per fixed frequency. Detailed derivation below. | +| Average ranks, normal p-values, Benjamini–Hochberg q-values; [`util.dart`](../lib/src/onehz/util.dart) | Tie-averaged ranks; numerical normal-tail approximation; sorted p-values with reverse running minimum | New values can change old ranks; a changed test family can change every q-value. Cache unchanged inputs, then rebuild the affected ranking/test family. | +| `correctRr`; [`rr_correction.dart`](../lib/src/onehz/foundations/rr_correction.dart) | Successive RR differences; centered local medians and quartile thresholds; beat classification; compensatory-pair reconciliation; spline correction of isolated artifacts, dropping artifact runs; cumulative/reanchored beat times | Cache stable cleaned regions only after proving their dependencies. New beats change centered neighborhoods and unresolved spline anchors. Recompute the affected suffix; retain raw beats and clock checkpoints. | +| `Baselines.update/deviation`; [`ewma_baselines.dart`](../lib/src/onehz/foundations/ewma_baselines.dart) | Winsorized EWMA center, EWMA absolute deviation against the old center, early adaptation, outlier/absence/staleness gates; z/delta/ratio | Already has a serializable one-night state update. Save state before the provisional night so reruns replace that night rather than folding it twice. | +| `inverseVarianceFuse`; [`fusion.dart`](../lib/src/onehz/foundations/fusion.dart) | `sum(value/variance)/sum(1/variance)`; fused variance `1/sum(1/variance)` over trusted inputs | Two sums and provenance. Usually only a few channels, so direct reevaluation is already cheap. | + +### Cardiac, load, and workout calculations + +| Calculation and source | Current math | Incremental design | +|---|---|---| +| `hrvTime`, `nnDiffAcf1`; [`hrv_time.dart`](../lib/src/onehz/clinical/hrv_time.dart) | RMSSD `sqrt(sum(dRR^2)/pairs)`; pNN50 `100*count(abs(dRR)>50)/pairs`; SDNN; SDANN as SD of 5-minute means; SDNN-index as mean of 5-minute SDs; pooled difference lag-1 autocorrelation controls jitter refusal | NN moments, difference moments, threshold counts, pair-product summaries, last contiguous beat/difference, and per-bin summaries. Correct seams and mutable bins explicitly. | +| `nocturnalRmssd`, `sleepSessionWindowedRmssd`; same file | First takes median of eligible 5-minute RMSSDs and a pooled jitter gate; second takes mean of locally cleaned, gap-aware 5-minute RMSSDs | Cache per-window outputs under fixed bounds and masks; replace the open/changed window. Preserve median versus mean and each cleaner's seam rule. | +| `hrvFreq`; [`hrv_freq.dart`](../lib/src/onehz/clinical/hrv_freq.dart) | Welch averages of band-integrated Lomb–Scargle spectra, 50% overlap; segment length `10/lowestBandHz`; LF/HF and normalized units; artifact/length gates | Cache completed segments per band, sum powers and count accepted segments. Global quality gates still apply at publication. | +| `decelerationCapacity`, `accelerationCapacity`; [`prsa.dart`](../lib/src/onehz/clinical/prsa.dart) | Select capped RR-change anchors; average their aligned profiles; capacity `[X(0)+X(1)-X(-1)-X(-2)]/4` | Sum profiles and anchor count. Delay finalizing anchors until all required right-hand context and the current loop's eligibility boundary have arrived. | +| `nocturnalRhr`, `hrDip`; [`nocturnal.dart`](../lib/src/onehz/clinical/nocturnal.dart) | Lowest eligible 30-minute rolling HR mean and p1 HR; dip `100*(dayMean-nightMean)/dayMean` | RHR already uses a sliding sum inside a batch call. Retain its queue, counts, best completed window, and percentile structure across calls. Dip needs two sums/counts. Cadence and sleep-bound changes invalidate dependencies. | +| `baevskyStressIndex`; [`stress_si.dart`](../lib/src/onehz/clinical/stress_si.dart) | 50 ms RR histogram; `SI=AMoPct/(2*modeSeconds*rangeSeconds)` in up-to-256-beat windows stepped by 128; median SI and component summaries | Cache stable full windows, recompute the partial tail, maintain histogram/min/max and ordered window outputs. Preserve mode tie ordering. | +| `irregularBeatScreen`; [`irregular_rhythm.dart`](../lib/src/onehz/clinical/irregular_rhythm.dart) | `SD1=SD(dRR)/sqrt(2)`; `SD2=sqrt(2*SDNN^2-SD1^2)`; pNNx; aggregate gates plus fraction of independently positive time windows | Moments and difference counts per valid input run/window, plus scored-window counts. Preserve this function's actual adjacency contract, which differs from `hrvTime`. | +| `cardiacCoherence`; [`cardiac_coherence.dart`](../lib/src/onehz/clinical/cardiac_coherence.dart) | Find spectral peak; power around peak divided by remaining power; score `100*ratio/(1+ratio)` | Cached spectral sums if grid fixed. Current low grid endpoint depends on span, so rebuild when its frequency grid changes. | +| `banisterTrimp`, `StrainScorer`, `trimpStrain`; [`load_trimp.dart`](../lib/src/onehz/clinical/load_trimp.dart) | `sum(duration*x*c*exp(b*x))`, `x=clamp((HR-RHR)/(HRmax-RHR),0,1)`; coefficients differ by sex; legacy 0–100 strain is a log map | Cache raw TRIMP contributions under fixed anchors. Minute-input and timestamp-input paths have different duration semantics. Replace the last sample's provisional duration. | +| `baselineTrimp`, `strainScore/Metric`, `dailyQuietWakingHrr`; same file | Quiet overhead `wakeMinutes*q*c*exp(b*q)`; log-map net TRIMP onto 0–21; quiet level is median HRR subject to coverage/range gates | Accumulate raw TRIMP and wake-minute count, then reevaluate the scalar map. Changing quiet level changes the baseline; changing HR anchors changes every TRIMP contribution. | +| `ctlAtlTsb`; same file | Seed with first `primeDays` mean, then CTL/ATL exponential recurrences; TSB `CTL-ATL` | Persist state and seed count. Replace today's load from yesterday's checkpoint. A history edit requires replay after the edit. | +| `illnessCusum`; [`illness_cusum.dart`](../lib/src/onehz/clinical/illness_cusum.dart) | Prior-calendar-day robust RHR z; `C=max(0,C+z-k)`; persistence/recovery state and gap resets | Save prior-day state and rolling baseline values. Score each new day once; revisions replay forward. | +| `readinessLnRmssd`; [`readiness_lnrmssd.dart`](../lib/src/onehz/clinical/readiness_lnrmssd.dart) | Prior-night lnRMSSD mean/SD standardization and short rolling summaries | Small rolling moments under the exact history convention. Usually cheaper to memoize by history revision. | +| `cosinor`; [`cosinor.dart`](../lib/src/onehz/clinical/cosinor.dart) | OLS on `y=M+beta*cos(w*t)+gamma*sin(w*t)`; amplitude/phase; residual R² and adjusted R² | Additive 3×3 normal matrix, 3-vector, `sum(y^2)`, and count. Refit a constant-size system. | +| Heart-rate zones; [`hr_zones.dart`](../lib/src/onehz/workout/hr_zones.dart) | Threshold bands; timestamp-based time credit capped by measured median cadence | Per-zone durations, last timestamp/label, gap statistics. Reclassify on zone-bound changes; replace tail credit. | +| Observed HR ceiling; [`observed_max_hr.dart`](../lib/src/onehz/workout/observed_max_hr.dart) | Highest sustained window minimum HR, confirmed by a short motion burst; gap and bounded-span guards | Retain bounded candidates and sliding minima/motion summaries. Preserve motion corroboration and tie behavior; a plain maximum is insufficient. | +| `Calories.dailyEnergy/minuteEnergy/hourlyRollup/estimateBoutCalories`; [`calories.dart`](../lib/src/onehz/workout/calories.dart) | Affine Keytel HR energy rate with caps; HRR flex gate; surplus above basal rate; below gate, measured walking cadence gives `(MET-1)*basal`; basal total prorated by caller duration | Cache each minute's bill and provenance; sum active, walking, coverage and hourly contributions. Reprice affected contributions when profile, anchors, cadence spans, or sleep mask change. | +| `Calories.metFromCadenceSpm`; same file | Below 100 spm absent; otherwise `min(6,3+0.1*(spm-100))` | Already constant work. | +| `autoDetectWorkouts`, `AutoWorkoutDetector`; [`auto_detect.dart`](../lib/src/onehz/workout/auto_detect.dart) | Sustained HR elevation, brief-dip/gap rules, merging, motion confirmation, saved-span exclusion; bout mean/max | Keep the active candidate and mergeable tail; finalize only once merge look-ahead is resolved. Saved-session edits invalidate overlap decisions. | +| `hrRecovery`; [`hr_recovery.dart`](../lib/src/onehz/workout/hr_recovery.dart) | Peak near end minus median near +60 s; candidate-tau least-squares fits `a+b*exp(-t/tau)` with residual and boundary gates | Small event window. Cache fixed-tau basis sums if repeated fitting matters; otherwise run once when tail settles. Refit if end time changes. | +| `vo2maxSubmaxEstimate`; [`vo2max.dart`](../lib/src/onehz/clinical/vo2max.dart) | ACSM speed/grade submax VO2; `VO2max=3.5+(VO2submax-3.5)/HRR` with bout gates | Already scalar. Cache/revise upstream bout summaries. | +| Sport classification; [`sport.dart`](../lib/src/onehz/workout/sport.dart) | Injectable classifier; default returns the untyped detected label | Cache by settled bout and classifier-input revision, not by day timestamp. | + +### Motion, sleep, and respiration + +| Calculation and source | Current math | Incremental design | +|---|---|---| +| `enmoSeries`, `calibrateGRef`; [`enmo.dart`](../lib/src/onehz/motion/enmo.dart) | Magnitude `sqrt(ax²+ay²+az²)`; auto gravity from medians of locally still magnitudes; ENMO mean `max(0,magnitude-gRef)`; minute mean absolute deviation; dynamic magnitude after trailing per-axis mean subtraction | Retain the trailing gravity queue/sums and minute summaries. Closed-minute MAD can run once on its samples. Auto-calibrated `gRef` is a changing dependency; it cannot silently become a fixed constant. | +| `relativeIntensityBands`; same file | Moving-value p50/p75/p90 thresholds or caller-provided frozen cuts, then class counts | Fixed cuts permit incremental labeling/counts. Whole-record cuts can change every old label. | +| `pedometer`, `calcSteps`, `livePedometer`; [`steps.dart`](../lib/src/onehz/motion/steps.dart) | High-rate moving average, local extrema, adaptive thresholds, candidate timing/regulation, calibrated count/cadence and confidence | Reuse existing independent minute/buffer boundaries first. Arbitrary streaming must preserve filter/extremum/regulation state and the whole-input dynamic reference; chunk totals are not freely additive. | +| `calibrateCadence`, floor helpers, `dailyActiveMinutes`; same file | Calibration update; robust dynamic-floor history; count gate-passing covered minutes in clock-consecutive runs of minimum length | Freeze only through existing floor policy. Cache minute gates and current run; credit the whole run when it first qualifies, then one minute per extension. | +| `staticTilt`, `positionSeries`; [`orientation.dart`](../lib/src/onehz/motion/orientation.dart) | Gravity direction gives pitch/roll; mean rotation and magnitude jitter determine stillness/coverage | Cache completed fixed bins and carry the boundary pair. A changed average orientation changes angles, so calculate once per affected bin. | +| `branchedEnergyFusion`; [`energy_fusion.dart`](../lib/src/onehz/motion/energy_fusion.dart) | Normalize acceleration/HR; branch by motion and HR validity/intensity; integrate relative load over time | Cache per-point contributions and boundary duration with fixed anchors and calibration. | +| `immobilityMask`, `vanHeesSleepWindow`; [`van_hees.dart`](../lib/src/onehz/sleep/van_hees.dart) | z-angle `atan2(z,sqrt(x²+y²))`; centered median smoothing; forward sustained angle-change test; bout merging and main-window selection | Intermediate angles and stable windows can cache, but forward windows revise the tail and main-window selection can change. Keep full sleep passes. | +| `segmentSleep`, `AdvancedSleepStager`, `cardioStager`, legacy staging wrappers; [`segment.dart`](../lib/src/onehz/sleep/segment.dart), [`advanced_stager.dart`](../lib/src/onehz/sleep/advanced_stager.dart), [`cardio_stager.dart`](../lib/src/onehz/sleep/cardio_stager.dart), [`stager.dart`](../lib/src/onehz/sleep/stager.dart) | Candidate/window rules; local cardiac/motion features; whole-night robust distributions; weighted stage axes and temporal consolidation; TIB/TST/SOL/WASO/REM-latency accounting | Cache finished-night results when inputs are unchanged. Changing distributions or session bounds can relabel old epochs. Stage counts alone do not make the detector incremental. | +| `hrLedSleepWindow`; [`hr_fallback.dart`](../lib/src/onehz/sleep/hr_fallback.dart) | Sustained nocturnal HR dip against waking/reference HR with confirmation guards | Cache intermediate bins/candidates, but changed baseline or window selection invalidates decisions. Full sleep mode. | +| `CausalStager.observe`; [`causal_stager.dart`](../lib/src/onehz/sleep/causal_stager.dart) | Past-only features, expanding references, quality/age gates, serializable next state | Existing online interface. Keep its separate estimate contract; substituting it for offline staging changes results. | +| `detectNaps`; [`nap.dart`](../lib/src/onehz/sleep/nap.dart) | Immobility bouts outside main sleep, masks and bout chaining, HR dip against awake baseline, independent TST/TIB and quality | Cache primitives, but changed main sleep, awake baseline, or new bout completion can revise older nap candidates. Batch until dependencies are stable. | +| `detectSleepCycles`, `sleepCyclesMetric`; [`cycles.dart`](../lib/src/onehz/sleep/cycles.dart) | Per-minute filtered RMSSD, centered smoothing, whole-record population z-score, prominence peaks with tallest-first distance pruning | Cache minute RMSSD. Whole-record z/prominence and competition between peaks can revise old cycles. Keep complete cycle detection. | +| `nightHrvShape`; [`night_hrv_shape.dart`](../lib/src/onehz/sleep/night_hrv_shape.dart) | Per-bin gap-aware RMSSD and sampling bands; means of first/last third and their ratio | Cache bins, reevaluate thirds when night length changes. | +| `circadianNonparametric`; [`circadian_np.dart`](../lib/src/onehz/sleep/circadian_np.dart) | IS phase-profile variance / total variance; IV successive-difference energy / variance; circular M10/L5 windows; RA `(M10-L5)/(M10+L5)` | Overall moments, adjacency energy, phase sums/counts; recompute the small average-day profile and its circular windows. | +| `phillipsSri`; [`sri.dart`](../lib/src/onehz/sleep/sri.dart) | `200*validEpochAgreement/validCases-100` across adjacent clock-aligned days | Cache each adjacent-day pair's agreement/cases. Revising day `d` changes pairs `d-1,d` and `d,d+1`. Thin pairs still enter the total even if omitted from the displayed breakdown. | +| `rsaRespRate`, `riivRespRate`, `fuseRespRate`; [`resp_rate.dart`](../lib/src/onehz/respiration/resp_rate.dart) | Window spectra/peaks, sampling-ceiling and quality gates, consensus, then trusted estimate fusion | Cache completed window spectra/peaks. RSA adapts segment length on short records and grid to each segment's span/Nyquist; only fixed completed windows have reusable fixed-grid state. | +| `breathingRateVariability`; [`brv_trend.dart`](../lib/src/onehz/respiration/brv_trend.dart) | Mean, sample SD, coefficient of variation, Theil–Sen slope over resolved respiratory windows | Moments update cheaply; exact robust slope adds pairwise work. Small series can stay batch. | +| `cvhrApneaScreen`, `cvhrPersonalDistribution`; [`cvhr_apnea.dart`](../lib/src/onehz/respiration/cvhr_apnea.dart) | Gap-separated resampling, local smoothing/detrending, prominence/width/spacing cycle rules, cycles per analyzed hour; across-night weighted rates/quantiles | Cache stable segments and weighted count/duration sums; prominence references and unfinished cycles still need reevaluation. Keep nightly detection batch initially. | +| `relativeOdi`; [`relative_odi.dart`](../lib/src/onehz/respiration/relative_odi.dart) | Relative red/IR AC-to-DC ratio, rolling baseline and duration-gated drops per analyzed hour | Rolling windows/event state are possible in principle. The audited app permanently refuses this output for its channel data. Do not reactivate it as an optimization. | +| `cardiopulmonaryCoupling`; [`cpc.dart`](../lib/src/onehz/sleep/cpc.dart) | Always absent; withdrawn because no independent respiration channel is supplied | No expensive calculation to optimize. | + +### Wellness and human-history calculations + +| Calculation and source | Current math | Incremental design | +|---|---|---| +| `readinessComposite`; [`readiness_composite.dart`](../lib/src/onehz/wellness/readiness_composite.dart) | Robust z or ordinary-z fallback, quantization/input gates, weighted signed z, `100/(1+exp(-weightedZ/weightSum))` | Cache prior-night baseline summaries by date/membership revision, then recompute the small scalar blend. Never fold today into its own reference. | +| `multivariateAnomaly`; [`anomaly.dart`](../lib/src/onehz/wellness/anomaly.dart) | Robust feature centers/scales, correlation matrix with ridge, Mahalanobis distance `sqrt(zᵀR⁻¹z)`, significance/persistence gates | Reuse unchanged baseline results. Pearson correlation is invariant to the positive affine standardization, so raw complete-row covariance summaries can replace repeated standardized-row work; retain feature eligibility and robust centers/scales. | +| `cusumChangePoints`, `segmentChangePoints`; [`changepoint.dart`](../lib/src/onehz/wellness/changepoint.dart) | Online robust pre-regime z with two accumulators/resets; offline greedy binary segmentation using prefix-sum SSE, global-variance/log-length penalty and effect gates | Online state plus exact robust baseline; replay revisions. Offline prefix sums extend cheaply, but best splits and penalty change globally, so rerun segmentation. | +| `nightlySkinTemp`, `tempCircadian`; [`temp_circadian.dart`](../lib/src/onehz/wellness/temp_circadian.dart) | Settled mean relative to night's median/family band; motion/ambient masking, median centering, cosinor and missing-aware phase variance/adjacent differences | Cache raw bins and masks; changing median can change settled membership. Cosinor sums and centered variance can update algebraically under unchanged masks. Preserve device units and missing epochs. | +| `tempIllnessFlag`, `menstrualCoverline`; [`temp_health.dart`](../lib/src/onehz/wellness/temp_health.dart) | Prior-night robust z and consecutive-night/luteal rules; coverline is prior-six maximum followed by three above-threshold nights | Checkpoints and small queues. Coverline is currently uncalled in the app and needs a defensible sensor-unit threshold before use. | +| `cycleLengthSeries`; [`cycle_lengths.dart`](../lib/src/onehz/wellness/cycle_lengths.dart) | Sort/deduplicate logged onsets, interval lengths, absolute successive length differences and extremes | Append updates last interval and difference. Inserting/editing an old date revises neighboring intervals and extrema. | +| `sleepDebt`; [`sleep_regularity.dart`](../lib/src/onehz/human/sleep_regularity.dart) | `p75(freeNightSleep)-median(recentSleep)`; absent debt without free nights | Ordered short histories. This implementation is a signed comparison, not a cumulative sum of nightly deficits. | +| `socialJetlag`, `chronotype`; [`circadian_lifestyle.dart`](../lib/src/onehz/human/circadian_lifestyle.dart) | Circular medians via resultant-anchored unwrapping; shortest-arc free/work difference; MSF sleep-duration correction | Sin/cos sums update circular mean anchor; median still needs ordered/reunwrapped history when anchor changes. Cache small nightly histories. | +| `alcoholNightFlag`, `roughNight`; [`event_detection.dart`](../lib/src/onehz/human/event_detection.dart) | Robust baseline deltas/MDC comparisons, count directional signs, derive state and optional hypothesis band | Cache reference summaries; constant-size reevaluation when nightly inputs change. | +| `percentileOfYou`, `personalRecord`; [`percentile_of_you.dart`](../lib/src/onehz/human/percentile_of_you.dart) | Rank `100*(below+0.5*equal)/n`; prior extreme plus MDC | Ordered values and extrema; preserve ties and history exclusion. | +| `glassBoxReadiness`; [`readiness_glassbox.dart`](../lib/src/onehz/human/readiness_glassbox.dart) | Deprecated baseline/percentile driver breakdown | Memoize by input revision; do not introduce a second canonical readiness score. | +| `sleepNeed`, `sleepPerformance`, `recommendedBedtime/Wake`, `strainTarget`; [`coaching.dart`](../lib/src/onehz/human/coaching.dart) | Need `clamp(base+debt+45min*strain/21-naps,6h,11h)`; performance ratio; bed/wake arithmetic with efficiency; recovery-band/load-ratio rules | Already constant-size arithmetic. Trigger only on changed upstream values. | +| `journalCorrelations`, `journalNumericCorrelations`; same file | Tag-group mean differences/pooled SD/effect size; numeric rank correlation and robust slopes; deterministic permutation significance and family-wide q-values | Group moments can update; ranks, permutation distributions, and q-values can change globally. Cache the full result under input revision and rerun on relevant log/outcome changes. | +| `scanAssociations`; [`associations.dart`](../lib/src/onehz/human/associations.dart) | Calendar/lag alignment, weekday-median adjustment, ranks, Spearman correlations, block permutations, contrast medians, FDR and redundancy filtering | Keep batch evaluation. New rows change alignment, adjusted old values, ranks and the test family. Reuse unchanged scan inputs/results rather than incrementing a final rho. | +| `weekdayEffect`; [`weekday_effect.dart`](../lib/src/onehz/human/weekday_effect.dart) | Weekday medians, tied-rank Kruskal–Wallis statistic, deterministic permutation tests and peak/trough effect gate | Small daily histories; recompute when the underlying nightly series changes. | +| `sessionMorningEffects`; [`session_cost.dart`](../lib/src/onehz/human/session_cost.dart) | Following morning minus its prior robust baseline, per-session-type median deltas and MDC | Cache paired morning contributions and per-type ordered values; history/session edits invalidate affected pairs. Preserve the current positional lookback. | +| `overreachingConjunction`; [`overreaching_conjunction.dart`](../lib/src/onehz/human/overreaching_conjunction.dart) | Acute/chronic load ratio and count of recent RHR values above baseline plus half-scale gate | Cache short history summaries; reevaluate the conjunction when its inputs change. | +| `alertnessForecast`; [`alertness_forecast.dart`](../lib/src/onehz/human/alertness_forecast.dart) | Exponential sleep/wake process, circadian/ultradian cosines, decaying inertia; normalize forecast by whole-horizon extrema and select lowest 2-hour window | Exact process recurrence exists; the forecast is small and can be cached as a whole by sleep/nap/phase inputs. Horizon changes can rescale every published point. | + +The old algorithm index contains stale entries: `sleepAccounting` is gone, +`accounting.dart` supplies the shared enum, CPC is withdrawn, and the implemented +sleep-debt calculation is the percentile-minus-median comparison above. + +## Exact update rules + +### 1. Cache contributions before nonlinear output transforms + +For `A = sum(f(x_i; theta))`, where `theta` is unchanged: + +```text +A_new = A_old + sum(f(new inputs; theta)) +A_replaced = A_old - f(old input; theta) + f(replacement; theta) +``` + +Cache raw contributions, counts, and quality evidence. Reevaluate logarithms, +ratios, clamps and publication gates from those summaries. Adding rounded or +clamped scores loses information. + +If input is a minute's mean HR, calculate the minute mean first: +`f(mean(HR samples))` generally differs from `mean(f(HR samples))`. +A partially filled minute therefore replaces one contribution as it grows. + +### 2. Stable means and variance + +Retain `n`, mean `mu`, and `M2 = sum((x-mu)^2)`. On adding `x`: + +```text +n' = n + 1 +d = x - mu +mu' = mu + d/n' +M2' = M2 + d*(x-mu') +``` + +On removing `x`, for `n > 1`: + +```text +n' = n - 1 +mu' = mu + (mu-x)/n' +M2' = M2 - (x-mu)*(x-mu') +``` + +Reset when removing the only point. Sample SD is `sqrt(M2/(n-1))`; +population SD is `sqrt(M2/n)`. Preserve absence for insufficient counts. +These centered updates avoid cancellation in `sum(x²)-sum(x)²/n`. + +For exact medians/percentiles, an ordered multiset with subtree counts supports +insertion/removal and rank selection without sorting the whole history. The +baseline median absolute deviation still needs distances from the *current* +median; storing only yesterday's median and MAD loses that information. +Small 14–28-night histories are cheap enough to rebuild when changed. + +For disjoint summaries A and B, let `d=muB-muA`, `n=nA+nB`: + +```text +mu = muA + d*nB/n +M2 = M2A + M2B + d²*nA*nB/n +``` + +This permits merging fixed bins. Pair-dependent metrics additionally need +boundary state; merging variance summaries alone does not recover RMSSD. + +### 3. RMSSD, pNNx, and the autocorrelation gate + +For each eligible successive difference `d`, cache difference count `p`, +`D1=sum(d)`, `D2=sum(d²)`, and threshold-exceedance counts. + +```text +RMSSD = sqrt(D2/p) +pNNx = 100*over/p +RMSSD' = sqrt((p*RMSSD²+d²)/(p+1)) +``` + +Across chunk boundaries, add a difference only when the two beats satisfy +the caller's actual adjacency/gap rule. Across 5-minute bins, apply the +function's bin policy; do not add a cross-bin pair to an estimator that +currently excludes it. + +The lag-1 difference autocorrelation can also avoid rescanning old differences. +For adjacent differences within the same run retain: + +```text +K = number of adjacent-difference pairs +P = sum(d_previous*d_current) +E = sum(d_previous+d_current) +m = D1/p +ACF1 = (P-m*E+K*m²) / (D2-D1²/p) +``` + +This algebra matches `nnDiffAcf1`, including its use of a global mean across +runs and a numerator that omits seam pairs. Retain the 30-difference minimum +and constant-series null result. Use centered/compensated accumulation or +rebuilding where cancellation matters. Reevaluate refusal and confidence +after updates; an old accepted RMSSD can become refused. + +### 4. TRIMP and strain + +With fixed resting HR `R`, maximum `H`, coefficients `c,b`, and time weight `dt`: + +```text +x = clamp((HR-R)/(H-R), 0, 1) +f(x) = c*x*exp(b*x) +TRIMP = sum(dt*f(x)) +baseline = wakeMinutes*f(q) +net = TRIMP-baseline +u = clamp(net/400, 0, 1) +strain = 21*log(1+14*u)/log(15) +``` + +For net at or below zero, current strain is zero. Awake minute count is part +of the state because additional quiet time changes the subtraction too. +The separate legacy 0–100 map remains a separate API. + +For timestamp-weighted scoring, the last sample provisionally gets median +cadence. Arrival of the next sample replaces that duration with the capped +actual gap, then gives the new tail its provisional credit. Updating the tail +can require subtraction. A changed median cadence can affect all durations. +The minute-input TRIMP path used by the audited day pipeline treats each +supplied minute as one minute and has no such timestamp-tail correction. + +Calculus identifies why anchor changes require reevaluation. Inside the +unclamped HRR interval: + +```text +f'(x) = c*exp(b*x)*(1+b*x) +dx/dR = (HR-H)/(H-R)² +dx/dH = -(HR-R)/(H-R)² +``` + +An anchor change affects old samples, and clamp crossings change which +derivative applies. A first-order Taylor update is an approximation. Keep +exact recalculation for changed anchors. A discrete HR or minute-mean +histogram can reprice contributions exactly by unique value when weights +and masks are also retained. + +Likewise, inside the strain map's unclamped interval, for `a=14/400`: + +```text +ds/dnet = 21*a / (log(15)*(1+a*net)) +``` + +Use this to understand sensitivity, not to accumulate score increments. +Reevaluating one logarithm is already constant work. + +### 5. Calories and zones + +For fixed profile and anchors: + +```text +flexHR = RHR + 0.40*(HRmax-RHR) +B = max(0, 10*kg + 6.25*cm - 5*age + sexConstant)/1440 +K(HR) = 60*max(0, cHR*min(HR,HRmax)+cWeight*kg+cAge*age+c0)/251.04 +``` + +`B` is basal kcal/minute. A minute at/above flex bills `max(0,K-B)`. +Otherwise, eligible measured cadence bills `(MET-1)*B`; otherwise it adds +no active surplus, with the existing per-minute abstention/provenance. +Walking is included in active total, so do not add it a second time. +Daily total is caller-prorated basal plus active. + +Retain per-minute contributions, hourly sums, and coverage counts. Zones +similarly sum time by the assigned zone. The final display is cheap; +profile/gate/zone/source changes require repricing affected minutes. + +### 6. Motion without rescanning a day + +The trailing gravity remover already updates per-axis sums inside one call: +`axisSum += incoming - expired`. Retain its queue between calls. Its output +depends on real timestamps and excludes samples at/older than the window +boundary; preserve that boundary exactly. + +For fixed gravity reference `g`, ENMO sum is additive. With an ordered set +of minute magnitudes, reference changes can also be handled exactly: + +```text +sum(max(0,m-g)) = sum(m where m>g) - g*count(m>g) +``` + +The minute MAD here means *mean* absolute deviation, unlike the baseline +MAD that means *median* absolute deviation. Because its center moves on +every append, absolute deviations cannot simply be added. If `mu=sum(m)/n`, +`k=count(m<=mu)` and `L=sum(m where m<=mu)`: + +```text +meanAbsoluteDeviation = 2*(k*mu-L)/n +``` + +An ordered structure with prefix counts/sums supports this query. For roughly +60 samples per minute, retaining the minute's magnitudes and recomputing only +that minute is simpler and likely preferable to a new tree. + +For movement bouts of minimum length `L`, retain current run length `r`. +Credit nothing below `L`, credit all `L` minutes when the run reaches `L`, +then one minute per extension. Missing/uncovered/nonpassing minutes break +the run. Replacing a past minute may split/merge a run, so recompute the +connected affected region rather than adjusting one boolean count. + +### 7. Lomb–Scargle can update exactly on a fixed grid + +This is the largest algebraic opportunity beyond simple sums. For each fixed +frequency `f`, set `w=2*pi*f`. Retain: + +```text +C = sum(cos(w*t)) S = sum(sin(w*t)) +CC = sum(cos(w*t)²) SS = sum(sin(w*t)²) +CS = sum(cos(w*t)*sin(w*t)) +YC = sum(y*cos(w*t)) YS = sum(y*sin(w*t)) +``` + +Also retain global count, `sum(y)`, variance state, and time extrema. Every +new point adds to these sums. The current implementation's changing mean +and phase can be recovered without visiting old samples: + +```text +mu = sum(y)/n +phi = 0.5*atan2(2*CS, CC-SS) # phi = w*tau +a = cos(phi) +b = sin(phi) +Uc = YC-mu*C +Us = YS-mu*S + +cNum = a*Uc+b*Us +sNum = a*Us-b*Uc +cDen = a²*CC+b²*SS+2*a*b*CS +sDen = a²*SS+b²*CC-2*a*b*CS + +PSD(f) = span/(n-1) * (cNum²/cDen+sNum²/sDen) +``` + +Preserve the source's zero-frequency, zero-denominator, too-few-points, +zero-variance and zero-span cases. PSD still has the source's physical units. +Band integration remains the same rectangular sum over its exact grid. + +For `F` frequencies, adding `deltaN` samples costs `O(deltaN*F)` and rendering +the spectrum costs `O(F)`, instead of `O(N*F)` each pass. Cached state costs +`O(F)` per active spectral window. A sliding window also needs outgoing +contributions or the raw samples to subtract them. + +`hrvFreq` already uses fixed-size, overlapping Welch segments per band. +Compute completed segments once, retain accepted power sums/counts, and +process only newly complete segments. This can save most repeated nightly +spectral work even without implementing streaming spectral sums. + +RSA and cardiac coherence have adaptive grids. Preserve each existing grid +choice: rebuild when it changes, or cache completed-window outputs. Choosing +a new universal fixed grid would change the estimator and needs separate +accuracy evaluation. Use a stable time origin; reanchoring times requires +rotating/rebuilding the trigonometric summaries, not relabeling timestamps. + +### 8. Linear fits and exponential processes + +Cosinor is linear in its three fitted coefficients. With design vector +`v=[1,cos(w*t),sin(w*t)]`, retain: + +```text +A = sum(v*vᵀ) +b = sum(v*y) +Q = sum(y²) +beta = solve(A,b) +SSE = Q-2*betaᵀ*b+betaᵀ*A*beta +SST = Q-sum(y)²/n +``` + +At the exact least-squares solution, SSE simplifies to `Q-betaᵀ*b`. +The matrix has fixed size. Amplitude, phase, R², adjusted R² and their gates +are then scalar calculations. Use centered sums or incremental QR if +conditioning/cancellation warrants it; compare to the current residual pass. + +Each recovery-fit tau candidate similarly uses fixed basis `e=exp(-t/tau)`. +Retain `n,sum(e),sum(e²),sum(y),sum(e*y),sum(y²)` and solve the two-parameter +fit. Its complete residual sum follows from those sums. Keep the current +candidate grid, tie selection, amplitude, boundary and residual gates. + +The alertness process solves `dS/dt=-(S-target)/tau`: + +```text +S(t+dt) = target + (S(t)-target)*exp(-dt/tau) +``` + +This is an exact recurrence over a constant-target interval, not an Euler +approximation. Follow the current time-step/nap membership policy to preserve +results. Normalizing the final forecast by its whole-horizon extrema remains +a global dependency, so caching the small complete forecast is preferable. + +CTL/ATL and EWMA baselines are already recurrences. Persist their checkpoint +states instead of replaying unchanged history. Winsor/outlier logic is +state-dependent, so revised historical values require forward replay. + +## Cache validity and full-pass policy + +The app should pass an explicit calculation mode. Only an awake periodic +update with proven unchanged dependencies qualifies for incremental execution. +During sleep, on heavy/finalization passes, and on forced recalculation, use +the complete current pipeline, then replace the cache with rebuilt state. +If sleep eligibility is uncertain, the safe initial policy is a complete pass. +The audited light/heavy job kind alone does not encode this distinction. + +Cache keys need calculation/state version, device family, profile/anchors, +source-selection revision, sleep bounds/masks, bin origin, timezone/day bounds, +baseline membership, and relevant manual edits. Include all parameters that +the function actually reads. Finalized results and provisional tail state +must have different lifetimes. + +Store sufficient statistics at full precision. Store per-bin contributions +so edits can remove old values before adding replacements. Do not reconstruct +state from rounded `Metric.toJson` scalars. + +The existing app fingerprint includes profile and previous-day inputs, but its +raw day component is `MAX(rec_ts):COUNT(*)`. That detects many insertions; it +does not detect replacement of a value at an existing timestamp with unchanged +count/max. An incremental cache requires a mutation revision/change log or +another proof that covers replacements, RR changes, and source/mask changes. +`MAX:COUNT` alone is insufficient. + +Publish cache state and derived output for the same captured input revision. +Commit together, or associate them with an identical validated revision and +discard mismatches. A failed/cancelled calculation must not advance the +checkpoint. Retrying an identical revision must be idempotent. Missing or +incompatible state falls back to a complete calculation. + +Late data invalidates its affected bins and downstream dependencies. A changed +NN beat also affects neighbor differences and downstream clock state where +times are accumulated. Centered filters compose their support widths; the RR +corrector's spline-anchor search and run reconciliation can extend dependency +reach. A fixed “last 91 beats” replay rule is not established by this audit. +Start with complete RR correction until stable-prefix equivalence is proved. + +Preserve absence, quality gates, coverage denominators, confidence, notes, +drivers and provenance. Updates are allowed to lower confidence or withdraw +a previously present metric. Valid data in a gap must never be inferred from +an accumulator's previous state. + +## Implementation order and validation + +1. Add mode/dependency revisions in `edge` and plain explicit accumulator states + in analytics. Keep existing batch entry points as the comparison reference. +2. Cache minute HR/motion summaries and their nonlinear TRIMP, energy and zone + contributions. Replace open/changed minutes. Reuse unchanged completed-night + results; keep every sleep-time calculation complete. +3. Cache baseline summaries and whole cross-day results by the inputs they read. + Daytime HR changes should not rerun unchanged nightly association analyses. + Advance daily recurrent states from a prior-day checkpoint, replacing today. +4. Cache completed Welch windows and shared cleaned RR/intermediate features. + The app currently invokes `detectSleepCycles` directly and again through + `sleepCyclesMetric`; share that result while preserving the envelope. +5. Add fixed-grid spectral accumulators if measurements justify their state + size and complexity. Prove RR stable-prefix boundaries before incremental + correction; keep global sleep-stage and retrospective scans complete. + +Compare incremental results against the current batch functions at every +prefix and varied chunk boundary, including one sample at a time. Exercise +duplicates, out-of-order insertion, value replacement, gaps, off-wrist data, +mutable partial minutes, changed anchors, cadence-mode changes, source edits, +sleep edits, midnight/DST, baseline expiration, interruption and restart. +Match publication gates and discrete decisions exactly; measure numerical +differences before selecting tolerances. Near a decision threshold, floating +differences can change absence or labels and need explicit handling. + +Real-capture comparisons should cover every device family with fixtures and +the existing sleep/causal/pedometer validation paths when those paths change. +Instrument reads, decoded rows, processed samples/windows, CPU time, isolate +payload/heap and serialized cache size. Accumulator improvements will not +remove full-day DB reads, decoding, or bundle serialization on their own. + +No performance speedup was measured in this audit. As a work-count example, +a 5-minute update at the end of 12 hours has 300 new seconds versus 43,200 +old-plus-new seconds: about 144× less scanning in an eligible append-only +component. That is not an app-level speedup estimate. + +The scratch algebra check ran 3,031 comparisons of append/remove variance, +pooled multi-run ACF1, direct versus cached Lomb–Scargle PSD, minute mean +absolute deviation, changed-reference ENMO, and nonlinear contribution +replacement. All passed at relative tolerance `2e-9` / absolute tolerance +`2e-7`; maximum absolute spectral difference was about `1.02e-10` on those +synthetic inputs. This checks the derivations, not a Dart implementation or +real-world speed. The scratch script was not added to the runtime library. +It is available in this session at `/tmp/openstrap_incremental_math_check.py`. diff --git a/docs/INCREMENTAL_USAGE.md b/docs/INCREMENTAL_USAGE.md new file mode 100644 index 00000000..2fbb4fa1 --- /dev/null +++ b/docs/INCREMENTAL_USAGE.md @@ -0,0 +1,90 @@ +# Incremental calculations + +Import `package:openstrap_analytics/onehz.dart`. The batch functions remain +available and are the reference algorithms used by the tests. + +| State | Cached math | Rebuild conditions | +|---|---|---| +| `RunningMoments` | Centered count, mean and sum of squared deviations; add, remove and merge | Caller owns membership when removing a value | +| `IntHistogram` | Sorted `value -> count` list of an integer-valued series; add, remove and merge | Caller owns membership when removing a value; whole numbers only | +| `IncrementalLombScargle` | Seven trigonometric sums per frequency, centered moments and time span | Earlier point edits, removals, a changed origin, or `force: true` | +| `IncrementalHrvTime` | NN moments, contiguous differences, lag-one products and five-minute bins | Earlier beat/time edits, removals, timing-mode changes, or force | +| `IncrementalEnmoSeries` | Trailing gravity sums and motion-minute buckets | Earlier sample edits, parameter changes, automatic gravity calibration, or force | +| `IncrementalMinuteMetrics` | TRIMP and energy contributions indexed by minute key | Changed minutes are repriced; changed HR anchors, sex or profile reprice all minutes | +| `CalculationCache` | Results with complete, owned dependency snapshots | Changed dependencies, explicit full evaluation, or eviction | + +`IntHistogram.percentile(p)` and `.median` return exactly what +`percentileSorted` returns for the expanded sorted list (same interpolation +between the two neighbouring order statistics, found from cumulative counts), +and null when empty. `add` takes a whole number (`70` or `70.0`; anything else +throws), `remove` throws if the value is absent, and `p` outside 0 to 100 +throws. Use it only for integral data: heart rate is whole bpm with about 45 +distinct values in a real night, but accelerometer magnitudes are almost all +distinct (30,848 of 32,041), so a histogram there is as long as the data. If +`calibrateGRef` ever needs to be incremental, use an order-statistic structure. + +Batch callers that take a median or percentile of heart rate over a long or +sliding window, and what adopting it would need: + +| Caller | Window | Notes | +|---|---|---| +| `_rollingMedianValidOnly` (`sleep/hr_fallback.dart`) | Centered, per sample | Best fit: a sliding add/remove. Input is `List`, so adopt only where the source is whole bpm. | +| `StrainScorer.estimateHRmax` (`clinical/load_trimp.dart`) | Trailing HR history | A high percentile of a long history; a running histogram avoids the re-sort per call. | +| `nocturnalRhr` p1 (`clinical/nocturnal.dart`) | The night's on-skin HR | One percentile per call, rebuilt each time. | +| `causal_stager` / `cardio_stager` local gates | ±180 epochs, p25 and p50 | Epoch HR is a mean, not a whole number. Not eligible unless the epoch value is made integral, which would change output. | +| `hrBaseline` (`sleep/advanced_stager.dart`) | Whole record | Median of all HR samples; a single call. | + +No existing incremental state recomputes an HR median per pass, so none was +switched; the histogram has no caller in the app yet. + +Pass the complete current input to `sync`. Each state checks that retained data +still matches, so replacing an old timestamp or value cannot silently leave a +stale result. Input comparison and output construction still visit retained +data. The work counters count samples whose math was evaluated, not every +array read or allocation. + +Minute keys must be unique and aligned with HR and optional cadence. Day duration +must be a finite whole number of minutes, matching `Calories.dailyEnergy`. +Changing day duration or the quiet-waking strain reference changes the summary +without repricing minute contributions. + +Use `includeMinuteSeries: false` when a caller needs only TRIMP, strain and +energy totals. It avoids rebuilding the minute output list. Turning that view +back on exposes the same retained contributions without repricing them. + +ENMO with an automatic gravity reference uses the full `enmoSeries` algorithm. +That reference comes from the whole record's median and can change earlier +minutes. Pass an explicit measured reference to use the incremental gravity +queue. The RR corrector also remains a full calculation because new beats can +change which earlier beats survive correction. + +HRV preserves time gaps, partially filled five-minute bins, the beat-timing +jitter refusal, confidence and notes. Ill-conditioned autocorrelation sums and +values near the refusal threshold fall back to the batch calculation. + +State checkpoints use `toJson` / `fromJson`, with version and type checks. +They retain double precision before public metric formatting. An unsupported +or malformed checkpoint throws; discard it and perform a full rebuild. + +Cache dependencies should be JSON-compatible maps, lists and scalars. Cached +results should be immutable objects or JSON-compatible collections. The cache +owns copies of collection results and dependencies and evicts the least +recently used entries at its configured limit. + +`CalculationMode.periodicAwake` is the only mode that permits reuse. Sleep, +heavy and forced runs must evaluate the full algorithms. The app owns awake +evidence, phone charging state and when a successfully persisted result may +replace its previous cache. These are orchestration decisions, so the math +package has no clock, battery API or database dependency. + +Run the deterministic timing comparison with: + +```sh +nix develop -c dart run tool/incremental_benchmark.dart +``` + +The benchmark reports local elapsed times and math-work counts. The parameterized +tests compare values, absent results, confidence, notes and provenance against +the batch algorithms after appends, revisions, deletions and checkpoint restores. +They also read subsets of the checked-in July 2026 overnight capture. Missing +capture files fail these tests rather than skipping them. diff --git a/lib/onehz.dart b/lib/onehz.dart index 973f7859..389c3513 100644 --- a/lib/onehz.dart +++ b/lib/onehz.dart @@ -49,9 +49,7 @@ export 'src/onehz/workout/workout.dart'; export 'src/onehz/wellness/wellness.dart'; export 'src/onehz/human/human.dart'; - +export 'src/onehz/incremental.dart'; // Calculation-group registry (design 06): the groups this package declares. - - -export 'src/onehz/incremental_core.dart'; +export 'src/onehz/registry/group_registry.dart'; diff --git a/lib/src/onehz/incremental.dart b/lib/src/onehz/incremental.dart new file mode 100644 index 00000000..a4a7c4a2 --- /dev/null +++ b/lib/src/onehz/incremental.dart @@ -0,0 +1,27 @@ +/// Stateful counterparts of the independent batch calculations. +library; + +import 'dart:math' as math; +import 'types.dart'; +import 'clinical/hrv_time.dart'; +import 'clinical/load_trimp.dart'; +import 'motion/enmo.dart'; +import 'workout/calories.dart'; +import 'incremental_core.dart'; + +export 'motion/enmo.dart' show defaultGravityWindowS; +export 'incremental_core.dart'; + +part 'incremental_hrv.dart'; +part 'incremental_enmo.dart'; +part 'incremental_minutes.dart'; +part 'incremental_state.dart'; + +enum CalculationMode { + periodicAwake, + sleep, + heavy, + forced; + + bool get canReuse => this == periodicAwake; +} diff --git a/lib/src/onehz/incremental_enmo.dart b/lib/src/onehz/incremental_enmo.dart new file mode 100644 index 00000000..af19a49c --- /dev/null +++ b/lib/src/onehz/incremental_enmo.dart @@ -0,0 +1,198 @@ +part of 'incremental.dart'; + +class _MotionBin { + final List mags = []; + double magSum = 0, enmoSum = 0, dynSum = 0; + MotionMinute? row; + void render(int key) { + final mean = magSum / mags.length; + var deviation = 0.0; + for (final mag in mags) { + deviation += (mag - mean).abs(); + } + row = MotionMinute(key * 60000.0, mags.length, enmoSum / mags.length, + deviation / mags.length, mean, dynSum / mags.length); + } +} + +/// Explicit-reference motion updates retain the causal trailing axis queue. +/// Automatic calibration is retrospective and uses the complete batch path. +// van Hees 2013 ENMO and Vähä-Ypyä 2015 MAD, as in enmoSeries(); dynAmp remains +// the internal vector high-pass adaptation. Explicit gravity reference uses the +// same trailing queue; automatic calibration recomputes batch. Checked numeric +// field tolerance is max(1e-9 absolute, 1e-8 relative); sample counts and minute +// timestamps match exactly. +class IncrementalEnmoSeries { + List _valid = []; + final Map _bins = {}; + double? _gRef; + double _windowS = defaultGravityWindowS; + double _sx = 0, _sy = 0, _sz = 0; + int _lo = 0, _processedPoints = 0; + IncrementalEnmoSeries(); + + factory IncrementalEnmoSeries.fromJson(Map json) { + _version(json, 'IncrementalEnmoSeries'); + final state = IncrementalEnmoSeries(); + state._processedPoints = _count(json['processedPoints']); + state._gRef = + json['gRef'] == null ? null : _number(json['gRef'], finite: true); + state._windowS = _number(json['windowS'], finite: true); + state._sx = _number(json['sx'], finite: true); + state._sy = _number(json['sy'], finite: true); + state._sz = _number(json['sz'], finite: true); + state._lo = _count(json['lo']); + if (json['valid'] is! List || json['bins'] is! List) + throw const FormatException('Invalid motion state'); + state._valid = [ + for (final raw in json['valid'] as List) + (() { + final row = _numbers(raw, finite: true); + if (row.length != 4) + throw const FormatException('Invalid acceleration row'); + return AccelSample(row[0], row[1], row[2], row[3]); + })() + ]; + for (final raw in json['bins'] as List) { + final data = _map(raw); + if (data['key'] is! int) + throw const FormatException('Invalid motion minute'); + final key = data['key'] as int; + if (state._bins.containsKey(key)) + throw const FormatException('Duplicate motion minute'); + final bin = _MotionBin(); + bin.mags.addAll(_numbers(data['mags'], finite: true)); + bin.magSum = _number(data['magSum'], finite: true); + bin.enmoSum = _number(data['enmoSum'], finite: true); + bin.dynSum = _number(data['dynSum'], finite: true); + if (bin.mags.isEmpty || bin.enmoSum < 0 || bin.dynSum < 0) + throw const FormatException('Invalid motion sums'); + bin.render(key); + state._bins[key] = bin; + } + if (state._processedPoints < state._valid.length || + state._lo > math.max(0, state._valid.length - 1) || + state._bins.values.fold(0, (n, b) => n + b.mags.length) != + state._valid.length || + (state._gRef == null && state._valid.isNotEmpty)) + throw const FormatException('Inconsistent motion checkpoint'); + for (var i = 1; i < state._valid.length; i++) { + if (state._valid[i].tsMs < state._valid[i - 1].tsMs) + throw const FormatException('Unsorted motion state'); + } + return state; + } + int get processedPoints => _processedPoints; + EnmoResult sync(List samples, + {double? gRef, + int minSamplesPerMinute = 30, + double gravityWindowS = defaultGravityWindowS, + int? expectedMinutes, + bool force = false}) { + final valid = samples.where((s) => s.valid).toList() + ..sort((a, b) => a.tsMs.compareTo(b.tsMs)); + if (gRef == null || + !gRef.isFinite || + !gravityWindowS.isFinite || + valid.any((s) => + !s.tsMs.isFinite || + !s.x.isFinite || + !s.y.isFinite || + !s.z.isFinite)) { + final result = enmoSeries(samples, + gRef: gRef, + minSamplesPerMinute: minSamplesPerMinute, + gravityWindowS: gravityWindowS, + expectedMinutes: expectedMinutes); + _reset(); + _processedPoints += valid.length; + return result; + } + var append = !force && + gRef == _gRef && + gravityWindowS == _windowS && + valid.length >= _valid.length; + if (append) { + for (var i = 0; i < _valid.length; i++) { + final a = _valid[i], b = valid[i]; + if (a.tsMs != b.tsMs || a.x != b.x || a.y != b.y || a.z != b.z) { + append = false; + break; + } + } + } + if (!append) _reset(); + _gRef = gRef; + _windowS = gravityWindowS; + final changed = {}; + for (var i = _valid.length; i < valid.length; i++) { + final s = valid[i]; + _sx += s.x; + _sy += s.y; + _sz += s.z; + while (_lo < i && s.tsMs - valid[_lo].tsMs >= gravityWindowS * 1000) { + _sx -= valid[_lo].x; + _sy -= valid[_lo].y; + _sz -= valid[_lo].z; + _lo++; + } + final n = i - _lo + 1; + final dx = s.x - _sx / n, dy = s.y - _sy / n, dz = s.z - _sz / n; + final dyn = math.sqrt(dx * dx + dy * dy + dz * dz); + final mag = math.sqrt(s.x * s.x + s.y * s.y + s.z * s.z); + final key = (s.tsMs / 60000).floor(); + final bin = _bins[key] ??= _MotionBin(); + bin.mags.add(mag); + bin.magSum += mag; + bin.enmoSum += mag > gRef ? mag - gRef : 0; + bin.dynSum += dyn; + changed.add(key); + _processedPoints++; + } + _valid = valid; + for (final key in changed) { + _bins[key]!.render(key); + } + final keys = _bins.keys.toList()..sort(); + final covered = + _bins.values.where((b) => b.mags.length >= minSamplesPerMinute).length; + final span = keys.isEmpty ? 0 : keys.last - keys.first + 1; + final denominator = expectedMinutes ?? span; + return EnmoResult(gRef, [for (final key in keys) _bins[key]!.row!], + denominator <= 0 ? 0 : (covered / denominator).clamp(0.0, 1.0)); + } + + void _reset() { + _valid = []; + _bins.clear(); + _gRef = null; + _windowS = defaultGravityWindowS; + _sx = _sy = _sz = 0; + _lo = 0; + } + + Map toJson() => { + 'version': 1, + 'type': 'IncrementalEnmoSeries', + 'processedPoints': _processedPoints, + 'gRef': _gRef, + 'windowS': _windowS, + 'sx': _sx, + 'sy': _sy, + 'sz': _sz, + 'lo': _lo, + 'valid': [ + for (final s in _valid) [s.tsMs, s.x, s.y, s.z] + ], + 'bins': [ + for (final entry in _bins.entries) + { + 'key': entry.key, + 'mags': List.of(entry.value.mags), + 'magSum': entry.value.magSum, + 'enmoSum': entry.value.enmoSum, + 'dynSum': entry.value.dynSum, + } + ], + }; +} diff --git a/lib/src/onehz/incremental_hrv.dart b/lib/src/onehz/incremental_hrv.dart new file mode 100644 index 00000000..3c576a30 --- /dev/null +++ b/lib/src/onehz/incremental_hrv.dart @@ -0,0 +1,266 @@ +part of 'incremental.dart'; + +/// Appends NN moments, gap-aware differences and mutable five-minute bins. +// Task Force 1996 hrvTime() statistics with retained moments and gap-aware +// differences. Applicable jitter exemptions/fallbacks invoke batch hrvTime(); +// otherwise numerical accumulation differs. The jitter policy is an internal +// rule, ESTIMATE with low confidence. Welch 1967 and Hirsch & Bishop 1981 +// support spectral/respiratory components only; see rmssd_gate.dart. Checked numeric field tolerance is max(1e-9 +// absolute, 1e-8 relative); presence, tier, note, inputs and beat counts match +// exactly. +class IncrementalHrvTime { + List _nn = []; + List? _times; + RunningMoments _levels = RunningMoments(); + final List _bins = []; + RunningMoments _binMeans = RunningMoments(); + double _binSdSum = 0; + int _binIdx = 0; + int _pairs = 0, _over50 = 0, _lagPairs = 0, _processedPoints = 0; + double _diffSum = 0, _diffSquares = 0, _products = 0, _endpoints = 0; + double? _lastDiff; + bool _finite = true; + // Evidence for the jitter gate below the ACF floor, derived from the series + // (not checkpointed; rebuilt from `nn` / `times` on restore): the Welch + // segments `nnDiffNoiseShare` would see (64 points, stride 32, per + // contiguous run). See [sync] for why under 20 the batch can only refuse. + int _closedSegments = 0, _runDiffs = 0; + + IncrementalHrvTime(); + factory IncrementalHrvTime.fromJson(Map json) { + _version(json, 'IncrementalHrvTime'); + final state = IncrementalHrvTime(); + state._nn = _numbers(json['nn']); + state._times = json['times'] == null ? null : _numbers(json['times']); + state._processedPoints = _count(json['processedPoints']); + if (json['finite'] is! bool) + throw const FormatException('Invalid HRV mode'); + state._finite = json['finite'] as bool; + state._levels = RunningMoments.fromJson(_map(json['levels'])); + state._pairs = _count(json['pairs']); + state._over50 = _count(json['over50']); + state._lagPairs = _count(json['lagPairs']); + state._diffSum = _number(json['diffSum'], finite: true); + state._diffSquares = _number(json['diffSquares'], finite: true); + state._products = _number(json['products'], finite: true); + state._endpoints = _number(json['endpoints'], finite: true); + state._lastDiff = json['lastDiff'] == null + ? null + : _number(json['lastDiff'], finite: true); + if (json['binIdx'] is! int || json['bins'] is! List) + throw const FormatException('Invalid HRV bins'); + state._binIdx = json['binIdx'] as int; + for (final bin in json['bins'] as List) { + final moments = RunningMoments.fromJson(_map(bin)); + if (moments.count == 0) throw const FormatException('Empty HRV bin'); + state._bins.add(moments); + if (moments.count >= 2) { + state._binMeans.add(moments.mean!); + state._binSdSum += moments.sampleSd!; + } + } + final timed = + state._times != null && state._times!.length == state._nn.length; + if (state._processedPoints < state._nn.length || + state._over50 > state._pairs || + state._lagPairs > state._pairs || + state._pairs > math.max(0, state._nn.length - 1) || + state._diffSquares < 0 || + (state._finite && + (state._levels.count != state._nn.length || + state._nn.any((v) => !v.isFinite) || + (timed && state._times!.any((v) => !v.isFinite)))) || + (!state._finite && state._levels.count != 0) || + (state._finite && + timed && + state._bins.fold(0, (n, b) => n + b.count) != + state._nn.length) || + ((!timed || !state._finite) && state._bins.isNotEmpty)) { + throw const FormatException('Inconsistent HRV checkpoint'); + } + if (state._finite) { + for (var i = 0; i < state._nn.length; i++) { + state._evidence(i, state._nn, timed ? state._times : null); + } + } + return state; + } + + static int _segmentsOf(int diffs) => diffs + 1 < 64 ? 0 : (diffs + 1 - 64) ~/ 32 + 1; + + /// Updates the jitter-gate evidence for point [i] of [nn] ([times] when the + /// series is timed), splitting runs exactly as `hrvTime` does. + void _evidence(int i, List nn, List? times) { + if (i == 0) return; + if (times != null && times[i] - times[i - 1] > nn[i] + .5) { + _closedSegments += _segmentsOf(_runDiffs); + _runDiffs = 0; + } else { + _runDiffs++; + } + } + int get processedPoints => _processedPoints; + Metric sync(List nnMs, + {List? nnTimesMs, + double artifactFraction = 0, + bool force = false}) { + final timed = nnTimesMs != null && nnTimesMs.length == nnMs.length; + final oldTimed = _times != null && _times!.length == _nn.length; + final finite = nnMs.every((v) => v.isFinite) && + (!timed || nnTimesMs.every((v) => v.isFinite)); + if (!finite) { + final result = hrvTime(nnMs, + nnTimesMs: nnTimesMs, artifactFraction: artifactFraction); + _reset(); + _finite = false; + _nn = List.of(nnMs); + _times = nnTimesMs == null ? null : List.of(nnTimesMs); + _processedPoints += nnMs.length; + return result; + } + final append = !force && + _finite && + timed == oldTimed && + _prefix(_nn, nnMs) && + (!timed || _prefix(_times!, nnTimesMs)); + if (!append) _reset(); + for (var i = _nn.length; i < nnMs.length; i++) { + final value = nnMs[i]; + _evidence(i, nnMs, timed ? nnTimesMs : null); + _levels.add(value); + if (i > 0) { + if (timed && nnTimesMs[i] - nnTimesMs[i - 1] > value + .5) { + _lastDiff = null; + } else { + final d = value - nnMs[i - 1]; + _pairs++; + _diffSum += d; + _diffSquares += d * d; + if (d.abs() > 50) _over50++; + if (_lastDiff != null) { + _lagPairs++; + _products += _lastDiff! * d; + _endpoints += _lastDiff! + d; + } + _lastDiff = d; + } + } + if (timed) { + final idx = ((nnTimesMs[i] - nnTimesMs.first) / 300000).floor(); + if (_bins.isEmpty || idx != _binIdx) { + _bins.add(RunningMoments()); + _binIdx = idx; + } + final bin = _bins.last; + if (bin.count >= 2) { + _binMeans.remove(bin.mean!); + _binSdSum -= bin.sampleSd!; + } + bin.add(value); + if (bin.count >= 2) { + _binMeans.add(bin.mean!); + _binSdSum += bin.sampleSd!; + } + } + _processedPoints++; + } + _nn = List.of(nnMs); + _times = nnTimesMs == null ? null : List.of(nnTimesMs); + if (nnMs.length < 2) + return hrvTime(nnMs, + nnTimesMs: nnTimesMs, artifactFraction: artifactFraction); + double? acf; + if (_pairs >= 30) { + final m = _diffSum / _pairs; + final variance = _diffSquares - _diffSum * m; + if (variance > 0) + acf = (_products - m * _endpoints + _lagPairs * m * m) / variance; + // Reevaluate ill-conditioned sums and decisions at the refusal boundary + // using the batch function's direct centered differences. + if ((_diffSquares > 0 && variance.abs() < 1e-12 * _diffSquares) || + (acf != null && (acf - kNnDiffAcf1Floor).abs() < 1e-10)) { + _processedPoints += nnMs.length; + return hrvTime(nnMs, + nnTimesMs: nnTimesMs, artifactFraction: artifactFraction); + } + } + final jittery = acf != null && acf < kNnDiffAcf1Floor; + // Under the floor the batch refuses only when the spectrum shows no + // respiratory line on a low floor, and a refused timed night may still + // publish from windows on a breathing line steady in Hz (hrv_time.dart + // `_jitterRefused`, `_breathingRmssd`). Both read the difference runs, so + // when either could apply the batch decides, on the series handed in. + // With fewer than 20 segments neither can: the noise share is null (the + // night is refused), and the breathing test needs 12 windows each holding + // a contiguous run of 127 differences, which alone makes at least 3 + // segments per window (36). So a short or fragmented refused night stays + // on the sums. + if (jittery && _closedSegments + _segmentsOf(_runDiffs) >= 20) { + _processedPoints += nnMs.length; + return hrvTime(nnMs, + nnTimesMs: nnTimesMs, artifactFraction: artifactFraction); + } + final quality = + acf == null ? 1.0 : (1 - acf / kNnDiffAcf1Floor).clamp(0.0, 1.0); + return Metric( + value: HrvTime( + nBeats: nnMs.length, + sdnn: _levels.sampleSd, + rmssd: _pairs > 0 && !jittery + ? math.sqrt(_diffSquares / _pairs) + : null, + pnn50: _pairs > 0 && !jittery ? 100.0 * _over50 / _pairs : null, + sdann: _binMeans.count >= 2 ? _binMeans.sampleSd : null, + sdnnIndex: + _binMeans.count >= 2 ? _binSdSum / _binMeans.count : null, + diffAcf1: acf), + confidence: ((nnMs.length / 250.0).clamp(0.0, 1.0) * + quality * + (1 - artifactFraction)) + .clamp(.3, .95), + tier: Tier.high, + inputs_used: const ['rr_cleaned'], + note: jittery + ? 'rmssd_refused:acf1=${acf.toStringAsFixed(3)} — the NN successive ' + 'differences are essentially differenced white noise (−0.5 = pure, floor ' + '$kNnDiffAcf1Floor), so RMSSD/pNN50 would measure beat-timing jitter, not ' + 'vagal tone. SDNN/SDANN survive it and are the lead here. PRV not ECG-HRV.' + : 'PRV not ECG-HRV; RMSSD/pNN50 are quantization-sensitive at 1 Hz ' + '— lead with SDNN/SDANN'); + } + + void _reset() { + _nn = []; + _times = null; + _levels = RunningMoments(); + _bins.clear(); + _binMeans = RunningMoments(); + _binSdSum = 0; + _binIdx = 0; + _pairs = _over50 = _lagPairs = 0; + _diffSum = _diffSquares = _products = _endpoints = 0; + _lastDiff = null; + _finite = true; + _closedSegments = _runDiffs = 0; + } + + Map toJson() => { + 'version': 1, + 'type': 'IncrementalHrvTime', + 'nn': _nn.map(_encode).toList(), + 'times': _times?.map(_encode).toList(), + 'processedPoints': _processedPoints, + 'finite': _finite, + 'levels': _levels.toJson(), + 'bins': [for (final b in _bins) b.toJson()], + 'binIdx': _binIdx, + 'pairs': _pairs, + 'over50': _over50, + 'lagPairs': _lagPairs, + 'diffSum': _diffSum, + 'diffSquares': _diffSquares, + 'products': _products, + 'endpoints': _endpoints, + 'lastDiff': _lastDiff, + }; +} diff --git a/lib/src/onehz/incremental_minutes.dart b/lib/src/onehz/incremental_minutes.dart new file mode 100644 index 00000000..8affd79c --- /dev/null +++ b/lib/src/onehz/incremental_minutes.dart @@ -0,0 +1,376 @@ +part of 'incremental.dart'; + +class MinuteMetrics { + final Metric trimp; + final Metric strain; + final ({double total, double active, double basal, double walking})? energy; + final MinuteEnergySeries? minutes; + const MinuteMetrics( + {required this.trimp, required this.strain, this.energy, this.minutes}); +} + +class _MinuteBill { + final double hr; + final double? cadence; + final double trimp; + final MinuteEnergySource? source; + final double active, walking; + const _MinuteBill(this.hr, this.cadence, this.trimp, this.source, this.active, + this.walking); +} + +/// Retains nonlinear TRIMP and energy contributions by caller-supplied key. +/// Keys must be unique. Day duration must be a finite whole number of minutes, +/// matching the batch dailyEnergy API's integer duration. +// Banister 1991 / Morton 1990 TRIMP; Keytel 2005 / Mifflin-St Jeor 1990 / Tudor- +// Locke 2019 energy. Retains identical keyed minute contributions; edits replace +// them and dependency changes rebuild. Checked numeric field tolerance is +// max(1e-9 absolute, 1e-8 relative); presence, tier, inputs, notes, minute keys, +// counts, sources and abstention reasons match exactly. +class IncrementalMinuteMetrics { + final Map _bills = {}; + final List _keys = []; + final List<_MinuteBill> _orderedBills = []; + List? _parameters; + double _trimpTotal = 0; + double _hrActiveTotal = 0, _walkingTotal = 0; + double? _basalPerMinute; + MinuteEnergyPricer? _energyPricer; + bool _pricerReady = false; + int _processedMinutes = 0; + IncrementalMinuteMetrics(); + + factory IncrementalMinuteMetrics.fromJson(Map json) { + _version(json, 'IncrementalMinuteMetrics'); + final state = IncrementalMinuteMetrics(); + state._processedMinutes = _count(json['processedMinutes']); + state._trimpTotal = _number(json['trimpTotal'], finite: true); + state._basalPerMinute = json['basalPerMinute'] == null + ? null + : _number(json['basalPerMinute'], finite: true); + if (json['parameters'] != null) { + final parameters = json['parameters']; + if (parameters is! List || + parameters.length != 4 || + parameters[2] is! String || + !Sex.values.any((s) => s.name == parameters[2])) { + throw const FormatException('Invalid minute parameters'); + } + for (final i in [0, 1]) { + if (parameters[i] != null) _number(parameters[i]); + } + final profile = parameters[3]; + if (profile != null) { + if (profile is! List || profile.length != 4 || profile[3] is! String) + throw const FormatException('Invalid minute profile'); + for (var i = 0; i < 3; i++) { + _number(profile[i]); + } + } + state._parameters = [ + parameters[0], + parameters[1], + parameters[2], + profile == null ? null : List.of(profile as List), + ]; + } + if (json['bills'] is! List) + throw const FormatException('Invalid minute bills'); + var summedTrimp = 0.0; + for (final raw in json['bills'] as List) { + final data = _map(raw); + if (data['key'] is! int) + throw const FormatException('Invalid minute key'); + final key = data['key'] as int; + if (state._bills.containsKey(key)) + throw const FormatException('Duplicate minute key'); + final hr = _number(data['hr']); + final cadence = data['cadence'] == null ? null : _number(data['cadence']); + final trimp = _number(data['trimp'], finite: true); + final active = _number(data['active'], finite: true); + final walking = _number(data['walking'], finite: true); + final sourceName = data['source']; + MinuteEnergySource? source; + if (sourceName != null) { + if (sourceName is! String || + !MinuteEnergySource.values.any((s) => s.name == sourceName)) + throw const FormatException('Invalid energy source'); + source = MinuteEnergySource.values.byName(sourceName); + } + if (trimp < 0 || + active < 0 || + walking < 0 || + (source != MinuteEnergySource.cadence && walking != 0) || + (source != MinuteEnergySource.hr && active != 0) || + (state._basalPerMinute == null && source != null)) + throw const FormatException('Invalid minute contribution'); + state._bills[key] = + _MinuteBill(hr, cadence, trimp, source, active, walking); + state._keys.add(key); + state._orderedBills.add(state._bills[key]!); + summedTrimp += trimp; + state._hrActiveTotal += active; + state._walkingTotal += walking; + } + if (state._processedMinutes < state._bills.length || + (state._parameters == null && state._bills.isNotEmpty) || + (summedTrimp - state._trimpTotal).abs() > + 1e-9 * math.max(1, summedTrimp)) + throw const FormatException('Inconsistent minute checkpoint'); + // Older version-one checkpoints have all minute contributions and can + // reconstruct these totals. New checkpoints retain the unrounded sums. + for (final field in ['hrActiveTotal', 'walkingTotal']) { + if (!json.containsKey(field)) continue; + final saved = _number(json[field], finite: true); + final rebuilt = + field == 'hrActiveTotal' ? state._hrActiveTotal : state._walkingTotal; + if (saved < 0 || (saved - rebuilt).abs() > 1e-9 * math.max(1, rebuilt)) { + throw const FormatException('Inconsistent energy total'); + } + if (field == 'hrActiveTotal') { + state._hrActiveTotal = saved; + } else { + state._walkingTotal = saved; + } + } + return state; + } + int get processedMinutes => _processedMinutes; + + MinuteMetrics sync( + List minuteKeys, + List hr, { + List? cadenceSpm, + double? restingHr, + double? maxHr, + Sex sex = Sex.male, + WorkoutUserProfile? profile, + int dayMinutes = 1440, + double? quietHrr, + // As in [strainScoreMetric]. Defaults to calibrating: leaving it out can + // only understate confidence, never claim a settled level. + bool quietSettled = false, + bool force = false, + bool includeMinuteSeries = true, + }) { + if (minuteKeys.length != hr.length || + (cadenceSpm != null && cadenceSpm.length != hr.length)) + throw ArgumentError('Minute keys, HR and cadence must align'); + // A proven ordered prefix already has unique retained keys. Validate only + // new keys in that case; arbitrary edits use one set for validation and + // removal. Keep our own keys so caller mutations cannot poison this proof. + var keyPrefix = minuteKeys.length >= _keys.length; + final changed = []; + if (keyPrefix) { + for (var i = 0; i < _keys.length; i++) { + if (minuteKeys[i] != _keys[i]) { + keyPrefix = false; + break; + } + final old = _orderedBills[i]; + if (!_same(old.hr, hr[i]) || !_same(old.cadence, cadenceSpm?[i])) { + changed.add(i); + } + } + } + Set? retained; + if (keyPrefix) { + final added = minuteKeys.length - _keys.length > 1 ? {} : null; + for (var i = _keys.length; i < minuteKeys.length; i++) { + if (_bills.containsKey(minuteKeys[i]) || + (added != null && !added.add(minuteKeys[i]))) { + throw ArgumentError('Minute keys must be unique'); + } + changed.add(i); + } + } else { + retained = minuteKeys.toSet(); + if (retained.length != minuteKeys.length) { + throw ArgumentError('Minute keys must be unique'); + } + } + final parameters = [ + restingHr == null ? null : _encode(restingHr), + maxHr == null ? null : _encode(maxHr), + sex.name, + profile == null + ? null + : [ + _encode(profile.weightKg), + _encode(profile.heightCm), + _encode(profile.age), + profile.sex + ], + ]; + final reprice = force || !_equalState(_parameters, parameters); + final trimpValid = restingHr != null && + maxHr != null && + restingHr.isFinite && + maxHr.isFinite && + maxHr > restingHr; + if (reprice || !_pricerReady) { + _energyPricer = profile == null || restingHr == null || maxHr == null + ? null + : Calories.minuteEnergyPricer( + profile: profile, hrmax: maxHr, restingHr: restingHr); + _basalPerMinute = _energyPricer?.basalKcalPerMin; + _pricerReady = true; + } + if (reprice) { + _bills.clear(); + _trimpTotal = 0; + _hrActiveTotal = _walkingTotal = 0; + } + if (!keyPrefix && !reprice) { + for (final key in _bills.keys.toList()) { + if (!retained!.contains(key)) { + final old = _bills.remove(key)!; + _trimpTotal -= old.trimp; + _hrActiveTotal -= old.active; + _walkingTotal -= old.walking; + } + } + } + final workIndices = + reprice || !keyPrefix ? Iterable.generate(hr.length) : changed; + for (final i in workIndices) { + final key = minuteKeys[i], value = hr[i]; + final cadence = cadenceSpm?[i]; + final old = _bills[key]; + if (old != null && _same(old.hr, value) && _same(old.cadence, cadence)) + continue; + var trimp = 0.0; + if (trimpValid && value.isFinite && value > 0) { + final hrr = ((value - restingHr) / (maxHr - restingHr)).clamp(0.0, 1.0); + trimp = hrr * StrainScorer.banisterY(hrr, female: sex == Sex.female); + } + MinuteEnergySource? source; + var active = 0.0, walking = 0.0; + if (_energyPricer != null) { + final bill = _energyPricer!.price(value, cadence); + source = bill.source; + active = bill.active; + walking = bill.walking; + } + _trimpTotal += trimp - (old?.trimp ?? 0); + _hrActiveTotal += active - (old?.active ?? 0); + _walkingTotal += walking - (old?.walking ?? 0); + final bill = _MinuteBill(value, cadence, trimp, source, active, walking); + _bills[key] = bill; + if (keyPrefix && !reprice) { + if (i < _orderedBills.length) { + _orderedBills[i] = bill; + } else { + _orderedBills.add(bill); + } + } + _processedMinutes++; + } + if (keyPrefix) { + for (var i = _keys.length; i < minuteKeys.length; i++) { + _keys.add(minuteKeys[i]); + } + } else { + _keys + ..clear() + ..addAll(minuteKeys); + } + if (!keyPrefix || reprice) { + _orderedBills + ..clear() + ..addAll([for (final key in minuteKeys) _bills[key]!]); + } + _parameters = parameters; + // Rounding after removals can leave tiny negative residue in a zero day. + if (_trimpTotal < 0 && _trimpTotal > -1e-9) _trimpTotal = 0; + if (_hrActiveTotal < 0 && _hrActiveTotal > -1e-9) _hrActiveTotal = 0; + if (_walkingTotal < 0 && _walkingTotal > -1e-9) _walkingTotal = 0; + if (_bills.isEmpty) { + _trimpTotal = _hrActiveTotal = _walkingTotal = 0; + } + final trimp = !trimpValid || hr.isEmpty + ? banisterTrimp(const [], restingHr: restingHr, maxHr: maxHr, sex: sex) + : Metric( + value: _trimpTotal, + confidence: .6, + tier: Tier.estimate, + inputs_used: const ['hr_per_min', 'resting_hr', 'max_hr'], + note: 'Banister exponential TRIMP (wrist HR estimate)'); + final strain = strainScoreMetric(trimp.value, + wakeMinutes: hr.length.toDouble(), + quietHrr: quietHrr, + quietSettled: quietSettled, + female: sex == Sex.female); + final minutes = !includeMinuteSeries || _basalPerMinute == null + ? null + : Calories.assembleMinuteEnergySeries([ + for (final key in minuteKeys) + ( + minute: key, + source: _bills[key]!.source, + active: _bills[key]!.active, + walking: _bills[key]!.walking + ), + ], basalKcalPerMin: _basalPerMinute!); + final basal = (_basalPerMinute ?? 0) * dayMinutes; + final active = minutes?.active ?? (_hrActiveTotal + _walkingTotal); + final walking = minutes?.walking ?? _walkingTotal; + return MinuteMetrics( + trimp: trimp, + strain: strain, + minutes: minutes, + energy: _basalPerMinute == null + ? null + : ( + total: basal + active, + active: active, + basal: basal, + walking: walking + )); + } + + Map toJson() => { + 'version': 1, + 'type': 'IncrementalMinuteMetrics', + 'processedMinutes': _processedMinutes, + 'parameters': _parameters == null + ? null + : [ + _parameters![0], + _parameters![1], + _parameters![2], + _parameters![3] == null + ? null + : List.of(_parameters![3] as List), + ], + 'trimpTotal': _trimpTotal, + 'hrActiveTotal': _hrActiveTotal, + 'walkingTotal': _walkingTotal, + 'basalPerMinute': _basalPerMinute, + 'bills': [ + for (final entry in _bills.entries) + { + 'key': entry.key, + 'hr': _encode(entry.value.hr), + 'cadence': entry.value.cadence == null + ? null + : _encode(entry.value.cadence!), + 'trimp': entry.value.trimp, + 'source': entry.value.source?.name, + 'active': entry.value.active, + 'walking': entry.value.walking, + } + ], + }; +} + +bool _equalState(Object? a, Object? b) { + if (a is List && b is List) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!_equalState(a[i], b[i])) return false; + } + return true; + } + return a == b; +} diff --git a/lib/src/onehz/incremental_state.dart b/lib/src/onehz/incremental_state.dart new file mode 100644 index 00000000..45453fb3 --- /dev/null +++ b/lib/src/onehz/incremental_state.dart @@ -0,0 +1,50 @@ +part of 'incremental.dart'; + +void _version(Map json, String type) { + if (json['version'] is! int || json['version'] != 1 || json['type'] != type) { + throw FormatException('Unsupported $type checkpoint'); + } +} + +int _count(Object? value) { + if (value is! int || value < 0) throw const FormatException('Invalid count'); + return value; +} + +double _number(Object? value, {bool finite = false}) { + if (value is num && (!finite || value.isFinite)) return value.toDouble(); + if (!finite) { + if (value == 'NaN') return double.nan; + if (value == 'Infinity') return double.infinity; + if (value == '-Infinity') return double.negativeInfinity; + } + throw const FormatException('Invalid number'); +} + +Object _encode(double value) => value.isFinite + ? value + : value.isNaN + ? 'NaN' + : value > 0 + ? 'Infinity' + : '-Infinity'; +List _numbers(Object? value, {bool finite = false}) { + if (value is! List) throw const FormatException('Expected numeric list'); + return [for (final item in value) _number(item, finite: finite)]; +} + +Map _map(Object? value) { + if (value is! Map) + throw const FormatException('Expected map'); + return value; +} + +bool _same(double? a, double? b) => + a == b || (a != null && b != null && a.isNaN && b.isNaN); +bool _prefix(List previous, List next) { + if (next.length < previous.length) return false; + for (var i = 0; i < previous.length; i++) { + if (!_same(previous[i], next[i])) return false; + } + return true; +} diff --git a/lib/src/onehz/registry/group_registry.dart b/lib/src/onehz/registry/group_registry.dart new file mode 100644 index 00000000..97c89073 --- /dev/null +++ b/lib/src/onehz/registry/group_registry.dart @@ -0,0 +1,417 @@ +// group_registry.dart — the calculation-group registry (design 06 phase 1b). +// +// Types moved verbatim from edge's lib/compute/groups/group_types.dart (same +// API). Pure Dart, const-constructible. test/registry_closure_test.dart pins +// the declared groups and their source closures. + +/// What a group is. Foundations carry no cost rule; edge transforms are the +/// gating/assembly steps edge runs around analytics results. +enum GroupKind { algorithm, foundation, edgeTransform } + +/// How far in time a declared input reaches. +enum InputScope { day, dayAndNext, range, global } + +class InputDecl { + /// An id from edge's input catalogue. + final String id; + final InputScope scope; + const InputDecl(this.id, this.scope); +} + +class GroupDecl { + /// Stable; never reused after a split or merge. + final String id; + final GroupKind kind; + + /// Plain integer, bumped when the group's algorithm changes, including + /// output rounding or abstention changes. Dependency-only changes propagate + /// through [deps]; they do not require another bump of this counter. + final int counter; + + /// Exported function names, at most 3. + final List members; + + /// Exported functions whose output depends on this group, beyond the + /// cost-bearing [members]. Uncapped (design 06 section 13); the cost rule + /// and the member cap look at [members] only. Each function is owned by + /// exactly one group, as a member or an api entry. + final List api; + + /// Group ids this group reads, foundations included. + final List deps; + final List inputs; + + /// Repo-relative paths (lib/src/...) this group's output depends on. + final List files; + + const GroupDecl({ + required this.id, + required this.kind, + required this.counter, + required this.members, + this.api = const [], + this.deps = const [], + this.inputs = const [], + this.files = const [], + }); +} + +class GroupRegistry { + final int registryVersion; + final List groups; + const GroupRegistry(this.registryVersion, this.groups); +} + +/// The groups analytics declares (registry schema version 1). +/// +/// `members` are the group's cost-bearing entry functions (at most 3); `api` +/// lists the other exported functions whose output depends on the group, so +/// every function edge calls has exactly one owner. `files` is the import +/// closure of members and api, minus the files of the foundations the group +/// depends on. Names only in lib/aggregate.dart (`prsaAggregator`) count as +/// public. test/registry_closure_test.dart re-derives all of this from source. +const GroupRegistry kAnalyticsGroupRegistry = GroupRegistry(1, [ + GroupDecl( + id: 'found.core', + kind: GroupKind.foundation, + counter: 2, + members: ['median'], + api: ['needBaselineNote'], + files: ['lib/src/onehz/types.dart', 'lib/src/onehz/util.dart'], + ), + GroupDecl( + id: 'found.rr_correction', + kind: GroupKind.foundation, + counter: 2, + members: ['correctRr'], + deps: ['found.core'], + files: [ + 'lib/src/onehz/foundations/rr_correction.dart', + 'lib/src/onehz/foundations/rr_correction_kernel.dart', + ], + ), + GroupDecl( + id: 'found.baseline_ewma', + kind: GroupKind.foundation, + counter: 1, + members: ['Baselines.foldHistory', 'Baselines.deviation'], + api: ['mdc'], + deps: ['found.core'], + files: [ + 'lib/src/onehz/foundations/baseline.dart', + 'lib/src/onehz/foundations/ewma_baselines.dart', + ], + ), + GroupDecl( + id: 'found.calibration', + kind: GroupKind.foundation, + counter: 1, + members: ['calibrationFor'], + api: ['unknownFamilyNote'], + files: ['lib/src/onehz/device.dart'], + ), + GroupDecl( + id: 'found.energy', + kind: GroupKind.foundation, + counter: 1, + members: ['Calories.dailyEnergy'], + api: ['Calories.minuteEnergy', 'Calories.estimateBoutCalories'], + files: ['lib/src/onehz/workout/calories.dart'], + ), + GroupDecl( + id: 'found.staging_helpers', + kind: GroupKind.foundation, + counter: 1, + members: ['consolidateSleepStages'], + deps: ['found.core'], + files: [ + 'lib/src/onehz/sleep/accounting.dart', + 'lib/src/onehz/sleep/stager.dart', + ], + ), + GroupDecl( + id: 'sleep.stage', + kind: GroupKind.algorithm, + counter: 2, + members: ['segmentSleep', 'stageMinuteRanges', 'cardioStager'], + api: [ + 'resetCardioObservations', + 'takeCardioObservations', + 'detectSleepCycles', + 'sleepCyclesMetric', + 'hrLedSleepWindow', + 'zAngle', + 'habitualMidsleepSecFromHistory', + 'phillipsSri', + 'sleepDebt', + 'sleepNeed', + 'sleepPerformance', + 'recommendedBedtime', + 'recommendedWake', + ], + deps: [ + 'found.core', + 'found.rr_correction', + 'found.staging_helpers', + 'found.calibration', + ], + files: [ + 'lib/src/aggregate/aggregator.dart', + 'lib/src/aggregate/json_support.dart', + 'lib/src/aggregate/primitives.dart', + 'lib/src/onehz/clinical/hrv_jitter_evidence.dart', + 'lib/src/onehz/clinical/hrv_time.dart', + 'lib/src/onehz/clinical/hrv_time_aggregator.dart', + 'lib/src/onehz/clinical/rmssd_gate.dart', + 'lib/src/onehz/human/coaching.dart', + 'lib/src/onehz/human/sleep_regularity.dart', + 'lib/src/onehz/incremental_core.dart', + 'lib/src/onehz/sleep/advanced_stager.dart', + 'lib/src/onehz/sleep/band_offset.dart', + 'lib/src/onehz/sleep/cardio_row_cache.dart', + 'lib/src/onehz/sleep/cardio_stager.dart', + 'lib/src/onehz/sleep/cycles.dart', + 'lib/src/onehz/sleep/epoch_rows.dart', + 'lib/src/onehz/sleep/hr_fallback.dart', + 'lib/src/onehz/sleep/hypnogram.dart', + 'lib/src/onehz/sleep/hypnogram_aggregator.dart', + 'lib/src/onehz/sleep/segment.dart', + 'lib/src/onehz/sleep/sri.dart', + 'lib/src/onehz/sleep/sri_aggregator.dart', + 'lib/src/onehz/sleep/stage_ranges.dart', + 'lib/src/onehz/sleep/van_hees.dart', + ], + ), + GroupDecl( + id: 'hrv.freq', + kind: GroupKind.algorithm, + counter: 2, + members: ['hrvFreq'], + api: [ + 'nightHrvShape', + 'hrvTime', + 'nocturnalRmssd', + 'rrCoverage', + 'sleepSessionRmssdDetail', + ], + deps: ['found.core', 'found.rr_correction'], + files: [ + 'lib/src/aggregate/aggregator.dart', + 'lib/src/aggregate/json_support.dart', + 'lib/src/aggregate/primitives.dart', + 'lib/src/onehz/clinical/hrv_freq.dart', + 'lib/src/onehz/clinical/hrv_freq_stream.dart', + 'lib/src/onehz/clinical/hrv_jitter_evidence.dart', + 'lib/src/onehz/clinical/hrv_time.dart', + 'lib/src/onehz/clinical/hrv_time_aggregator.dart', + 'lib/src/onehz/clinical/rmssd_gate.dart', + 'lib/src/onehz/incremental_core.dart', + 'lib/src/onehz/sleep/night_hrv_shape.dart', + ], + ), + GroupDecl( + id: 'night.rhythm', + kind: GroupKind.algorithm, + counter: 2, + members: ['rsaRespRate', 'cvhrApneaScreen', 'decelerationCapacity'], + api: [ + 'accelerationCapacity', + 'breathingRateVariability', + 'irregularBeatScreenDetailed', + 'prsaAggregator', + 'cosinor', + 'circadianNonparametric', + 'chronotype', + 'socialJetlag', + ], + deps: ['found.core', 'found.rr_correction'], + files: [ + 'lib/src/aggregate/aggregator.dart', + 'lib/src/aggregate/json_support.dart', + 'lib/src/aggregate/primitives.dart', + 'lib/src/onehz/clinical/cardiac_coherence.dart', + 'lib/src/onehz/clinical/cosinor.dart', + 'lib/src/onehz/clinical/cosinor_aggregator.dart', + 'lib/src/onehz/clinical/irregular_diagnostics.dart', + 'lib/src/onehz/clinical/irregular_rhythm.dart', + 'lib/src/onehz/clinical/irregular_window.dart', + 'lib/src/onehz/clinical/prsa.dart', + 'lib/src/onehz/clinical/prsa_aggregator.dart', + 'lib/src/onehz/foundations/baseline.dart', + 'lib/src/onehz/foundations/fusion.dart', + 'lib/src/onehz/human/circadian_lifestyle.dart', + 'lib/src/onehz/human/percentile_of_you.dart', + 'lib/src/onehz/incremental_core.dart', + 'lib/src/onehz/respiration/brv_trend.dart', + 'lib/src/onehz/respiration/cvhr_apnea.dart', + 'lib/src/onehz/respiration/relative_odi.dart', + 'lib/src/onehz/respiration/resp_rate.dart', + 'lib/src/onehz/sleep/circadian_np.dart', + 'lib/src/onehz/sleep/circadian_np_aggregator.dart', + 'lib/src/onehz/sleep/circadian_window.dart', + ], + ), + GroupDecl( + id: 'day.blocks', + kind: GroupKind.algorithm, + counter: 1, + members: ['detectNaps', 'autoDetectWorkouts', 'positionSeries'], + api: ['hrRecovery', 'AutoWorkoutDetector.motionPoints'], + deps: ['found.core', 'sleep.stage', 'found.calibration'], + files: [ + 'lib/src/aggregate/aggregator.dart', + 'lib/src/aggregate/json_support.dart', + 'lib/src/aggregate/primitives.dart', + 'lib/src/onehz/clinical/hrv_jitter_evidence.dart', + 'lib/src/onehz/clinical/hrv_time.dart', + 'lib/src/onehz/clinical/hrv_time_aggregator.dart', + 'lib/src/onehz/clinical/rmssd_gate.dart', + 'lib/src/onehz/incremental_core.dart', + 'lib/src/onehz/sleep/advanced_stager.dart', + 'lib/src/onehz/sleep/band_offset.dart', + 'lib/src/onehz/sleep/cardio_row_cache.dart', + 'lib/src/onehz/sleep/cardio_stager.dart', + 'lib/src/onehz/sleep/hypnogram.dart', + 'lib/src/onehz/sleep/hypnogram_aggregator.dart', + 'lib/src/onehz/motion/orientation.dart', + 'lib/src/onehz/sleep/nap.dart', + 'lib/src/onehz/sleep/van_hees.dart', + 'lib/src/onehz/workout/auto_detect.dart', + 'lib/src/onehz/workout/hr_recovery.dart', + 'lib/src/onehz/workout/sport.dart', + ], + ), + GroupDecl( + id: 'readiness.stress_illness', + kind: GroupKind.algorithm, + counter: 1, + members: ['readinessComposite', 'baevskyStressIndex', 'illnessCusum'], + api: [ + 'readinessLnRmssd', + 'glassBoxReadiness', + 'hrvInput', + 'respInput', + 'rhrInput', + 'tempInput', + 'nocturnalRhr', + 'cusumChangePoints', + 'multivariateAnomaly', + 'overreachingConjunction', + 'tempIllnessFlag', + ], + deps: [ + 'found.core', + 'night.rhythm', + 'hrv.freq', + 'found.rr_correction', + 'found.baseline_ewma', + ], + files: [ + 'lib/src/aggregate/aggregator.dart', + 'lib/src/aggregate/json_support.dart', + 'lib/src/aggregate/primitives.dart', + 'lib/src/onehz/clinical/cosinor.dart', + 'lib/src/onehz/clinical/cosinor_aggregator.dart', + 'lib/src/onehz/clinical/illness_cusum.dart', + 'lib/src/onehz/clinical/load_trimp.dart', + 'lib/src/onehz/clinical/load_trimp_aggregator.dart', + 'lib/src/onehz/clinical/nocturnal.dart', + 'lib/src/onehz/clinical/readiness_lnrmssd.dart', + 'lib/src/onehz/clinical/stress_si.dart', + 'lib/src/onehz/device.dart', + 'lib/src/onehz/human/overreaching_conjunction.dart', + 'lib/src/onehz/human/readiness_glassbox.dart', + 'lib/src/onehz/incremental_core.dart', + 'lib/src/onehz/wellness/anomaly.dart', + 'lib/src/onehz/wellness/changepoint.dart', + 'lib/src/onehz/wellness/readiness_composite.dart', + 'lib/src/onehz/wellness/temp_circadian.dart', + 'lib/src/onehz/wellness/temp_health.dart', + ], + ), + GroupDecl( + id: 'activity.load', + kind: GroupKind.algorithm, + counter: 2, + members: ['banisterTrimp', 'dailyActiveMinutes', 'strainScore'], + api: [ + 'strainFromNetTrimp', + 'strainScoreFromSeries', + 'strainCurveFromSeries', + 'strainTarget', + 'ctlAtlTsb', + 'netTrimpAboveQuiet', + 'dailyQuietWakingHrr', + 'personalQuietWakingHrr', + 'dailyDynSummary', + 'personalDynFloorFromDailySummaries', + 'shouldRefreezeFloor', + 'calibrateGRef', + 'enmoSeries', + 'sessionHrCeiling', + 'HeartRateZones.timeInZone', + 'HeartRateZones.zonesFromMaxHr', + 'HeartRateZones.reserveZones', + ], + deps: [ + 'found.core', + 'found.energy', + 'found.calibration', + 'found.baseline_ewma', + ], + files: [ + 'lib/src/aggregate/aggregator.dart', + 'lib/src/aggregate/json_support.dart', + 'lib/src/aggregate/primitives.dart', + 'lib/src/onehz/clinical/load_trimp.dart', + 'lib/src/onehz/clinical/load_trimp_aggregator.dart', + 'lib/src/onehz/clinical/vo2max.dart', + 'lib/src/onehz/human/coaching.dart', + 'lib/src/onehz/incremental_core.dart', + 'lib/src/onehz/motion/energy_fusion.dart', + 'lib/src/onehz/motion/enmo.dart', + 'lib/src/onehz/motion/steps.dart', + 'lib/src/onehz/workout/hr_zones.dart', + 'lib/src/onehz/workout/observed_max_hr.dart', + 'lib/src/onehz/workout/time_in_zone_aggregator.dart', + ], + ), + GroupDecl( + id: 'body.spo2_temp_coverage', + kind: GroupKind.algorithm, + counter: 1, + members: ['hrDip', 'nightlySkinTemp'], + api: ['hrDipFromDayTotals'], + deps: ['found.core', 'found.baseline_ewma', 'found.calibration'], + files: [ + 'lib/src/aggregate/aggregator.dart', + 'lib/src/aggregate/json_support.dart', + 'lib/src/onehz/clinical/cosinor.dart', + 'lib/src/onehz/clinical/cosinor_aggregator.dart', + 'lib/src/onehz/clinical/nocturnal.dart', + 'lib/src/onehz/device.dart', + 'lib/src/onehz/wellness/temp_circadian.dart', + ], + ), + GroupDecl( + id: 'human.insights', + kind: GroupKind.algorithm, + counter: 1, + members: ['alertnessForecast', 'scanAssociations', 'alcoholNightFlag'], + api: [ + 'roughNight', + 'sessionMorningEffects', + 'weekdayEffect', + 'cycleLengthSeries', + ], + deps: ['found.core', 'found.baseline_ewma'], + files: [ + 'lib/src/onehz/human/alertness_forecast.dart', + 'lib/src/onehz/human/associations.dart', + 'lib/src/onehz/human/event_detection.dart', + 'lib/src/onehz/human/session_cost.dart', + 'lib/src/onehz/human/weekday_effect.dart', + 'lib/src/onehz/wellness/cycle_lengths.dart', + ], + ), +]); diff --git a/lib/src/onehz/workout/calories.dart b/lib/src/onehz/workout/calories.dart index 2edf051b..a4926d07 100644 --- a/lib/src/onehz/workout/calories.dart +++ b/lib/src/onehz/workout/calories.dart @@ -326,41 +326,26 @@ class Calories { 'cadenceSpmPerMin (${cadenceSpmPerMin.length}) must align with ' 'hrPerMin (${hrPerMin.length}): one entry per wake minute'); } - // (weight/height/age still can't be told apart from their defaults here: - // WorkoutUserProfile's own constructor bakes 70/170/30 in at construction, - // so by the time a profile object arrives there is no way left to tell "the - // caller passed 70kg" from "nobody set a weight". Making those three - // nullable at the source is the real fix and a bigger change.) - final weightKg = profile.weightKg > 0 ? profile.weightKg : 70.0; - final heightCm = profile.heightCm > 0 ? profile.heightCm : 170.0; - final age = profile.age > 0 ? profile.age : 30.0; - final coeffs = resolveCoeffs(profile.sex); - final flexHr = activeGateHr(hrmax, restingHr); - if (flexHr == null) return null; - - final bmrDay = mifflinBmrKcalDay(weightKg, heightCm, age, profile.sex); - final basalPerMin = bmrDay / 1440.0; + // The prep and the per-minute billing are SHARED with [minuteEnergy] + // ([_energyCtx], [_bill]) — one definition of the computation, so the + // per-minute series and this total cannot drift. (weight/height/age still + // can't be told apart from their defaults there: WorkoutUserProfile's own + // constructor bakes 70/170/30 in at construction, so by the time a profile + // object arrives there is no way left to tell "the caller passed 70kg" from + // "nobody set a weight". Making those three nullable at the source is the + // real fix and a bigger change.) + final ctx = _energyCtx(profile, hrmax, restingHr); + if (ctx == null) return null; + final basalPerMin = ctx.basalPerMin; var active = 0.0; var walking = 0.0; for (var i = 0; i < hrPerMin.length; i++) { - final hr = hrPerMin[i]; - // `hr < flexHr` is false for NaN, so an unfiltered non-finite minute - // would bill active and carry its NaN into the day total. - if (hr.isFinite && hr >= flexHr) { - final activePerMin = - activeKcalPerS(coeffs, hr, hrmax, weightKg, age) * 60.0; - final surplus = activePerMin - basalPerMin; - if (surplus > 0) active += surplus; - continue; // HR billed the minute — cadence never doubles it. - } - // Below flex (or no HR at all — measured gait stands on its own): - // a measured walking cadence prices the minute the HR gate refused. - final cad = cadenceSpmPerMin?[i]; - if (cad == null) continue; - final met = metFromCadenceSpm(cad); - if (met == null) continue; - walking += (met - 1.0) * basalPerMin; + final b = _bill(ctx, hrPerMin[i], cadenceSpmPerMin?[i]); + // Adding the +0.0 of a minute that billed nothing is exact, so this is + // the same sum, in the same order, as skipping those minutes. + active += b.active; + walking += b.walking; } active += walking; final basal = basalPerMin * dayMinutes; @@ -372,6 +357,271 @@ class Calories { ); } + // --------------------------------------------------------------------------- + // Shared by [dailyEnergy] and [minuteEnergy]: the ONE definition of how a + // minute is billed. Nothing here is new arithmetic — it is the body that used + // to sit inside dailyEnergy's loop, moved so the two entry points cannot + // diverge. dailyEnergy's outputs are bit-for-bit what they were (golden + // tests in test/onehz/minute_energy_test.dart). + // --------------------------------------------------------------------------- + + static ({ + CalorieCoeffs coeffs, + double weightKg, + double age, + double hrmax, + double flexHr, + double basalPerMin, + })? _energyCtx(WorkoutUserProfile profile, double hrmax, double restingHr) { + final weightKg = profile.weightKg > 0 ? profile.weightKg : 70.0; + final heightCm = profile.heightCm > 0 ? profile.heightCm : 170.0; + final age = profile.age > 0 ? profile.age : 30.0; + final flexHr = activeGateHr(hrmax, restingHr); + if (flexHr == null) return null; + return ( + coeffs: resolveCoeffs(profile.sex), + weightKg: weightKg, + age: age, + hrmax: hrmax, + flexHr: flexHr, + basalPerMin: mifflinBmrKcalDay(weightKg, heightCm, age, profile.sex) / + 1440.0, + ); + } + + /// One minute's bill. `source` is null when the minute has nothing measured + /// (no usable HR and no billable cadence). `active` is the HR surplus over + /// basal (≥ 0) and `walking` the cadence surplus; a minute is only ever one + /// of the two. + static ({MinuteEnergySource? source, double active, double walking}) _bill( + ({ + CalorieCoeffs coeffs, + double weightKg, + double age, + double hrmax, + double flexHr, + double basalPerMin, + }) ctx, + double hr, + double? cadenceSpm, + ) { + // `hr < flexHr` is false for NaN, so an unfiltered non-finite minute + // would bill active and carry its NaN into the day total. + if (hr.isFinite && hr >= ctx.flexHr) { + final activePerMin = activeKcalPerS( + ctx.coeffs, hr, ctx.hrmax, ctx.weightKg, ctx.age) * + 60.0; + final surplus = activePerMin - ctx.basalPerMin; + // HR billed the minute — cadence never doubles it. + return ( + source: MinuteEnergySource.hr, + active: surplus > 0 ? surplus : 0.0, + walking: 0.0, + ); + } + // Below flex (or no HR at all — measured gait stands on its own): + // a measured walking cadence prices the minute the HR gate refused. + final met = cadenceSpm == null ? null : metFromCadenceSpm(cadenceSpm); + if (met != null) { + return ( + source: MinuteEnergySource.cadence, + active: 0.0, + walking: (met - 1.0) * ctx.basalPerMin, + ); + } + return ( + source: (hr.isFinite && hr > 0) ? MinuteEnergySource.rest : null, + active: 0.0, + walking: 0.0, + ); + } + + /// INTRADAY ENERGY: the computation [dailyEnergy] does, one record per + /// minute. Same inputs, same gate ([activeGateHr]), same billing — so the + /// series folds back to the daily figures EXACTLY: + /// + /// `series.active` == `dailyEnergy(...).active` (`==`, not "close") + /// `series.walking` == `dailyEnergy(...).walking` + /// `series.basalKcalPerMin * dayMinutes` == `dailyEnergy(...).basal` + /// + /// [hrPerMin] is index-aligned per-minute mean HR (bpm). Unlike [dailyEnergy] + /// the caller should pass the series WITH its gaps in place (0 / NaN for a + /// minute with no HR) rather than compacted, so each record keeps its place + /// on the clock. [cadenceSpmPerMin] is aligned the same way (see + /// [dailyEnergy]). [epochMinutes], when given, is each entry's epoch minute + /// (`tsSec ~/ 60`) and becomes [MinuteEnergy.minute]; otherwise `minute` is + /// the index. Both must match [hrPerMin]'s length or this throws. + /// + /// ABSTENTION, per minute, never filled or interpolated: + /// * HR usable (finite, > 0) → the minute is measured. At/above the gate + /// it bills the Keytel surplus ([MinuteEnergySource.hr]); below it, it + /// is measured rest — basal only, `active` 0 ([MinuteEnergySource.rest]). + /// * HR unusable but a billable cadence ([metFromCadenceSpm]) → walking + /// surplus ([MinuteEnergySource.cadence]), exactly as [dailyEnergy] does. + /// * Otherwise the record abstains: `abstained` is + /// [MinuteEnergyAbstain.noHr] and basal/active/walking/total are all null. + /// A minute's record depends on that minute's inputs alone. + /// + /// Returns null when the anchors cannot define a gate, like [dailyEnergy]. + /// + /// The series covers exactly the minutes it is given. [dailyEnergy]'s basal + /// floor is pro-rated over `dayMinutes` (the whole covered day, sleep + /// included) and is not a per-minute sum; use + /// [MinuteEnergySeries.basalKcalPerMin] for that rate. + static MinuteEnergySeries? minuteEnergy( + List hrPerMin, { + required WorkoutUserProfile profile, + required double hrmax, + required double restingHr, + List? cadenceSpmPerMin, + List? epochMinutes, + }) { + if (cadenceSpmPerMin != null && + cadenceSpmPerMin.length != hrPerMin.length) { + throw ArgumentError( + 'cadenceSpmPerMin (${cadenceSpmPerMin.length}) must align with ' + 'hrPerMin (${hrPerMin.length}): one entry per minute'); + } + if (epochMinutes != null && epochMinutes.length != hrPerMin.length) { + throw ArgumentError( + 'epochMinutes (${epochMinutes.length}) must align with ' + 'hrPerMin (${hrPerMin.length}): one entry per minute'); + } + final ctx = _energyCtx(profile, hrmax, restingHr); + if (ctx == null) return null; + final basal = ctx.basalPerMin; + + final minutes = []; + // The same two accumulators, in the same order, as dailyEnergy. + var hrActive = 0.0; + var walking = 0.0; + var covered = 0; + for (var i = 0; i < hrPerMin.length; i++) { + final b = _bill(ctx, hrPerMin[i], cadenceSpmPerMin?[i]); + final minute = epochMinutes?[i] ?? i; + final src = b.source; + if (src == null) { + minutes.add(MinuteEnergy._abstained(minute)); + continue; + } + covered++; + hrActive += b.active; + walking += b.walking; + final cad = src == MinuteEnergySource.cadence; + final active = cad ? b.walking : b.active; + minutes.add(MinuteEnergy._( + minute: minute, + source: src, + basal: basal, + active: active, + walking: cad ? b.walking : null, + total: basal + active, + )); + } + return MinuteEnergySeries._( + minutes: minutes, + basalKcalPerMin: basal, + active: hrActive + walking, + walking: walking, + coveredMinutes: covered, + ); + } + + /// Prepares fixed profile and anchor math once for independent minute bills. + /// Returns null under the same anchor gate as [dailyEnergy]. + // Keytel et al. 2005 active energy, Mifflin-St Jeor 1990 basal energy and Tudor- + // Locke et al. 2019 cadence METs, as in dailyEnergy(). Shared minute bills + // preserve bill order and separate HR/walking accumulation; hourly output sums + // covered records without extrapolation. + static MinuteEnergyPricer? minuteEnergyPricer({ + required WorkoutUserProfile profile, + required double hrmax, + required double restingHr, + }) { + final ctx = _energyCtx(profile, hrmax, restingHr); + if (ctx == null) return null; + return MinuteEnergyPricer._( + ctx.basalPerMin, (hr, cadence) => _bill(ctx, hr, cadence)); + } + + /// Assembles previously priced minute bills without evaluating HR or cadence. + /// Intended for keyed incremental callers that retain the exact bills and + /// replace only changed minutes. Walking is already included in active. + // Keytel et al. 2005 active energy, Mifflin-St Jeor 1990 basal energy and Tudor- + // Locke et al. 2019 cadence METs, as in dailyEnergy(). Shared minute bills + // preserve bill order and separate HR/walking accumulation; hourly output sums + // covered records without extrapolation. + static MinuteEnergySeries assembleMinuteEnergySeries( + List< + ({ + int minute, + MinuteEnergySource? source, + double active, + double walking + })> + bills, { + required double basalKcalPerMin, + }) { + final minutes = []; + var hrActive = 0.0, walking = 0.0; + var covered = 0; + for (final bill in bills) { + final source = bill.source; + if (source == null) { + minutes.add(MinuteEnergy._abstained(bill.minute)); + continue; + } + covered++; + final cadence = source == MinuteEnergySource.cadence; + if (cadence) { + walking += bill.walking; + } else { + hrActive += bill.active; + } + final active = cadence ? bill.walking : bill.active; + minutes.add(MinuteEnergy._( + minute: bill.minute, + source: source, + basal: basalKcalPerMin, + active: active, + walking: cadence ? bill.walking : null, + total: basalKcalPerMin + active, + )); + } + return MinuteEnergySeries._( + minutes: minutes, + basalKcalPerMin: basalKcalPerMin, + active: hrActive + walking, + walking: walking, + coveredMinutes: covered, + ); + } + + /// Sums [minutes] into hourly buckets, ascending, for hours that have at + /// least one record. Only COVERED minutes are summed and nothing is scaled up + /// for the rest: an hour with 30 covered minutes reports 30 minutes of + /// energy and `coverage` 0.5, and an hour with none reports null sums, not + /// zero. The bucket is `floor((minute + minuteOffset) / 60)`: with index + /// minutes that is "hours since the series start"; with epoch minutes pass + /// the local UTC offset in minutes (e.g. 330) to bucket by local hour. + // Keytel et al. 2005 active energy, Mifflin-St Jeor 1990 basal energy and Tudor- + // Locke et al. 2019 cadence METs, as in dailyEnergy(). Shared minute bills + // preserve bill order and separate HR/walking accumulation; hourly output sums + // covered records without extrapolation. + static List hourlyRollup( + List minutes, { + int minuteOffset = 0, + }) { + final byHour = >{}; + for (final m in minutes) { + (byHour[(m.minute + minuteOffset) ~/ 60] ??= []).add(m); + } + final hours = byHour.keys.toList()..sort(); + return [ + for (final h in hours) HourlyEnergy._from(h, byHour[h]!), + ]; + } + /// Estimate (kcal, kJ) for a workout bout. Each sample is weighted by the /// ELAPSED time to the next sample (capped at [mergeGapCapS], which defaults /// to [defaultMergeGapCapS] = 150 s), so a sparse stream is counted over real @@ -445,3 +695,159 @@ class Calories { ); } } + +/// Reusable fixed-profile calorie pricing, with no minute-record allocation. +class MinuteEnergyPricer { + final double basalKcalPerMin; + final ({MinuteEnergySource? source, double active, double walking}) Function( + double, double?) _price; + MinuteEnergyPricer._(this.basalKcalPerMin, this._price); + + ({MinuteEnergySource? source, double active, double walking}) price( + double hr, double? cadenceSpm) => + _price(hr, cadenceSpm); +} + +/// Why a minute carries no energy. +enum MinuteEnergyAbstain { + /// No usable HR (0, negative, NaN, infinite) and no billable cadence. + noHr, +} + +/// What a non-abstained minute was billed from. +enum MinuteEnergySource { + /// HR at/above the gate: Keytel surplus over basal. + hr, + + /// HR unusable or below the gate, but a measured walking cadence at/above + /// the CADENCE-Adults floor: walking surplus. + cadence, + + /// Measured HR below the gate: resting, basal only. + rest, +} + +/// One minute of [Calories.minuteEnergy]. kcal for THAT minute. Either +/// [abstained] is set and every number is null, or [source] is set and +/// basal/active/total are non-null. +class MinuteEnergy { + /// Index into the input, or the epoch minute when the caller supplied them. + final int minute; + final MinuteEnergySource? source; + final MinuteEnergyAbstain? abstained; + + /// Mifflin–St Jeor BMR for one minute. + final double? basal; + + /// Surplus over basal: the Keytel HR surplus, or the walking surplus for a + /// [MinuteEnergySource.cadence] minute. 0 for [MinuteEnergySource.rest]. + final double? active; + + /// The cadence surplus, already inside [active]. Null unless + /// [source] is [MinuteEnergySource.cadence]. + final double? walking; + + /// `basal + active`. + final double? total; + + const MinuteEnergy._({ + required this.minute, + required MinuteEnergySource this.source, + required double this.basal, + required double this.active, + required this.walking, + required double this.total, + }) : abstained = null; + + const MinuteEnergy._abstained(this.minute) + : source = null, + abstained = MinuteEnergyAbstain.noHr, + basal = null, + active = null, + walking = null, + total = null; +} + +/// Result of [Calories.minuteEnergy]. +class MinuteEnergySeries { + /// One record per input minute, in input order. + final List minutes; + + /// Basal kcal per minute (BMR ÷ 1440). `dailyEnergy(...).basal` is this × + /// `dayMinutes`. + final double basalKcalPerMin; + + /// Σ active over the series, folded exactly as [Calories.dailyEnergy] folds + /// it: `==` its `active`, walking surplus included. + final double active; + + /// Σ walking surplus; `==` `dailyEnergy(...).walking`. + final double walking; + + /// Minutes that did not abstain. + final int coveredMinutes; + + const MinuteEnergySeries._({ + required this.minutes, + required this.basalKcalPerMin, + required this.active, + required this.walking, + required this.coveredMinutes, + }); + + int get abstainedMinutes => minutes.length - coveredMinutes; +} + +/// One hour of [Calories.hourlyRollup]: sums over the hour's covered minutes. +class HourlyEnergy { + /// `floor((minute + minuteOffset) / 60)`. + final int hour; + + /// Covered minutes in this hour (0–60). + final int coveredMinutes; + + /// `coveredMinutes / 60`. + double get coverage => coveredMinutes / 60.0; + + /// Null when the hour has no covered minute. + final double? basal; + final double? active; + final double? total; + + /// Null when no minute of the hour was billed from cadence. + final double? walking; + + const HourlyEnergy._({ + required this.hour, + required this.coveredMinutes, + required this.basal, + required this.active, + required this.walking, + required this.total, + }); + + factory HourlyEnergy._from(int hour, List ms) { + var covered = 0; + var basal = 0.0, active = 0.0, total = 0.0, walking = 0.0; + var anyWalking = false; + for (final m in ms) { + if (m.abstained != null) continue; + covered++; + basal += m.basal!; + active += m.active!; + total += m.total!; + if (m.walking != null) { + walking += m.walking!; + anyWalking = true; + } + } + return HourlyEnergy._( + hour: hour, + coveredMinutes: covered, + basal: covered == 0 ? null : basal, + active: covered == 0 ? null : active, + total: covered == 0 ? null : total, + walking: anyWalking ? walking : null, + ); + } +} diff --git a/test/aggregate/adopted_aggregators_test.dart b/test/aggregate/adopted_aggregators_test.dart new file mode 100644 index 00000000..803a4e41 --- /dev/null +++ b/test/aggregate/adopted_aggregators_test.dart @@ -0,0 +1,228 @@ +// The public batch function of each toleranced calculation IS its aggregator +// run over the whole input in one slice: empty -> addSlice(all) -> finish. +// So the batch result must be BIT-IDENTICAL to that one-slice fold (not merely +// within the chunked-fold tolerance, which the per-calculation *_aggregator +// tests and the laws harness already check). +// +// Hypnogram metrics and SRI are already wired this way. This file covers the +// rest: HRV time domain, Banister TRIMP, Cosinor, time in zone, CTL/ATL/TSB, +// PRSA DC/AC, circadian IS/IV/M10/L5/RA, readiness lnRMSSD. +// +// Readiness already shares readinessLnRmssdFromSuffix with its aggregator, so +// its test is expected to pass today (the no-dates path). + +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import '../support/hrv_beats.dart'; +import '../support/property.dart' show Rng; + +final _bd = ByteData(8); +int _bits(double d) { + _bd.setFloat64(0, d); + return _bd.getInt64(0); +} + +/// Equal as bit patterns (NaN == NaN, 0.0 != -0.0); other values by ==. +bool _same(Object? a, Object? b) { + if (a is double && b is double) return _bits(a) == _bits(b); + if (a is List && b is List) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!_same(a[i], b[i])) return false; + } + return true; + } + return a == b; +} + +List _meta(Metric m) => + [m.present, m.confidence, m.tier, m.inputs_used.join(','), m.note]; + +void _bitIdentical(List got, List want, String tag) { + expect(_same(got, want), isTrue, + reason: '$tag: batch is not bit-identical to the one-slice aggregator ' + 'run\n batch $got\n aggregator $want'); +} + +DoubleColumn _col(List x) => DoubleColumn(Float64List.fromList(x)); + +void main() { + test('HRV time domain: hrvTime == hrvTimeAggregator one slice', () { + for (final (mode, seed) in [(0, 11), (1, 12), (2, 13), (3, 14)]) { + final beats = hrvSeries(Rng(seed), 4000, mode); + final nn = beats.nnMs.toList(); + final times = beats.timeMs.toList(); + final batch = hrvTime(nn, nnTimesMs: times); + final agg = hrvTimeAggregator(originMs: beats.timeMs[0]).run(beats); + final b = batch.value ?? HrvTime(nBeats: beats.length); + _bitIdentical( + [...hrvFields(b), b.nBeats], [...hrvFields(agg), agg.nBeats], + 'hrv mode $mode'); + } + }); + + test('Banister TRIMP: banisterTrimp == banisterTrimpAggregator one slice', + () { + final r = Rng(21); + final hr = [ + for (var i = 0; i < 1500; i++) + r.nextBool(.05) ? double.nan : 45 + r.nextDouble() * 150 + ]; + for (final sex in [Sex.male, Sex.female]) { + final batch = + banisterTrimp(hr, restingHr: 52, maxHr: 192, sex: sex); + final agg = banisterTrimpAggregator(restingHr: 52, maxHr: 192, sex: sex) + .run(_col(hr)); + _bitIdentical([..._meta(batch), batch.value], + [..._meta(agg), agg.value], 'trimp $sex'); + } + }); + + test('Cosinor: cosinor == cosinorAggregator one slice', () { + final r = Rng(31); + final n = 72 * 4; + final t = [for (var i = 0; i < n; i++) i * 0.25]; + final y = [ + for (final h in t) + 60 + + 12 * math.cos(2 * math.pi * (h - 15) / 24) + + (r.nextDouble() - .5) * 6 + ]; + final batch = cosinor(t, y); + final agg = cosinorAggregator() + .run(TimedValues(Float64List.fromList(t), Float64List.fromList(y))); + List f(Metric m) => [ + ..._meta(m), + m.value?.mesor, + m.value?.amplitude, + m.value?.acrophaseRad, + m.value?.acrophaseHours, + m.value?.periodHours, + m.value?.r2, + m.value?.r2Adj, + ]; + _bitIdentical(f(batch), f(agg), 'cosinor'); + }); + + test('Time in zone: HeartRateZones.timeInZone == timeInZoneAggregator ' + 'one slice', () { + final r = Rng(41); + final zones = HeartRateZones.zonesFromMaxHr(190); + final ts = []; + final hr = []; + var t = 1.7e12; + var h = 120.0; + for (var i = 0; i < 6000; i++) { + // Jittered ~1 Hz clock, so the credited seconds are not whole numbers + // and the float sums depend on summation order. + t += r.nextBool(.02) + ? 1000.0 * r.intIn(5, 90) + : 1000.0 + r.intIn(-37, 37) / 8; + h = (h + (r.nextDouble() - .5) * 14).clamp(40.0, 195.0); + ts.add(t); + hr.add(r.nextBool(.03) ? 0 : h); + } + final batch = HeartRateZones.timeInZone( + [for (var i = 0; i < ts.length; i++) HrSample(ts[i], hr[i])], zones); + final agg = timeInZoneAggregator(zones).run( + HrStream(Float64List.fromList(ts), Float64List.fromList(hr))); + List f(TimeInHeartRateZone? z) => + z == null ? [null] : [...z.seconds, z.belowZone1]; + expect(batch, isNotNull); + _bitIdentical(f(batch), f(agg), 'time in zone'); + }); + + test('CTL/ATL/TSB: ctlAtlTsb == ctlAtlTsbAggregator one slice', () { + List f(Metric m) => + [..._meta(m), m.value?.ctl, m.value?.atl, m.value?.tsb]; + for (final (seed, days) in [(51, 120), (52, 45), (53, 200), (54, 365)]) { + final r = Rng(seed); + final trimp = [ + for (var i = 0; i < days; i++) + i % 7 == 6 ? 0.0 : 40 + r.nextDouble() * 160 + ]; + final batch = ctlAtlTsb(trimp); + final agg = ctlAtlTsbAggregator().run(_col(trimp)); + expect(batch.present, isTrue); + _bitIdentical(f(batch), f(agg), 'ctl/atl/tsb seed $seed, $days days'); + } + }); + + test('PRSA DC/AC: decelerationCapacity/accelerationCapacity == ' + 'prsaAggregator one slice', () { + final r = Rng(61); + var v = 900.0; + final nn = [ + for (var i = 0; i < 3000; i++) + v = (v + (r.nextDouble() - .5) * 90).clamp(700.0, 1100.0) + ]; + List f(Metric m) => [ + ..._meta(m), + m.value?.capacity, + m.value?.profile, + m.value?.anchors, + m.value?.kind, + ]; + for (final dec in [true, false]) { + final batch = + dec ? decelerationCapacity(nn) : accelerationCapacity(nn); + final agg = prsaAggregator(deceleration: dec).run(_col(nn)); + expect(batch.present, isTrue); + _bitIdentical(f(batch), f(agg), dec ? 'DC' : 'AC'); + } + }); + + test('Circadian IS/IV/M10/L5/RA: circadianNonparametric == ' + 'circadianNpAggregator one slice', () { + final r = Rng(71); + const perDay = 96; + final x = [ + for (var i = 0; i < perDay * 6; i++) + 50 + + 40 * math.sin(2 * math.pi * (i % perDay - 30) / perDay) + + r.nextDouble() * 25 + ]; + final batch = circadianNonparametric(x, perDay); + final agg = circadianNpAggregator(perDay) + .run(EpochSeries(Float64List.fromList(x))); + List f(Metric m) => [ + ..._meta(m), + m.value?.interdailyStability, + m.value?.intradailyVariability, + m.value?.m10, + m.value?.l5, + m.value?.relativeAmplitude, + m.value?.m10StartEpoch, + m.value?.l5StartEpoch, + ]; + expect(batch.present, isTrue); + _bitIdentical(f(batch), f(agg), 'circadian'); + }); + + test('Readiness lnRMSSD: readinessLnRmssd == readinessLnRmssdAggregator ' + 'one slice (shares readinessLnRmssdFromSuffix; may already pass)', () { + final r = Rng(81); + final hist = [for (var i = 0; i < 40; i++) 3.6 + (r.nextDouble() - .5) * .6]; + List f(Metric m) => [ + ..._meta(m), + m.value?.lnRmssdToday, + m.value?.rolling7Mean, + m.value?.rolling7Median, + m.value?.cvPct, + m.value?.z, + m.value?.swc, + m.value?.band, + m.value?.saturationFlag, + ]; + final batch = readinessLnRmssd(hist, meanNnTodayMs: 880); + final agg = + readinessLnRmssdAggregator(meanNnTodayMs: 880).run(_col(hist)); + expect(batch.present, isTrue); + _bitIdentical(f(batch), f(agg), 'readiness'); + }); +} diff --git a/test/aggregate/persistence_test.dart b/test/aggregate/persistence_test.dart new file mode 100644 index 00000000..8d2ca5f8 --- /dev/null +++ b/test/aggregate/persistence_test.dart @@ -0,0 +1,986 @@ +// Round 10: persistent aggregator states and versioned checkpoints. +// +// SHAPE DECISIONS (the spec left these open; the implementation follows them): +// +// * Non-finite doubles are encoded as the strings 'NaN', 'Infinity' and +// '-Infinity' (the same strings incremental_core.dart's _readDouble already +// accepts). Finite doubles stay JSON numbers; -0.0 stays -0.0. Decoders +// accept an int where a double is expected. A decoder rejects any other +// string, bool, null, list or map in a double slot. +// * Every map-shaped encoding writes every key it has, with null for an +// absent value, and the decoder REQUIRES every key (a missing key is a +// FormatException, not a default). Decoders throw FormatException (never +// TypeError / RangeError / StateError) for any malformed input. +// * ListBatch element codec: `RowCodec(Object? Function(A) encode, +// A Function(Object?) decode)` plus the named constructor +// `ListBatch.coded(List rows, RowCodec codec)`. A coded batch +// keeps its codec through copyRange/join; `toJson()` is the JSON list of +// encoded rows. `RowCodec.batchFromJson(Object?)` returns a coded +// ListBatch (FormatException if the input is not a list; the row decode +// function itself must throw FormatException on a bad row). A plain +// `ListBatch(rows).toJson()` throws UnsupportedError. +// * Every persistable batch type X (DoubleColumn, Beats, TimedValues, +// HrStream, Epochs, EpochSeries) has `Object? toJson()` and a +// `factory X.fromJson(Object? json)` (FormatException on malformed input). +// The JSON must survive jsonEncode (so NaN/inf use the string encoding). +// * First / Last / Lookback / Prefix take an optional NAMED parameter +// `batchFromJson` (`B Function(Object?)`): First({batchFromJson}), +// Last({batchFromJson}), Lookback(k, inner, {batchFromJson}), +// Prefix(k, head, tail, {batchFromJson}). Persistable only when given +// (and, for Lookback/Prefix, when the stored inner/tail is persistable; +// Prefix keeps no head state, so the head does not matter). +// Without batchFromJson, encodeState/decodeState throw UnsupportedError. +// * GroupBy encodes a JSON map with the int keys as decimal strings; a key +// that is not a decimal int is a FormatException. +// * `both(a, b)` encodes its state as the two-element list [first, second]; +// any other length or type is a FormatException. +// * Moments.encodeState is exactly RunningMoments.toJson (a map), and +// Moments.decodeState is RunningMoments.fromJson behind a FormatException +// guard. +// * Aggregator defaults: persistable false; encodeState/decodeState throw +// UnsupportedError; Batch.toJson throws UnsupportedError. +// * The checkpoint envelope lives in lib/src/aggregate/checkpoint.dart and is +// exported from package:openstrap_analytics/aggregate.dart. +// encodeCheckpoint on a non-persistable aggregator throws UnsupportedError. + +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart' show RunningMoments; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +const nan = double.nan; +const inf = double.infinity; + +Object? wire(Object? x) => jsonDecode(jsonEncode(x)); + +bool close(double a, double b) => + a == b || + (a.isNaN && b.isNaN) || + (a - b).abs() <= 1e-9 * [1.0, a.abs(), b.abs()].reduce((x, y) => x > y ? x : y); + +bool closeOpt(double? a, double? b) => + a == null || b == null ? a == b : close(a, b); + +int bits(double x) => (ByteData(8)..setFloat64(0, x)).getInt64(0); + +bool sameCol(DoubleColumn? a, DoubleColumn? b) { + if (a == null || b == null) return a == b; + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!close(a.values[i], b.values[i])) return false; + } + return true; +} + +/// Mixed columns: specials, NaN, infinities, signed zero, large and tiny. +List allCases({bool finiteOnly = false, bool moderate = false}) { + final r = Rng(0x10A); + final specials = finiteOnly + ? [-0.0, 0.0, 5e-324, 0.1 + 0.2, -123456.789] + : moderate + ? [-0.0, 0.0, 5e-324, 0.1 + 0.2, -123456.789] + : [-0.0, 0.0, 5e-324, 0.1 + 0.2, 1.7976931348623157e308, -inf, inf]; + double draw() => r.nextBool(.2) + ? specials[r.intIn(0, specials.length - 1)] + : (r.nextDouble() - .5) * 2000; + return [ + col([]), + col([1]), + col([-0.0]), + if (!finiteOnly) col([nan, nan]), + for (var i = 0; i < 24; i++) + col([ + for (var n = r.intIn(0, 30); n > 0; n--) + !finiteOnly && r.nextBool(.12) ? nan : draw() + ]), + ]; +} + +/// For every split: decode(wire(encode(fold [0,k)))) continued by combine and +/// by addSlice must agree with the one-pass result, as must the full state. +void expectResumes, R>( + Aggregator agg, List cases, bool Function(R, R) eq) { + expect(agg.persistable, isTrue); + S rt(S s) => agg.decodeState(wire(agg.encodeState(s))); + for (var c = 0; c < cases.length; c++) { + final b = cases[c]; + final n = b.length; + final want = agg.run(b); + expect(eq(agg.finish(rt(agg.fold(b))), want), isTrue, + reason: 'full state, case $c'); + expect(eq(agg.finish(rt(agg.empty())), agg.finish(agg.empty())), isTrue, + reason: 'empty state'); + for (var k = 0; k <= n; k++) { + final viaCombine = + agg.finish(agg.combine(rt(agg.fold(b, 0, k)), agg.fold(b, k, n))); + expect(eq(viaCombine, want), isTrue, reason: 'combine, case $c split $k'); + final viaAdd = agg.finish(agg.addSlice(rt(agg.fold(b, 0, k)), b, k, n)); + expect(eq(viaAdd, want), isTrue, reason: 'addSlice, case $c split $k'); + final viaLeft = + agg.finish(agg.combine(agg.fold(b, 0, k), rt(agg.fold(b, k, n)))); + expect(eq(viaLeft, want), isTrue, reason: 'right restored, case $c split $k'); + } + } +} + +/// Variants of a JSON value with one thing broken. Lists keep their length +/// (shorter lists are legitimate for variable-length encodings). +Iterable mutants(Object? node, {bool removeKeys = true}) sync* { + const repls = ['zzz', true, {'bogus': 1}]; + if (node is Map) { + for (final key in node.keys) { + if (removeKeys) yield {...node}..remove(key); + for (final r in repls) { + yield {...node, key: r}; + } + for (final v in mutants(node[key], removeKeys: removeKeys)) { + yield {...node, key: v}; + } + } + } else if (node is List) { + for (var i = 0; i < node.length; i++) { + for (final r in repls) { + yield [...node]..[i] = r; + } + for (final v in mutants(node[i], removeKeys: removeKeys)) { + yield [...node]..[i] = v; + } + } + } +} + +void expectMalformed, R>( + Aggregator agg, Iterable payloads) { + var n = 0; + for (final p in payloads) { + n++; + expect(() => agg.decodeState(p), throwsFormatException, + reason: 'payload ${jsonEncode(p)}'); + } + expect(n, greaterThan(0)); +} + +/// Mutants of a real mid-stream encoding plus generic junk. +void expectRejectsDamage, R>( + Aggregator agg, B batch, + {bool removeKeys = true, List junk = const []}) { + final valid = wire(agg.encodeState(agg.fold(batch, 0, batch.length ~/ 2))); + expectMalformed( + agg, [...mutants(valid, removeKeys: removeKeys), null, 'zzz', true, ...junk]); +} + +Float64List _values(DoubleColumn b) => b.values; +Float64List _rowsAsDoubles(ListBatch b) => + Float64List.fromList([for (final r in b.rows) r.toDouble()]); + +/// Test-defined persistable aggregator: for each length-k window, adds the +/// first element. State is a plain finite double. +class _WindowFirst extends Aggregator { + const _WindowFirst(this.k); + final int k; + @override + double empty() => 0; + @override + double addSlice(double s, DoubleColumn b, int start, int end) { + for (var i = start + k - 1; i < end; i++) { + s += b.values[i - k + 1]; + } + return s; + } + + @override + double combine(double l, double r) => l + r; + @override + double finish(double s) => s; + @override + bool get persistable => true; + @override + Object? encodeState(double s) => s; + @override + double decodeState(Object? json) { + if (json is! num) throw const FormatException('number expected'); + return json.toDouble(); + } +} + +/// No persistence at all. +class _Plain extends Aggregator { + const _Plain(); + @override + int empty() => 0; + @override + int addSlice(int s, DoubleColumn b, int start, int end) => s + end - start; + @override + int combine(int l, int r) => l + r; + @override + int finish(int s) => s; +} + +typedef _IB = ListBatch; + +final _intCodec = RowCodec((v) => v, (o) { + if (o is! int) throw const FormatException('int expected'); + return o; +}); + +_IB _ib(List rows) => ListBatch.coded(rows, _intCodec); + +/// Sums rows and counts them; finish mixes both, so losing either shows. +/// With [dropCount] the codec silently forgets the count. +class _SumCount extends Aggregator, _IB, int> { + const _SumCount({this.dropCount = false, this.persists = true}); + final bool dropCount; + final bool persists; + @override + List empty() => [0, 0]; + @override + List addSlice(List s, _IB b, int start, int end) { + for (var i = start; i < end; i++) { + s[0] += b.rows[i]; + } + s[1] += end - start; + return s; + } + + @override + List combine(List l, List r) => + [l[0] + r[0], l[1] + r[1]]; + @override + int finish(List s) => s[0] * 1000 + s[1]; + @override + bool get persistable => persists; + @override + Object? encodeState(List s) => dropCount ? [s[0]] : [s[0], s[1]]; + @override + List decodeState(Object? json) { + if (json is! List || json.isEmpty || json.first is! int) { + throw const FormatException('bad'); + } + return [json.first as int, dropCount ? 0 : json[1] as int]; + } +} + +void main() { + group('non-persistable defaults', () { + test('an aggregator that overrides nothing is not persistable', () { + const a = _Plain(); + expect(a.persistable, isFalse); + expect(() => a.encodeState(0), throwsUnsupportedError); + expect(() => a.decodeState(0), throwsUnsupportedError); + }); + test('plain ListBatch and Batch.toJson throw UnsupportedError', () { + expect(() => ListBatch([1, 2]).toJson(), throwsUnsupportedError); + }); + }); + + group('persistable flags', () { + final sum = Sum(_values); + test('every primitive block is persistable', () { + expect(const Count().persistable, isTrue); + expect(CountFinite(_values).persistable, isTrue); + expect(CountWhere(_values, (v) => v > 0).persistable, isTrue); + expect(sum.persistable, isTrue); + expect(SumOf(_values, (v) => v * 2).persistable, isTrue); + expect(SumSq(_values).persistable, isTrue); + expect(Min(_values).persistable, isTrue); + expect(Max(_values).persistable, isTrue); + expect(Moments(_values).persistable, isTrue); + expect(Ewma(_values, .3).persistable, isTrue); + }); + test('First/Last/Lookback/Prefix are persistable only with batchFromJson', + () { + expect(First().persistable, isFalse); + expect(Last().persistable, isFalse); + expect( + First(batchFromJson: DoubleColumn.fromJson).persistable, + isTrue); + expect( + Last(batchFromJson: DoubleColumn.fromJson).persistable, + isTrue); + expect(Lookback(3, const _WindowFirst(3)) + .persistable, isFalse); + expect( + Lookback(3, const _WindowFirst(3), + batchFromJson: DoubleColumn.fromJson) + .persistable, + isTrue); + expect( + Prefix( + 2, const Count(), const _WindowFirst(1)) + .persistable, + isFalse); + expect( + Prefix( + 2, const Count(), _WindowFirst(1), + batchFromJson: DoubleColumn.fromJson) + .persistable, + isTrue); + expect(() => First().encodeState(First().empty()), + throwsUnsupportedError); + }); + test('a composite with a non-persistable child is not persistable', () { + const plain = _Plain(); + expect(both(sum, plain).persistable, isFalse); + expect(both(plain, sum).persistable, isFalse); + expect(both(sum, Count()).persistable, isTrue); + expect(plain.map((r) => r).persistable, isFalse); + expect(sum.map((r) => r).persistable, isTrue); + expect(GroupBy((b, i) => 0, plain).persistable, + isFalse); + expect( + GroupBy((b, i) => 0, const Count()).persistable, + isTrue); + expect( + Lookback(2, plain, + batchFromJson: DoubleColumn.fromJson) + .persistable, + isFalse); + expect( + Prefix(2, plain, plain, + batchFromJson: DoubleColumn.fromJson) + .persistable, + isFalse); + expect(() => both(sum, plain).encodeState(both(sum, plain).empty()), + throwsUnsupportedError); + }); + }); + + group('double encoding', () { + final agg = SumSq(_values); + test('non-finite doubles are the documented strings', () { + expect(agg.encodeState(nan), 'NaN'); + expect(agg.encodeState(inf), 'Infinity'); + expect(agg.encodeState(-inf), '-Infinity'); + expect(agg.decodeState('NaN'), isNaN); + expect(agg.decodeState('Infinity'), inf); + expect(agg.decodeState('-Infinity'), -inf); + }); + test('finite doubles round-trip bit-exactly through JSON text', () { + final r = Rng(7); + final xs = [ + 0.0, -0.0, 5e-324, -5e-324, 2.2250738585072014e-308, + 1.7976931348623157e308, -1.7976931348623157e308, 0.1 + 0.2, 1 / 3, + 9007199254740993.0, 123456789.123456789, + for (var i = 0; i < 200; i++) + (r.nextDouble() - .5) * (i.isEven ? 1e-300 : 1e300), + ]; + for (final x in xs) { + final back = agg.decodeState(wire(agg.encodeState(x))); + expect(bits(back), bits(x), reason: '$x'); + } + expect(agg.decodeState(wire(agg.encodeState(-0.0))).isNegative, isTrue); + }); + test('NaN and infinities survive jsonEncode (no JsonUnsupportedObjectError)', + () { + for (final x in [nan, inf, -inf]) { + final text = jsonEncode(agg.encodeState(x)); + final back = agg.decodeState(jsonDecode(text)); + expect(bits(back) == bits(x) || (x.isNaN && back.isNaN), isTrue); + } + }); + test('an int is accepted where a double is expected', () { + final v = agg.decodeState(3); + expect(v, 3.0); + expect(v, isA()); + expect(agg.decodeState(0), 0.0); + }); + test('malformed doubles', () { + expectMalformed(agg, [ + null, 'zzz', 'nan', 'inf', 'Infinit', '', '1.5', true, false, [], [1], + {}, {'v': 1} + ]); + }); + test('Min / Max state is a nullable double', () { + final mn = Min(_values); + final mx = Max(_values); + expect(mn.encodeState(null), isNull); + expect(mn.decodeState(null), isNull); + expect(mx.decodeState(null), isNull); + expect(mn.encodeState(-inf), '-Infinity'); + expect(mx.encodeState(inf), 'Infinity'); + expect(mn.decodeState(wire(mn.encodeState(-0.0)))!.isNegative, isTrue); + expectMalformed(mn, ['zzz', true, [], {}, 'nan']); + expectMalformed(mx, ['zzz', true, [], {}, 'nan']); + }); + }); + + group('counts', () { + final cs = >[ + const Count(), + CountFinite(_values), + CountWhere(_values, (v) => v > 0), + ]; + for (final (i, a) in cs.indexed) { + test('block $i round-trips on every split', () { + expectResumes(a, allCases(), (x, y) => x == y); + }); + test('block $i large ints are exact', () { + for (final big in [9007199254740993, 1 << 62, 9223372036854775807, 0]) { + expect(a.decodeState(wire(a.encodeState(big))), big); + } + }); + test('block $i malformed', () { + expectMalformed(a, [ + null, 'x', '5', 1.5, 2.0, -1, true, [], [1], {}, {'count': 1}, nan.toString() + ]); + }); + } + }); + + group('sums', () { + final aggs = >{ + 'Sum': Sum(_values), + 'SumOf': SumOf(_values, (v) => v * 2 - 1), + 'SumSq': SumSq(_values), + }; + aggs.forEach((name, a) { + test('$name round-trips on every split (moderate finite values)', () { + expectResumes(a, allCases(moderate: true), close); + }); + test('$name keeps +/-inf, NaN-from-cancellation and signed zero', () { + expectResumes(a, [ + col([1, inf, 2]), + col([-inf, 4]), + col([inf, -inf]), + col([-0.0]), + col([nan, 3, nan]), + ], close); + }); + test('$name empty state', () { + expect(a.finish(a.decodeState(wire(a.encodeState(a.empty())))), 0.0); + }); + test('$name malformed', () { + expectMalformed(a, [null, 'zzz', true, [], {}, 'nan']); + }); + }); + test('compensation survives the round trip', () { + final a = Sum(_values); + final b = col([1e16, 1.0, -1e16, 1.0, 1e-3, 1e16, 1.0, -1e16]); + final whole = a.run(b); + final half = a.fold(b, 0, 4); + final back = a.decodeState(wire(a.encodeState(half))); + expect(a.finish(a.addSlice(back, b, 4, 8)), whole); + }); + test('Sum and SumOf damaged mid-stream encodings are rejected', () { + final b = col([1.5, 2.5, 1e-9, 4, 5]); + expectRejectsDamage(Sum(_values), b); + expectRejectsDamage(SumOf(_values, (v) => v), b); + }); + }); + + group('min / max', () { + test('round trip on every split', () { + expectResumes(Min(_values), allCases(), closeOpt); + expectResumes(Max(_values), allCases(), closeOpt); + }); + test('infinite extrema and empty', () { + expectResumes(Min(_values), [col([3, -inf, 1]), col([])], closeOpt); + expectResumes(Max(_values), [col([3, inf, 1]), col([nan])], closeOpt); + }); + }); + + group('moments', () { + final a = Moments(_values); + bool eq(RunningMoments x, RunningMoments y) => + x.count == y.count && + closeOpt(x.mean, y.mean) && + closeOpt(x.sampleSd, y.sampleSd) && + closeOpt(x.populationSd, y.populationSd); + test('round trip on every split', () { + expectResumes(a, allCases(finiteOnly: true), eq); + }); + test('the encoding is RunningMoments.toJson', () { + final s = a.fold(col([1, 2, 4, 8])); + expect(a.encodeState(s), s.toJson()); + expect(a.encodeState(a.empty()), RunningMoments().toJson()); + }); + group('reachable non-finite moments (finite input overflowing m2)', () { + final over = col([1e155, -1e155]); // squared deviation overflows to inf + test('the state reaches Infinity from finite input', () { + expect(a.fold(over).populationSd, double.infinity); + }); + test('review case: encodeState survives jsonEncode and decodes back', () { + final state = a.fold(over); + late final String text; + expect(() => text = jsonEncode(a.encodeState(state)), returnsNormally); + final back = a.decodeState(jsonDecode(text)); + expect(back.count, 2); + expect(back.mean, state.mean); + expect(back.sampleSd, double.infinity); + expect(eq(a.finish(back), a.finish(state)), isTrue); + }); + test('the non-finite m2 uses the shared string encoding', () { + final json = wire(a.encodeState(a.fold(over))) as Map; + expect(json['m2'], 'Infinity'); + expect(json['count'], 2); + }); + // Split invariance once m2 has overflowed is out of domain (Welford's + // merge of an infinite m2 is not defined to match the one-pass fold); + // what must hold is that the reachable state round-trips, tested above. + test('a non-finite m2 is accepted only in the encoded string form', () { + final good = Map.of( + wire(a.encodeState(a.fold(over))) as Map); + expect(() => a.decodeState({...good, 'm2': 'Infinity'}), + returnsNormally); + for (final bad in [ + 'NaN', '-Infinity', 'inf', 'infinity', null, true, [], {}, + ]) { + expect(() => a.decodeState({...good, 'm2': bad}), + throwsFormatException, + reason: 'm2 ${jsonEncode(bad)}'); + } + }); + test('count and invariants are still validated with an infinite m2', () { + final good = Map.of( + wire(a.encodeState(a.fold(over))) as Map); + expectMalformed(a, [ + {...good, 'count': 1}, // one row cannot have spread + {...good, 'count': 0}, // nor none + {...good, 'count': -2}, + {...good, 'count': 1.5}, + {...good, 'origin': 'NaN'}, + {...good, 'origin': 'Infinity'}, + {...good, 'meanOffset': 'NaN'}, + {...good, 'version': 2}, + ]); + }); + }); + + test('malformed and inconsistent', () { + final good = Map.of( + wire(a.encodeState(a.fold(col([1, 2, 4])))) as Map); + expectMalformed(a, [ + null, 'zzz', 3, true, [], {}, + ...mutants(good), + {...good, 'count': -1}, + {...good, 'count': 1.5}, + {...good, 'version': 2}, + {...good, 'type': 'IntHistogram'}, + {...good, 'm2': -1.0}, + {...good, 'count': 0}, // origin/m2 non-zero with count 0 + {...good, 'count': 1}, // one row but m2 non-zero + {...good, 'origin': 'NaN'}, + ]); + }); + }); + + group('first / last', () { + final r = Rng(21); + final cases = [ + col([]), + col([nan]), + col([-0.0, 1, 2]), + col([inf, nan, -inf, 4]), + for (var i = 0; i < 6; i++) + col([for (var n = r.intIn(0, 9); n > 0; n--) r.nextDouble()]), + ]; + test('DoubleColumn batches (state holds a one-row copy)', () { + expectResumes(First(batchFromJson: DoubleColumn.fromJson), + cases, sameCol); + expectResumes(Last(batchFromJson: DoubleColumn.fromJson), + cases, sameCol); + }); + test('coded ListBatch batches', () { + bool eq(_IB? a, _IB? b) => + a == null || b == null ? a == b : a.rows.join(',') == b.rows.join(','); + final cs = [_ib([]), _ib([7]), _ib([1, 2, 3, 4]), _ib([-5, 9])]; + expectResumes(First<_IB>(batchFromJson: _intCodec.batchFromJson), cs, eq); + expectResumes(Last<_IB>(batchFromJson: _intCodec.batchFromJson), cs, eq); + }); + test('malformed', () { + final f = First(batchFromJson: DoubleColumn.fromJson); + final l = Last(batchFromJson: DoubleColumn.fromJson); + for (final a in [f, l]) { + expectRejectsDamage(a, col([1, 2, 3, 4])); + expectMalformed(a, [3, [], [1, 2], 'NaN']); + } + }); + }); + + group('group by', () { + _IB ints(List r) => _ib(r); + int key(_IB b, int i) => b.rows[i] ~/ 10; + final counts = GroupBy<_IB, int, int>(key, const Count<_IB>()); + final sums = GroupBy<_IB, dynamic, double>( + key, Sum<_IB>(_rowsAsDoubles)); + final cases = [ + ints([]), + ints([5]), + ints([1, 2, 11, 12, 13, 1, 2, -15, -16, 25]), + ints([for (var i = 0; i < 60; i++) (i * 7919) % 97 - 30]), + ints([1 << 40, 1 << 40, -(1 << 40), 3]), + ]; + test('round trip on every split (Count and Sum inner)', () { + expectResumes(counts, cases, (a, b) => a.toString() == b.toString()); + expectResumes( + sums, + cases, + (a, b) => + a.length == b.length && + a.keys.every((k) => b.containsKey(k) && close(a[k]!, b[k]!))); + }); + test('int keys are encoded as strings, huge and negative keys survive', () { + final big = GroupBy<_IB, int, int>( + (b, i) => b.rows[i], const Count<_IB>()); + final s = big.fold(ints([1 << 62, -7, -(1 << 62), 0, 9223372036854775807])); + final enc = big.encodeState(s) as Map; + expect(enc.keys.every((k) => k is String), isTrue); + final back = big.decodeState(wire(enc)); + expect(back.keys.toSet(), + {1 << 62, -7, -(1 << 62), 0, 9223372036854775807}); + }); + test('empty state is an empty map', () { + expect(counts.encodeState(counts.empty()), {}); + expect(counts.decodeState({}), isEmpty); + }); + test('malformed', () { + expectMalformed(counts, [ + null, 'zzz', 3, true, [], [1], + {'abc': 1}, {'1.5': 1}, {'': 1}, {'1': 'x'}, {'1': 1.5}, {'1': -1}, + {'1': null}, {'1': {}}, + ]); + final valid = wire(counts.encodeState(counts.fold(cases[2]))) as Map; + expectMalformed(counts, mutants(valid, removeKeys: false)); + }); + }); + + group('lookback', () { + DoubleColumn iota(int n) => col([for (var i = 0; i < n; i++) i * 1.5 + .25]); + for (final k in [1, 2, 3, 5]) { + final lb = Lookback(k, _WindowFirst(k), + batchFromJson: DoubleColumn.fromJson); + test('k=$k resumes mid-stream (edges and inner state restored)', () { + expectResumes(lb, [for (final n in [0, 1, 2, 4, 5, 6, 13]) iota(n)], + close); + }); + test('k=$k damaged encodings are rejected', () { + expectRejectsDamage(lb, iota(9)); + expectRejectsDamage(lb, iota(1)); + }); + } + test('a resumed seam equals the unbroken run for random cuts', () { + final r = Rng(5); + final lb = Lookback(4, const _WindowFirst(4), + batchFromJson: DoubleColumn.fromJson); + final b = col([for (var i = 0; i < 80; i++) r.nextDouble()]); + var s = lb.empty(); + var at = 0; + while (at < 80) { + final to = (at + r.intIn(0, 9)).clamp(0, 80); + s = lb.decodeState(wire(lb.encodeState(lb.addSlice(s, b, at, to)))); + at = to; + } + expect(close(lb.finish(s), lb.run(b)), isTrue); + }); + test('without batchFromJson encode/decode throw UnsupportedError', () { + final lb = Lookback(3, const _WindowFirst(3)); + expect(() => lb.encodeState(lb.empty()), throwsUnsupportedError); + expect(() => lb.decodeState(null), throwsUnsupportedError); + }); + }); + + group('prefix', () { + for (final k in [0, 1, 2, 5]) { + final p = Prefix( + k, const Count(), const _WindowFirst(1), + batchFromJson: DoubleColumn.fromJson); + bool eq((int, double) a, (int, double) b) => a.$1 == b.$1 && close(a.$2, b.$2); + test('k=$k resumes mid-stream', () { + expectResumes(p, [for (final n in [0, 1, 2, 3, 4, 7, 12]) col([for (var i = 0; i < n; i++) i * 2.5 - 3])], + eq); + }); + test('k=$k damaged encodings are rejected', () { + expectRejectsDamage(p, col([for (var i = 0; i < 12; i++) i * 1.5])); + expectRejectsDamage(p, col([1, 2])); + }); + } + }); + + group('ewma', () { + final a = Ewma(_values, .3); + bool eq(Affine x, Affine y) => close(x.scale, y.scale) && close(x.offset, y.offset); + test('round trip on every split', () { + expectResumes(a, allCases(moderate: true), eq); + }); + test('identity and non-finite maps survive', () { + expectResumes(a, [col([]), col([nan, nan]), col([inf, 1]), col([-0.0])], eq); + final id = a.decodeState(wire(a.encodeState(a.empty()))); + expect(a.finish(id).scale, 1.0); + expect(a.finish(id).offset, 0.0); + }); + test('malformed', () { + expectRejectsDamage(a, col([1, 2, 3, 4])); + expectMalformed(a, [3, [], {}]); + }); + }); + + group('both and map', () { + final sum = Sum(_values); + final pair = both(const Count(), sum); + test('both round trips on every split', () { + expectResumes(pair, allCases(moderate: true), + ((int, double) a, (int, double) b) => a.$1 == b.$1 && close(a.$2, b.$2)); + }); + test('both state is a two-element list', () { + final enc = pair.encodeState(pair.fold(col([1, 2, 3]))); + expect(enc, isA()); + expect((enc as List).length, 2); + expect(enc.first, 3); + }); + test('both malformed', () { + final valid = wire(pair.encodeState(pair.fold(col([1, 2, 3])))) as List; + expectMalformed(pair, [ + null, 'zzz', 3, true, {}, [], [1], [1, 2, 3], [valid[0]], + ['x', valid[1]], [valid[0], 'x'], [-1, valid[1]], + ...mutants(valid, removeKeys: false), + ]); + }); + test('map delegates to the inner codec', () { + final m = const Count().map((c) => c.toString()); + expect(m.persistable, isTrue); + expect(m.encodeState(42), const Count().encodeState(42)); + expect(m.decodeState(wire(m.encodeState(42))), 42); + expectResumes(m, allCases(), (a, b) => a == b); + expectMalformed(m, [null, 'x', -1, 1.5]); + }); + }); + + group('batch codecs', () { + test('DoubleColumn round-trips non-finite and signed-zero values', () { + final b = col([nan, inf, -inf, -0.0, 0.0, 5e-324, 0.1 + 0.2, 1e308]); + final back = DoubleColumn.fromJson(wire(b.toJson())); + expect(back.length, b.length); + for (var i = 0; i < b.length; i++) { + expect(bits(back.values[i]) == bits(b.values[i]) || + (b.values[i].isNaN && back.values[i].isNaN), isTrue, reason: '$i'); + } + expect(DoubleColumn.fromJson(wire(col([]).toJson())).length, 0); + }); + test('DoubleColumn accepts ints and rejects junk', () { + expect(DoubleColumn.fromJson([1, 2.5, 'NaN']).values[0], 1.0); + for (final bad in [ + null, 'zzz', 3, true, {}, {'v': []}, [true], ['zzz'], [null], [[]], ['nan'] + ]) { + expect(() => DoubleColumn.fromJson(bad), throwsFormatException, + reason: jsonEncode(bad)); + } + }); + test('coded ListBatch keeps its codec through copyRange and join', () { + final b = _ib([1, 2, 3, 4]); + expect(b.toJson(), [1, 2, 3, 4]); + expect(b.copyRange(1, 3).toJson(), [2, 3]); + expect(b.copyRange(0, 1).join(b.copyRange(2, 4)).toJson(), [1, 3, 4]); + final back = _intCodec.batchFromJson(wire(b.toJson())); + expect(back.rows, [1, 2, 3, 4]); + expect(back.toJson(), [1, 2, 3, 4]); + expect(_intCodec.batchFromJson([]).rows, isEmpty); + }); + test('coded ListBatch rejects malformed input', () { + for (final bad in [null, 'zzz', 3, {}, true, [1, 'x'], [null], [1.5]]) { + expect(() => _intCodec.batchFromJson(bad), throwsFormatException, + reason: jsonEncode(bad)); + } + }); + + final specials = Float64List.fromList([nan, inf, -inf, -0.0, 0.0, 1e-310, 2.5]); + Float64List sp(int n) => Float64List.fromList( + [for (var i = 0; i < n; i++) specials[i % specials.length]]); + bool same(Float64List a, Float64List b) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!(bits(a[i]) == bits(b[i]) || (a[i].isNaN && b[i].isNaN))) return false; + } + return true; + } + + test('Beats', () { + for (final n in [0, 1, 7]) { + final b = Beats(sp(n), sp(n + 3).sublist(3)); + final back = Beats.fromJson(wire(b.toJson())); + expect(same(back.nnMs, b.nnMs) && same(back.timeMs, b.timeMs), isTrue); + } + }); + test('TimedValues', () { + final b = TimedValues(sp(7), sp(7).sublist(0, 7)); + final back = TimedValues.fromJson(wire(b.toJson())); + expect(same(back.tHours, b.tHours) && same(back.y, b.y), isTrue); + }); + test('HrStream', () { + final b = HrStream(sp(7), sp(7)); + final back = HrStream.fromJson(wire(b.toJson())); + expect(same(back.tsMs, b.tsMs) && same(back.hr, b.hr), isTrue); + }); + test('EpochSeries keeps firstIndex', () { + final b = EpochSeries(sp(7), firstIndex: 1440); + final back = EpochSeries.fromJson(wire(b.toJson())); + expect(same(back.x, b.x), isTrue); + expect(back.firstIndex, 1440); + expect(EpochSeries.fromJson(wire(EpochSeries(sp(0)).toJson())).length, 0); + }); + test('Epochs keeps the mask (or its absence) and firstIndex', () { + final plain = Epochs(Uint8List.fromList([1, 0, 1, 1]), firstIndex: 96); + final p = Epochs.fromJson(wire(plain.toJson())); + expect(p.asleep, [1, 0, 1, 1]); + expect(p.valid, isNull); + expect(p.firstIndex, 96); + final masked = Epochs(Uint8List.fromList([1, 0, 1]), + valid: Uint8List.fromList([1, 1, 0])); + final m = Epochs.fromJson(wire(masked.toJson())); + expect(m.asleep, [1, 0, 1]); + expect(m.valid, [1, 1, 0]); + expect(m.firstIndex, 0); + expect(Epochs.fromJson(wire(Epochs(Uint8List(0)).toJson())).length, 0); + }); + + test('malformed batch payloads and mismatched columns', () { + final beats = Beats(sp(4), sp(4)); + final tv = TimedValues(sp(4), sp(4)); + final hr = HrStream(sp(4), sp(4)); + final es = EpochSeries(sp(4), firstIndex: 3); + final ep = Epochs(Uint8List.fromList([1, 0, 1]), + valid: Uint8List.fromList([1, 1, 0]), firstIndex: 5); + final epPlain = Epochs(Uint8List.fromList([1, 0, 1]), firstIndex: 5); + final cases = <(String, Object? Function(), Object? Function(Object?))>[ + ('Beats', beats.toJson, Beats.fromJson), + ('TimedValues', tv.toJson, TimedValues.fromJson), + ('HrStream', hr.toJson, HrStream.fromJson), + ('EpochSeries', es.toJson, EpochSeries.fromJson), + ('Epochs', ep.toJson, Epochs.fromJson), + ('Epochs (no mask)', epPlain.toJson, Epochs.fromJson), + ]; + for (final (name, toJson, from) in cases) { + final valid = wire(toJson()); + for (final bad in [null, 'zzz', 3, true, [], {}, ...mutants(valid)]) { + expect(() => from(bad), throwsFormatException, + reason: '$name ${jsonEncode(bad)}'); + } + } + // Two-column batches: shortening one column is an inconsistency. + for (final (name, toJson, from) in cases.where( + (c) => c.$1 != 'EpochSeries' && c.$1 != 'Epochs (no mask)')) { + final valid = wire(toJson()) as Map; + for (final key in valid.keys.where((k) => valid[k] is List)) { + final cut = { + ...valid, + key: (valid[key] as List).sublist(0, (valid[key] as List).length - 1) + }; + expect(() => from(cut), throwsFormatException, reason: '$name $key'); + } + } + }); + }); + + group('roundTrip law (harness self-test)', () { + final cases = [ + _ib([]), + _ib([4]), + _ib([1, 2, 3, 4, 5]), + _ib([9, 8, 7]), + ]; + Set laws(List v) => {for (final x in v) x.law}; + + test('a faithful codec has no violations', () { + expect(checkLaws(const _SumCount(), cases), isEmpty); + }); + test('a codec that drops a field is caught as exactly {roundTrip}', () { + final v = checkLaws(const _SumCount(dropCount: true), cases); + expect(laws(v), {'roundTrip'}); + expect(v.every((x) => x.detail.isNotEmpty), isTrue); + }); + test('a non-persistable aggregator is not asked to round trip', () { + expect(checkLaws(const _SumCount(dropCount: true, persists: false), cases), + isEmpty); + }); + test('checkConversion reports it as law:roundTrip', () { + final f = checkConversion, _IB, int>( + (_) => const _SumCount(dropCount: true), + cases, + (b) => b.rows.fold(0, (s, x) => s + x) * 1000 + b.length); + expect({for (final x in f) x.path}, {'law:roundTrip'}); + }); + test('expectLawful fails for the lossy codec and passes for the good one', + () { + expectLawful(const _SumCount(), cases); + expect(() => expectLawful(const _SumCount(dropCount: true), cases), + throwsA(isA())); + }); + test('the built-in blocks pass the new law', () { + expect(checkLaws(Sum(_values), allCases(moderate: true), equals: close), isEmpty); + expect(checkLaws(const Count(), allCases()), isEmpty); + }); + }); + + group('checkpoint envelope', () { + final sum = Sum(_values); + final b = col([1.5, nan, 2.25, 1e-3, 4]); + test('encodes kind, version and state', () { + final env = encodeCheckpoint(const Count(), 5, + kind: 'k', version: 2); + expect(env, {'kind': 'k', 'version': 2, 'state': 5}); + }); + test('resume through JSON text continues to the one-pass result', () { + final env = encodeCheckpoint(sum, sum.fold(b, 0, 2), + kind: 'sum', version: 1); + final state = decodeCheckpoint(sum, wire(env), + kind: 'sum', version: 1); + expect(state, isNotNull); + expect(close(sum.finish(sum.addSlice(state, b, 2, 5)), sum.run(b)), isTrue); + }); + test('a non-finite state still encodes to JSON text', () { + final a = SumSq(_values); + final env = encodeCheckpoint(a, inf, kind: 'x', version: 1); + expect(decodeCheckpoint(a, wire(env), kind: 'x', version: 1), inf); + }); + test('kind or version mismatch refuses with null', () { + final env = wire(encodeCheckpoint(sum, sum.fold(b), kind: 'a', version: 3)); + expect(decodeCheckpoint(sum, env, kind: 'b', version: 3), isNull); + expect(decodeCheckpoint(sum, env, kind: 'a', version: 2), isNull); + expect(decodeCheckpoint(sum, env, kind: 'a', version: 4), isNull); + expect(decodeCheckpoint(sum, env, kind: 'a', version: 3), isNotNull); + }); + test('malformed state or envelope refuses with null, never throws', () { + const c = Count(); + Map env(Object? state) => + {'kind': 'k', 'version': 1, 'state': state}; + for (final bad in [ + env('zzz'), env(-1), env(1.5), env(null), env([]), + {'kind': 'k', 'version': 1}, + {'kind': 'k', 'state': 1}, + {'version': 1, 'state': 1}, + {'kind': 7, 'version': 1, 'state': 1}, + {'kind': 'k', 'version': '1', 'state': 1}, + {'kind': 'k', 'version': 1.0, 'state': 1}, + {'kind': 'k', 'version': null, 'state': 1}, + null, 'zzz', 3, true, [], [1, 2, 3], {}, + ]) { + expect(decodeCheckpoint(c, bad, kind: 'k', version: 1), isNull, + reason: jsonEncode(bad)); + } + expect(decodeCheckpoint(c, env(4), kind: 'k', version: 1), 4); + }); + test('a damaged nested state refuses rather than guessing', () { + final lb = Lookback(3, const _WindowFirst(3), + batchFromJson: DoubleColumn.fromJson); + final data = col([for (var i = 0; i < 9; i++) i * 1.0]); + final good = wire(encodeCheckpoint(lb, lb.fold(data, 0, 5), + kind: 'lb', version: 1)) as Map; + expect(decodeCheckpoint(lb, good, kind: 'lb', version: 1), isNotNull); + for (final m in mutants(good['state'])) { + expect( + decodeCheckpoint(lb, {...good, 'state': m}, + kind: 'lb', version: 1), + isNull, + reason: jsonEncode(m)); + } + }); + test('a non-persistable aggregator cannot be checkpointed', () { + expect(() => encodeCheckpoint(const _Plain(), 0, kind: 'k', version: 1), + throwsUnsupportedError); + }); + }); +} diff --git a/test/algo_version_test.dart b/test/algo_version_test.dart new file mode 100644 index 00000000..b51b688c --- /dev/null +++ b/test/algo_version_test.dart @@ -0,0 +1,20 @@ +// Counters move for direct algorithm/output changes, including rounding and +// abstention. Dependency-only changes propagate through deps. +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void main() { + test('PR output changes move their owning groups to counter 2', () { + const changed = { + 'found.core', // Lomb-Scargle first-sample origin. + 'found.rr_correction', // Artifact-run cap, already bumped. + 'sleep.stage', // Already bumped; also owns adopted HRV staging paths. + 'hrv.freq', // HRV aggregator rounding and invalid-time abstention. + 'night.rhythm', // Cosinor/circadian and gap-aware irregular screening. + 'activity.load', // TRIMP, CTL/ATL and time-in-zone aggregation. + }; + expect({for (final g in kAnalyticsGroupRegistry.groups) + if (g.counter != 1) g.id: g.counter}, + {for (final id in changed) id: 2}); + }); +} diff --git a/test/incremental/day_curves_test.dart b/test/incremental/day_curves_test.dart new file mode 100644 index 00000000..014b19a7 --- /dev/null +++ b/test/incremental/day_curves_test.dart @@ -0,0 +1,192 @@ +// Oracle tests for the three DAY-side curves that live in edge: +// dayHrvCurve, dayRespCurve, _daytimeHrv (verbatim ports in edge_oracles.dart) +// Chunked by wall time like derive passes: RR with ts < T and accel rows with +// tsSec < T arrive together, watermark = T. JSON restart between chunks. +import 'dart:math' as math; + +import 'package:test/test.dart'; + +import '../../tool/incremental/edge_oracles.dart'; +import '../../tool/incremental/hrv_incr.dart'; +import '../../tool/incremental/oracle_util.dart'; +import '../../tool/incremental/resp_incr.dart'; +import '../../tool/incremental/synth.dart'; + +class _Day { + final RrData rr; + final AccelSeries acc; + _Day(this.rr, this.acc); + DaySub prefix(int rrN, int accN) => DaySub( + rr.rr.sublist(0, rrN), + rr.ts.sublist(0, rrN), + acc.tsSec.sublist(0, accN), + acc.ax.sublist(0, accN), + acc.ay.sublist(0, accN), + acc.az.sublist(0, accN)); +} + +_Day _makeDay({bool backwards = false}) { + final rr = backwards + ? synthRr(const SynthConfig(seed: 81, hours: 6, backwardsPerHour: 10)) + : realShapedDay(); + final from = (rr.ts.first / 1000).floor() - 2; + final to = (rr.ts.last / 1000).ceil() + 3; + return _Day(rr, synthAccel(5, from, to)); +} + +/// wall-clock cut points (seconds) like derive passes: mostly 15 min, some odd. +List _cutSecs(_Day d, math.Random r) { + final first = (d.rr.ts.first / 1000).floor(), last = (d.rr.ts.last / 1000).ceil() + 3; + final cuts = []; + var t = first; + while (t < last) { + t += [60, 300, 900, 900, 900, 1800, 3600][r.nextInt(7)] + r.nextInt(7); + cuts.add(math.min(t, last + 1)); + } + return cuts; +} + +void main() { + final day = _makeDay(); + final sleepOnset = 1700000000, sleepOffset = 1700000000 + 32040; + + test('dayHrvCurve: 24 h in random wall-time chunks, restart each chunk', () { + final r = math.Random(1); + var st = DayHrvCurveState(); + var rrAt = 0; + var maxRing = 0; + final cuts = _cutSecs(day, r); + for (var k = 0; k < cuts.length; k++) { + final T = cuts[k]; + var rrTo = rrAt; + while (rrTo < day.rr.length && day.rr.ts[rrTo] < T * 1000.0) { + rrTo++; + } + st.fold(day.rr.rr.sublist(rrAt, rrTo), day.rr.ts.sublist(rrAt, rrTo)); + rrAt = rrTo; + st = DayHrvCurveState.fromJson(jsonRoundTrip(st.toJson())); + maxRing = math.max(maxRing, st.ringSize); + if (k % 7 == 0 || k == cuts.length - 1) { + final want = oracleDayHrvCurve(day.prefix(rrTo, 0)); + expect(st.out.map((m) => '${m['t']}:${m['v']}').toList(), + want.map((m) => '${m['t']}:${m['v']}').toList(), + reason: 'chunk $k'); + } + } + // ignore: avoid_print + print('dayHrvCurve: points=${st.out.length}, max ring beats=$maxRing'); + }); + + test('dayHrvCurve with backwards timestamps', () { + final d = _makeDay(backwards: true); + final st = DayHrvCurveState(); + final r = math.Random(2); + var at = 0; + for (final cut in randomCuts(r, d.rr.length, sizes: [1, 10, 500, 3000])) { + st.fold(d.rr.rr.sublist(at, cut), d.rr.ts.sublist(at, cut)); + at = cut; + } + final want = oracleDayHrvCurve(d.prefix(d.rr.length, 0)); + expect(st.out.map((m) => '${m['t']}:${m['v']}').toList(), + want.map((m) => '${m['t']}:${m['v']}').toList()); + }); + + for (final lag in [0, 400]) { + test('_daytimeHrv: 24 h, night masked, accel lag=${lag}s behind RR, restart', () { + final r = math.Random(3); + var st = DaytimeHrvState(sleepOnset, sleepOffset); + var rrAt = 0, accAt = 0; + final cuts = _cutSecs(day, r); + var maxPending = 0; + for (var k = 0; k < cuts.length; k++) { + final T = cuts[k]; + var rrTo = rrAt, accTo = accAt; + while (rrTo < day.rr.length && day.rr.ts[rrTo] < T * 1000.0) { + rrTo++; + } + while (accTo < day.acc.length && day.acc.tsSec[accTo] < T - lag) { + accTo++; + } + st.fold( + day.rr.rr.sublist(rrAt, rrTo), + day.rr.ts.sublist(rrAt, rrTo), + day.acc.tsSec.sublist(accAt, accTo), + day.acc.ax.sublist(accAt, accTo), + day.acc.ay.sublist(accAt, accTo), + day.acc.az.sublist(accAt, accTo), + T - lag); + rrAt = rrTo; + accAt = accTo; + st = DaytimeHrvState.fromJson(jsonRoundTrip(st.toJson()), sleepOnset, sleepOffset); + maxPending = math.max(maxPending, st.pending); + if (k % 5 == 0 || k == cuts.length - 1) { + // the oracle only sees what has been delivered AND processed: beats + // pending (second not yet reported) are excluded from the prefix. + final processedRr = rrTo - st.pending; + final want = oracleDaytimeHrv(day.prefix(processedRr, accTo), sleepOnset, sleepOffset); + final got = st.result(); + expect(got['timeline'].toString(), want['timeline'].toString(), reason: 'chunk $k'); + expect(got['mean_rmssd'], want['mean_rmssd']); + expect(got['n_buckets'], want['n_buckets']); + } + } + // ignore: avoid_print + print('daytimeHrv: max pending beats at a chunk edge=$maxPending ' + 'buckets=${st.result()['n_buckets']}'); + if (lag > 0) expect(maxPending, greaterThan(0)); + }); + } + + test('_daytimeHrv with no sleep window (onset=offset=0)', () { + final st = DaytimeHrvState(0, 0); + st.fold(day.rr.rr, day.rr.ts, day.acc.tsSec, day.acc.ax, day.acc.ay, day.acc.az, + (day.rr.ts.last / 1000).ceil() + 10); + final want = oracleDaytimeHrv(day.prefix(day.rr.length, day.acc.length), 0, 0); + expect(st.result().toString(), want.toString()); + }); + + for (final lag in [0, 400]) { + test('dayRespCurve: 24 h, accel lag=${lag}s behind RR, restart each chunk', () { + final r = math.Random(4); + var st = DayRespCurveState(); + var rrAt = 0, accAt = 0; + final cuts = _cutSecs(day, r); + var maxPending = 0, maxRing = 0; + for (var k = 0; k < cuts.length; k++) { + final T = cuts[k]; + var rrTo = rrAt, accTo = accAt; + while (rrTo < day.rr.length && day.rr.ts[rrTo] < T * 1000.0) { + rrTo++; + } + while (accTo < day.acc.length && day.acc.tsSec[accTo] < T - lag) { + accTo++; + } + st.fold( + day.rr.rr.sublist(rrAt, rrTo), + day.rr.ts.sublist(rrAt, rrTo), + day.acc.tsSec.sublist(accAt, accTo), + day.acc.ax.sublist(accAt, accTo), + day.acc.ay.sublist(accAt, accTo), + day.acc.az.sublist(accAt, accTo), + T - lag); + rrAt = rrTo; + accAt = accTo; + st = DayRespCurveState.fromJson(jsonRoundTrip(st.toJson())); + maxPending = math.max(maxPending, st.pending); + maxRing = math.max(maxRing, st.ringBeats); + if (k % 6 == 0 || k == cuts.length - 1) { + final processedRr = rrTo - st.pending; + final want = oracleDayRespCurve(day.prefix(processedRr, accTo)); + expect(st.curve.map((m) => '${m['t']}:${m['v']}').toList(), + want.map((m) => '${m['t']}:${m['v']}').toList(), + reason: 'chunk $k'); + } + } + // ignore: avoid_print + print('dayRespCurve lag=$lag: points=${st.curve.length} ' + 'max pending=$maxPending max ring beats=$maxRing'); + expect(st.curve, isNotEmpty); + if (lag > 0) expect(maxPending, greaterThan(0)); + }); + } +} diff --git a/test/onehz/incremental/calculation_cache_test.dart b/test/onehz/incremental/calculation_cache_test.dart new file mode 100644 index 00000000..cb552755 --- /dev/null +++ b/test/onehz/incremental/calculation_cache_test.dart @@ -0,0 +1,190 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void main() { + test('cache retains null values and distinguishes changed list order', () { + final cache = CalculationCache(); + var calls = 0; + Object? calculate() { + calls++; + return null; + } + + expect(cache.evaluate('nullable', [1, 2, null], calculate), isNull); + expect(cache.evaluate('nullable', [1, 2, null], calculate), isNull); + expect(calls, 1); + expect(cache.evaluate('nullable', [2, 1, null], calculate), isNull); + expect(calls, 2); + expect(cache.computations, 2); + expect(cache.hits, 1); + }); + test('cache protects typed lists while retaining their return type', () { + final cache = CalculationCache(); + final original = [1.25, 3.5]; + final first = cache.evaluate>('list', 1, () => original); + first.add(9); + original[0] = 8; + final next = cache.evaluate>('list', 1, () => [100.0]); + expect(next, [1.25, 3.5]); + next.clear(); + expect(cache.evaluate>('list', 1, () => [100.0]), [1.25, 3.5]); + }); + test( + 'cache snapshots nested dependencies and deep compares maps, null and NaN', + () { + final cache = CalculationCache(); + final dependency = { + 'hr': [ + 70.0, + null, + double.nan, + {'day': 1} + ], + 'profile': {'sex': 'female', 'weight': 61.0}, + }; + var calls = 0; + int calculate() => ++calls; + expect(cache.evaluate('load', dependency, calculate), 1); + expect( + cache.evaluate( + 'load', + { + 'profile': {'weight': 61.0, 'sex': 'female'}, + 'hr': [ + 70.0, + null, + double.nan, + {'day': 1} + ], + }, + calculate), + 1); + expect(calls, 1); + expect(cache.computations, 1); + expect(cache.hits, 1); + (dependency['hr'] as List)[0] = 71.0; + expect(cache.evaluate('load', dependency, calculate), 2); + ((dependency['hr'] as List)[3] as Map)['day'] = 2; + expect(cache.evaluate('load', dependency, calculate), 3); + (dependency['profile'] as Map)['sex'] = 'male'; + expect(cache.evaluate('load', dependency, calculate), 4); + expect(cache.evaluate('load', dependency, calculate), 4); + expect(cache.computations, 4); + expect(cache.hits, 2); + }); + test('cache protects mutable values on first return and subsequent hits', () { + final cache = CalculationCache(); + final produced = { + 'nested': [ + {'value': 7} + ], + 'values': [1, 2] + }; + Map calculate() => produced; + final first = cache.evaluate('nested', [1], calculate); + (first['nested'][0] as Map)['value'] = 999; + (produced['values'] as List).add(3); + final second = cache.evaluate('nested', [1], calculate); + expect(second, { + 'nested': [ + {'value': 7} + ], + 'values': [1, 2] + }); + (second['values'] as List).clear(); + expect(cache.evaluate('nested', [1], calculate), { + 'nested': [ + {'value': 7} + ], + 'values': [1, 2] + }); + expect(cache.computations, 1); + expect(cache.hits, 2); + }); + test( + 'cache exceptions preserve prior dependencies and do not count as computations', + () { + final cache = CalculationCache(); + expect(cache.evaluate('load', [1], () => 7), 7); + expect( + () => cache.evaluate('load', [2], () => throw StateError('bad')), + throwsStateError); + expect(cache.computations, 1); + expect(cache.hits, 0); + expect(cache.evaluate('load', [1], () => 99), 7); + expect(cache.evaluate('load', [2], () => 8), 8); + expect(cache.computations, 2); + expect(cache.hits, 1); + expect( + () => cache.evaluate('load', [2], () => throw StateError('bad'), + full: true), + throwsStateError); + expect(cache.evaluate('load', [2], () => 99), 8); + expect(cache.computations, 2); + }); + for (final mode in CalculationMode.values) { + test('cache mode $mode controls full bypass', () { + expect(mode.canReuse, mode == CalculationMode.periodicAwake); + final cache = CalculationCache(); + var calls = 0; + int calculate() => ++calls; + expect(cache.evaluate('load', null, calculate, full: !mode.canReuse), 1); + expect(cache.evaluate('load', null, calculate, full: !mode.canReuse), + mode.canReuse ? 1 : 2); + expect(cache.computations, mode.canReuse ? 1 : 2); + expect(cache.hits, mode.canReuse ? 1 : 0); + }); + } + test('cache keys are isolated, full compute refreshes, clear discards values', + () { + final cache = CalculationCache(); + var calls = 0; + int calculate() => ++calls; + expect(cache.evaluate('a', null, calculate), 1); + expect(cache.evaluate('b', null, calculate), 2); + expect(cache.evaluate('a', null, calculate, full: true), 3); + expect(cache.evaluate('a', null, calculate), 3); + cache.clear(); + expect(cache.evaluate('a', null, calculate), 4); + expect(calls, 4); + }); + test('cache defaults to bounded 128 entries', () { + final cache = CalculationCache(); + var calls = 0; + for (var i = 0; i < 129; i++) { + cache.evaluate('metric-$i', null, () => ++calls); + } + for (var i = 0; i < 129; i++) { + cache.evaluate('metric-$i', null, () => ++calls); + } + expect(calls, greaterThan(129), + reason: 'at least one old key must be evicted'); + expect(cache.computations, calls); + }); + for (final capacity in [1, 3, 7]) { + test('cache bounded capacity=$capacity does not retain every entry', () { + final cache = CalculationCache(maxEntries: capacity); + var calls = 0; + for (var i = 0; i < capacity + 1; i++) { + cache.evaluate('metric-$i', [i], () => ++calls); + } + for (var i = 0; i < capacity + 1; i++) { + cache.evaluate('metric-$i', [i], () => ++calls); + } + expect(calls, greaterThan(capacity + 1)); + }); + } + test('cache handles typed immutable results without encoding them as JSON', + () { + final cache = CalculationCache(); + const result = Metric( + value: 7, confidence: .7, tier: Tier.estimate, inputs_used: ['hr']); + expect(cache.evaluate('metric', [1], () => result), same(result)); + expect( + cache.evaluate>( + 'metric', [1], () => throw StateError('hit')), + same(result)); + expect(cache.computations, 1); + expect(cache.hits, 1); + }); +} diff --git a/test/onehz/incremental/checkpoint_ownership_test.dart b/test/onehz/incremental/checkpoint_ownership_test.dart new file mode 100644 index 00000000..b9603cad --- /dev/null +++ b/test/onehz/incremental/checkpoint_ownership_test.dart @@ -0,0 +1,51 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import '../support/incremental_compare.dart'; +import '../support/incremental_fixtures.dart'; + +void main() { + const original = + WorkoutUserProfile(weightKg: 80, heightCm: 180, age: 40, sex: 'male'); + for (final replacement in [ + const WorkoutUserProfile(weightKg: 95, heightCm: 180, age: 40, sex: 'male'), + const WorkoutUserProfile(weightKg: 80, heightCm: 160, age: 40, sex: 'male'), + const WorkoutUserProfile(weightKg: 80, heightCm: 180, age: 60, sex: 'male'), + const WorkoutUserProfile( + weightKg: 80, heightCm: 180, age: 40, sex: 'female'), + ]) { + test( + 'restored minute profile owns checkpoint ${replacement.weightKg}/' + '${replacement.heightCm}/${replacement.age}/${replacement.sex}', () { + final f = incrementalFixture(minutes: 35); + final originalState = IncrementalMinuteMetrics(); + originalState.sync(f.minuteKeys, f.hr, + cadenceSpm: f.cadence, + restingHr: 54, + maxHr: 186, + quietHrr: .12, + profile: original); + final json = originalState.toJson(); + final restored = IncrementalMinuteMetrics.fromJson(json); + final parameters = json['parameters'] as List; + final profile = parameters[3] as List; + profile.setAll(0, [ + replacement.weightKg, + replacement.heightCm, + replacement.age, + replacement.sex + ]); + final work = restored.processedMinutes; + final actual = restored.sync(f.minuteKeys, f.hr, + cadenceSpm: f.cadence, + restingHr: 54, + maxHr: 186, + quietHrr: .12, + profile: replacement); + minuteClose(actual, f.minuteKeys, f.hr, + cadence: f.cadence, profile: replacement); + expect(restored.processedMinutes - work, f.hr.length, + reason: 'changing profile reprices retained contributions'); + }); + } +} diff --git a/test/onehz/incremental/drift_test.dart b/test/onehz/incremental/drift_test.dart new file mode 100644 index 00000000..aaaae4ea --- /dev/null +++ b/test/onehz/incremental/drift_test.dart @@ -0,0 +1,123 @@ +// Long-run drift: thousands of incremental updates, compared with the batch +// oracle along the way. The shorter parity tests cannot show error that only +// builds up over a full night of appends or a long sliding window. +import 'dart:io'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/incremental_compare.dart'; + +double _rel(double? a, double? e) { + if (a == null || e == null) return a == e ? 0 : double.infinity; + final scale = e.abs(); + return scale < 1e-12 ? (a - e).abs() : (a - e).abs() / scale; +} + +/// Logs the worst relative error seen, so the margin under the bound is +/// visible in the test output and not only pass/fail. +void _report(String label, double worst) => + stdout.writeln('[drift] $label worst relative error ${worst.toStringAsExponential(2)}'); + +/// A night of beats on absolute epoch seconds, the magnitude production +/// timestamps carry, with a slow drift and respiratory modulation. +({List t, List y}) _night(int beats, {int seed = 7}) { + final rnd = math.Random(seed); + final t = [], y = []; + var clock = 1791028730.0; + for (var i = 0; i < beats; i++) { + final rr = 900 + + 120 * math.sin(i / 4000) + + 40 * math.sin(i * 2 * math.pi * .25 * .9) + + rnd.nextDouble() * 30; + clock += rr / 1000; + if (i % 2311 == 2310) clock += 45; // an occasional dropout + t.add(clock); + y.add(rr); + } + return (t: t, y: y); +} + +void main() { + final frequencies = [for (var k = 0; k <= 40; k++) .0033 + k * .01]; + + test('Lomb–Scargle over a full night of appends stays on the batch result', + () { + final night = _night(30000); + var state = IncrementalLombScargle(frequencies); + var worst = 0.0, worstAbsolute = 0.0; + var n = 0; + for (final checkpointAt in [500, 2000, 8000, 16000, 24000, 30000]) { + while (n < checkpointAt) { + n = math.min(checkpointAt, n + 13); + state.sync(night.t.sublist(0, n), night.y.sublist(0, n)); + } + final ts = night.t.sublist(0, n), ys = night.y.sublist(0, n); + final actual = state.sync(ts, ys)!; + // The oracle runs on times shifted by the first beat. The periodogram is + // shift-invariant, and the shift is exact here (Sterbenz), but on raw + // epoch seconds the batch's own trig arguments reach ~1e10 rad and lose + // ~1e-6 relative; checked against an 80-bit reference, that batch error + // is 4.6e-6 while shifted double precision is 3.8e-13. + final shifted = [for (final x in ts) x - ts.first]; + final expected = lombScargle(shifted, ys, frequencies)!; + final absolute = lombScargle(ts, ys, frequencies)!; + for (var i = 0; i < expected.spectrum.length; i++) { + final a = actual.spectrum[i].psd, e = expected.spectrum[i].psd; + worst = math.max(worst, _rel(a, e)); + expect(_rel(a, e), lessThan(1e-9), reason: 'f=${frequencies[i]}'); + worstAbsolute = + math.max(worstAbsolute, _rel(a, absolute.spectrum[i].psd)); + expect(_rel(a, absolute.spectrum[i].psd), lessThan(1e-5)); + } + // A restored checkpoint must continue the same way. + state = IncrementalLombScargle.fromJson(checkpoint(state.toJson())); + } + _report('lomb 30000 beats vs shifted batch', worst); + _report('lomb 30000 beats vs epoch-time batch', worstAbsolute); + }); + + test('moments over a long sliding window with removals stay on two-pass', + () { + // Large offset, small spread: the case where naive sum-of-squares loses + // every digit. 100 000 adds and 99 700 removals through a 300-value window. + final rnd = math.Random(11); + final window = []; + final state = RunningMoments(); + var worstMean = 0.0, worstSd = 0.0; + for (var i = 0; i < 100000; i++) { + final x = 1e6 + 50 * math.sin(i / 97) + rnd.nextDouble(); + window.add(x); + state.add(x); + if (window.length > 300) state.remove(window.removeAt(0)); + if (i % 997 == 0 || i == 99999) { + worstMean = math.max(worstMean, _rel(state.mean, mean(window))); + worstSd = math.max(worstSd, _rel(state.sampleSd, stddev(window))); + } + } + _report('moments window mean', worstMean); + _report('moments window sd', worstSd); + expect(worstMean, lessThan(1e-9)); + expect(worstSd, lessThan(1e-9)); + }); + + test('moments removed down to two values from a large set', () { + final rnd = math.Random(3); + final values = [ + for (var i = 0; i < 50000; i++) 800 + 100 * rnd.nextDouble() + ]; + final state = RunningMoments(); + for (final x in values) { + state.add(x); + } + var worstSd = 0.0; + while (values.length > 2) { + state.remove(values.removeLast()); + if (values.length % 1000 == 0 || values.length < 10) { + worstSd = math.max(worstSd, _rel(state.sampleSd, stddev(values))); + } + } + _report('moments shrink sd', worstSd); + expect(worstSd, lessThan(1e-9)); + }); +} diff --git a/test/onehz/incremental/enmo_series_test.dart b/test/onehz/incremental/enmo_series_test.dart new file mode 100644 index 00000000..2ac15aee --- /dev/null +++ b/test/onehz/incremental/enmo_series_test.dart @@ -0,0 +1,123 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/incremental_compare.dart'; +import '../support/incremental_fixtures.dart'; + +void main() { + for (final seed in [1, 42, 914]) { + for (final hz in [.5, 1.0, 2.0]) { + for (final chunk in [1, 31, 97]) { + test('ENMO explicit reference seed=$seed hz=$hz chunk=$chunk', () { + final f = incrementalFixture( + seed: seed, cadenceHz: hz, originMs: 1700000010000); + var state = IncrementalEnmoSeries(); + for (final n in prefixSizes(f.accel.length, chunk)) { + final a = f.accel.sublist(0, n); + enmoClose(state.sync(a, gRef: 1.013456789, expectedMinutes: 12), + enmoSeries(a, gRef: 1.013456789, expectedMinutes: 12)); + final work = state.processedPoints; + state.sync(a, gRef: 1.013456789, expectedMinutes: 12); + expect(state.processedPoints, work); + if (n == 31 || n == f.accel.length) { + state = + IncrementalEnmoSeries.fromJson(checkpoint(state.toJson())); + } + } + expect(state.processedPoints, lessThanOrEqualTo(f.accel.length)); + }); + } + } + } + for (final window in [0.0, 1.0, 15.0, 61.5]) { + for (final minimum in [1, 30, 60]) { + test('ENMO window=$window coverage minimum=$minimum', () { + final samples = [ + for (final t in [ + 0.0, + 999, + 1000, + 14999, + 15000, + 59999, + 60000, + 60001, + 119999, + 120000, + 199999 + ]) + AccelSample(t, t % 17 / 30, .1, 1), + ]; + final state = IncrementalEnmoSeries(); + for (var n = 0; n <= samples.length; n++) { + final a = samples.sublist(0, n); + enmoClose( + state.sync(a, + gRef: 1, + gravityWindowS: window, + minSamplesPerMinute: minimum), + enmoSeries(a, + gRef: 1, + gravityWindowS: window, + minSamplesPerMinute: minimum)); + } + }); + } + } + test('ENMO auto-calibration recomputes each changing prefix', () { + final f = incrementalFixture(); + var state = IncrementalEnmoSeries(); + for (final n in prefixSizes(f.accel.length, 29)) { + final a = f.accel.sublist(0, n); + enmoClose(state.sync(a), enmoSeries(a)); + state = IncrementalEnmoSeries.fromJson(checkpoint(state.toJson())); + } + }); + test('ENMO parameter changes, replace, remove, reorder and force', () { + final f = incrementalFixture(); + var a = [...f.accel]; + final state = IncrementalEnmoSeries(); + state.sync(a, gRef: 1); + enmoClose( + state.sync(a, + gRef: 1.05, + gravityWindowS: 31, + minSamplesPerMinute: 60, + expectedMinutes: 100), + enmoSeries(a, + gRef: 1.05, + gravityWindowS: 31, + minSamplesPerMinute: 60, + expectedMinutes: 100)); + final old = a[7]; + a[7] = AccelSample(old.tsMs, old.x + .07, old.y, old.z); + enmoClose(state.sync(a, gRef: 1), enmoSeries(a, gRef: 1)); + a = a.sublist(9, 140).reversed.toList(); + enmoClose(state.sync(a, gRef: 1), enmoSeries(a, gRef: 1)); + final work = state.processedPoints; + enmoClose(state.sync(a, gRef: 1, force: true), enmoSeries(a, gRef: 1)); + expect(state.processedPoints, greaterThan(work)); + enmoClose(state.sync([], gRef: 1), enmoSeries([], gRef: 1)); + }); + test('ENMO duplicate timestamps, invalid samples and constant vector', () { + final a = [ + for (var i = 0; i < 80; i++) + AccelSample((i ~/ 2) * 1000.0, .3, -.4, 1, valid: i % 3 != 0), + const AccelSample(500, 100, 100, 100, valid: false), + ]; + final state = IncrementalEnmoSeries(); + enmoClose(state.sync(a, gRef: 1), enmoSeries(a, gRef: 1)); + final invalid = + a.map((s) => AccelSample(s.tsMs, s.x, s.y, s.z, valid: false)).toList(); + enmoClose(state.sync(invalid), enmoSeries(invalid)); + }); + test('ENMO real capture streamed with batch-calibrated reference', () { + final f = realIncrementalFixture(); + expect(f.accel.length, 900); + final g = enmoSeries(f.accel).gRef; + final state = IncrementalEnmoSeries(); + for (final n in prefixSizes(f.accel.length, 113)) { + final a = f.accel.sublist(0, n); + enmoClose(state.sync(a, gRef: g), enmoSeries(a, gRef: g)); + } + }); +} diff --git a/test/onehz/incremental/hrv_time_test.dart b/test/onehz/incremental/hrv_time_test.dart new file mode 100644 index 00000000..9570da38 --- /dev/null +++ b/test/onehz/incremental/hrv_time_test.dart @@ -0,0 +1,158 @@ +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/incremental_compare.dart'; +import '../support/incremental_fixtures.dart'; + +void main() { + test('HRV jitter gate refuses only successive-difference outputs', () { + final jitter = incrementalFixture(seed: 42, jitter: true, gaps: false); + final oracle = hrvTime(jitter.nn, nnTimesMs: jitter.times); + expect(oracle.value!.diffAcf1, lessThan(kNnDiffAcf1Floor)); + expect(oracle.value!.rmssd, isNull); + expect(oracle.value!.pnn50, isNull); + expect(oracle.value!.sdnn, isNotNull); + expect(oracle.value!.sdann, isNotNull); + hrvClose( + IncrementalHrvTime().sync(jitter.nn, nnTimesMs: jitter.times), oracle); + }); + for (final seed in [1, 42, 914]) { + for (final jitter in [false, true]) { + for (final chunk in [1, 37, 251]) { + test( + 'HRV seams and evolving bins seed=$seed jitter=$jitter chunk=$chunk', + () { + final f = incrementalFixture(seed: seed, jitter: jitter); + var state = IncrementalHrvTime(); + for (final n in prefixSizes(f.nn.length, chunk)) { + final nn = f.nn.sublist(0, n), t = f.times.sublist(0, n); + hrvClose(state.sync(nn, nnTimesMs: t, artifactFraction: .17), + hrvTime(nn, nnTimesMs: t, artifactFraction: .17)); + final work = state.processedPoints; + state.sync(nn, nnTimesMs: t, artifactFraction: .17); + expect(state.processedPoints, work); + if (n % 37 == 0 || n == f.nn.length) { + state = IncrementalHrvTime.fromJson(checkpoint(state.toJson())); + } + } + expect(state.processedPoints, lessThanOrEqualTo(f.nn.length + 2)); + }); + } + } + } + for (final origin in [0.0, 299999.5, 1700000010123.125]) { + test('HRV five-minute exact boundaries and shifted origin=$origin', () { + final nn = List.generate(610, (i) => 900 + (i % 11) * 7.0); + final t = List.generate(610, (i) => origin + i * 1000); + final state = IncrementalHrvTime(); + for (final n in [2, 299, 300, 301, 302, 599, 600, 601, 602, 610]) { + final y = nn.sublist(0, n), ts = t.sublist(0, n); + hrvClose(state.sync(y, nnTimesMs: ts), hrvTime(y, nnTimesMs: ts)); + } + }); + } + for (final artifacts in [0.0, .15, .8, 1.0]) { + test('HRV artifact confidence reprice=$artifacts', () { + final f = incrementalFixture(); + final state = IncrementalHrvTime(); + state.sync(f.nn, nnTimesMs: f.times); + hrvClose( + state.sync(f.nn, nnTimesMs: f.times, artifactFraction: artifacts), + hrvTime(f.nn, nnTimesMs: f.times, artifactFraction: artifacts)); + }); + } + test('HRV replacements, dropped prefix and force rebuild', () { + final f = incrementalFixture(); + var nn = [...f.nn], times = [...f.times]; + final state = IncrementalHrvTime(); + state.sync(nn, nnTimesMs: times); + nn[119] += 60; + hrvClose(state.sync(nn, nnTimesMs: times), hrvTime(nn, nnTimesMs: times)); + times[113] += 18000; + hrvClose(state.sync(nn, nnTimesMs: times), hrvTime(nn, nnTimesMs: times)); + nn = nn.sublist(27, 700); + times = times.sublist(27, 700); + hrvClose(state.sync(nn, nnTimesMs: times), hrvTime(nn, nnTimesMs: times)); + final work = state.processedPoints; + hrvClose(state.sync(nn, nnTimesMs: times, force: true), + hrvTime(nn, nnTimesMs: times)); + expect(state.processedPoints - work, nn.length); + hrvClose(state.sync(nn), hrvTime(nn)); + hrvClose(state.sync(nn, nnTimesMs: [1]), hrvTime(nn, nnTimesMs: [1])); + }); + test('HRV constant series, singleton, empty and all separated seams', () { + final state = IncrementalHrvTime(); + for (final n in [0, 1, 2, 29, 30, 31, 32, 80]) { + final nn = List.filled(n, 800); + final times = List.generate(n, (i) => i * 5000.0); + hrvClose(state.sync(nn), hrvTime(nn)); + hrvClose(state.sync(nn, nnTimesMs: times), hrvTime(nn, nnTimesMs: times)); + } + }); + // Below the ACF floor upstream's hrvTime reads the difference runs: a + // respiratory line on a low floor exempts the night, and a refused night may + // publish from the windows on a breathing line steady in Hz. Both are batch + // decisions the stream must reproduce at every prefix. + ({List nn, List t}) breathing( + {required double hours, + required double baseMs, + required double driftMs, + required double breathMs, + required double breathHz, + required double jitterMs}) { + final r = math.Random(7); + final nn = [], t = []; + var clock = 0.0; + while (clock < hours * 3.6e6) { + final v = baseMs + + driftMs * clock / (hours * 3.6e6) + + breathMs * math.sin(2 * math.pi * breathHz * clock / 1000) + + jitterMs * (r.nextDouble() - .5); + clock += v; + nn.add(v); + t.add(clock); + } + return (nn: nn, t: t); + } + + for (final (name, x, note) in [ + ( + 'a respiratory line exempts the night', + breathing( + hours: 1, baseMs: 1350, driftMs: 80, breathMs: 40, breathHz: .3, + jitterMs: 3), + 'PRV not ECG-HRV; RMSSD/pNN50', + ), + ( + 'a refused night publishes from a breathing line steady in Hz', + breathing( + hours: 2.5, baseMs: 900, driftMs: 400, breathMs: 10, breathHz: .25, + jitterMs: 40), + 'breathing line steady in Hz', + ), + ]) { + test('HRV below the ACF floor: $name', () { + final whole = hrvTime(x.nn, nnTimesMs: x.t); + expect(whole.value!.diffAcf1, lessThan(kNnDiffAcf1Floor), + reason: 'the fixture is under the floor'); + expect(whole.value!.rmssd, isNotNull, reason: 'and the batch publishes'); + expect(whole.note, contains(note), reason: 'by the path named'); + var state = IncrementalHrvTime(); + for (final n in prefixSizes(x.nn.length, 701)) { + final nn = x.nn.sublist(0, n), t = x.t.sublist(0, n); + hrvClose(state.sync(nn, nnTimesMs: t), hrvTime(nn, nnTimesMs: t)); + state = IncrementalHrvTime.fromJson(checkpoint(state.toJson())); + } + }); + } + test('HRV real capture chunk parity includes cleaned beat times', () { + final f = realIncrementalFixture(); + expect(f.nn.length, greaterThan(300)); + final state = IncrementalHrvTime(); + for (final n in prefixSizes(f.nn.length, 149)) { + final nn = f.nn.sublist(0, n), t = f.times.sublist(0, n); + hrvClose(state.sync(nn, nnTimesMs: t), hrvTime(nn, nnTimesMs: t)); + } + }); +} diff --git a/test/onehz/incremental/minute_metrics_test.dart b/test/onehz/incremental/minute_metrics_test.dart new file mode 100644 index 00000000..bad8f0da --- /dev/null +++ b/test/onehz/incremental/minute_metrics_test.dart @@ -0,0 +1,251 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/energy_day.dart'; +import '../support/incremental_compare.dart'; +import '../support/incremental_fixtures.dart'; + +void main() { + for (final entry in energyProfiles.entries) { + for (final sex in Sex.values) { + for (final chunk in [1, 19, 128]) { + test('minutes profile=${entry.key} sex=$sex chunk=$chunk', () { + final f = incrementalFixture(minutes: 87); + var state = IncrementalMinuteMetrics(); + for (final n in prefixSizes(f.hr.length, chunk)) { + final keys = f.minuteKeys.sublist(0, n), hr = f.hr.sublist(0, n); + final cadence = f.cadence.sublist(0, n); + final value = state.sync(keys, hr, + cadenceSpm: cadence, + restingHr: 54, + maxHr: 186, + sex: sex, + profile: entry.value, + quietHrr: .12, + dayMinutes: 900); + minuteClose(value, keys, hr, + cadence: cadence, + sex: sex, + profile: entry.value, + dayMinutes: 900); + final work = state.processedMinutes; + state.sync(keys, hr, + cadenceSpm: cadence, + restingHr: 54, + maxHr: 186, + sex: sex, + profile: entry.value, + quietHrr: .12, + dayMinutes: 900); + expect(state.processedMinutes, work); + if (n == 31 || n == f.hr.length) { + state = + IncrementalMinuteMetrics.fromJson(checkpoint(state.toJson())); + } + } + expect(state.processedMinutes, f.hr.length, + reason: 'each new minute is billed once'); + }); + } + } + } + for (final anchors in <(double?, double?)>[ + (null, null), + (54, null), + (null, 186), + (54, 54), + (70, 60), + (double.nan, 186), + (54, double.infinity), + (54, 186), + ]) { + test('minute measured HR anchor gates $anchors', () { + final f = incrementalFixture(minutes: 31); + final profile = energyProfiles['male']!; + final state = IncrementalMinuteMetrics(); + final value = state.sync(f.minuteKeys, f.hr, + restingHr: anchors.$1, + maxHr: anchors.$2, + profile: profile, + quietHrr: .12); + minuteClose(value, f.minuteKeys, f.hr, + rhr: anchors.$1, maxHr: anchors.$2, profile: profile); + }); + } + for (final quiet in [null, 0, -.1, .12, double.nan]) { + test('minute quiet waking HRR gate $quiet', () { + final f = incrementalFixture(minutes: 31); + minuteClose( + IncrementalMinuteMetrics().sync(f.minuteKeys, f.hr, + restingHr: 54, maxHr: 186, quietHrr: quiet), + f.minuteKeys, + f.hr, + quietHrr: quiet); + }); + } + test('minute sparse arbitrary keyed edits bill only affected minutes', () { + final f = incrementalFixture(minutes: 50); + final profile = energyProfiles['female']!; + var keys = [...f.minuteKeys], hr = [...f.hr]; + var cadence = [...f.cadence]; + final state = IncrementalMinuteMetrics(); + MinuteMetrics sync() => state.sync(keys, hr, + cadenceSpm: cadence, + restingHr: 54, + maxHr: 186, + profile: profile, + quietHrr: .12); + void check() => + minuteClose(sync(), keys, hr, cadence: cadence, profile: profile); + check(); + expect(state.processedMinutes, 50); + hr[11] += 29; + check(); + expect(state.processedMinutes, 51); + cadence[22] = 123; + check(); + expect(state.processedMinutes, 52); + keys.insert(5, 27999999); + hr.insert(5, 95); + cadence.insert(5, 120); + check(); + expect(state.processedMinutes, 53); + keys.removeAt(8); + hr.removeAt(8); + cadence.removeAt(8); + check(); + expect(state.processedMinutes, 53, + reason: 'deletion removes existing bill'); + keys = keys.reversed.toList(); + hr = hr.reversed.toList(); + cadence = cadence.reversed.toList(); + check(); + expect(state.processedMinutes, 53, + reason: 'same keyed minutes may be reordered'); + keys[0] += 10000; + check(); + expect(state.processedMinutes, 54); + final work = state.processedMinutes; + minuteClose( + state.sync(keys, hr, + cadenceSpm: cadence, + restingHr: 54, + maxHr: 186, + profile: profile, + quietHrr: .12, + force: true), + keys, + hr, + cadence: cadence, + profile: profile); + expect(state.processedMinutes - work, keys.length); + }); + test( + 'minute parameter changes recompute sex, profiles, anchors and day totals', + () { + final f = incrementalFixture(minutes: 47); + final state = IncrementalMinuteMetrics(); + for (final entry in energyProfiles.entries) { + for (final sex in Sex.values) { + for (final rhr in [54.0, 65.0]) { + final value = state.sync(f.minuteKeys, f.hr, + cadenceSpm: f.cadence, + restingHr: rhr, + maxHr: 190, + profile: entry.value, + sex: sex, + quietHrr: .18, + dayMinutes: 700); + minuteClose(value, f.minuteKeys, f.hr, + cadence: f.cadence, + rhr: rhr, + maxHr: 190, + profile: entry.value, + sex: sex, + quietHrr: .18, + dayMinutes: 700); + } + } + } + final p = energyProfiles['male']!; + minuteClose( + state.sync(f.minuteKeys, f.hr, + restingHr: 54, + maxHr: 186, + profile: p, + dayMinutes: 0, + quietHrr: .12), + f.minuteKeys, + f.hr, + profile: p, + dayMinutes: 0); + minuteClose( + state.sync(f.minuteKeys, f.hr, + restingHr: 54, + maxHr: 186, + profile: p, + dayMinutes: 900, + quietHrr: .12), + f.minuteKeys, + f.hr, + profile: p, + dayMinutes: 900); + }); + test('minute nonfinite HR/cadence and absent HR use exact abstentions', () { + final day = energyDay(n: 100); + final keys = List.generate(day.hr.length, (i) => i * 3); + final p = energyProfiles['nonbinary']!; + final state = IncrementalMinuteMetrics(); + for (final n in prefixSizes(day.hr.length, 13)) { + final k = keys.sublist(0, n), + h = day.hr.sublist(0, n), + c = day.cadence.sublist(0, n); + minuteClose( + state.sync(k, h, + cadenceSpm: c, + restingHr: 54, + maxHr: 186, + profile: p, + quietHrr: .12), + k, + h, + cadence: c, + profile: p); + } + }); + test( + 'minute JSON checkpoint continues exact precision with keyed replacement', + () { + final f = incrementalFixture(minutes: 61); + final hr = f.hr.map((x) => x + .0123456789123).toList(); + final p = energyProfiles['male']!; + final first = IncrementalMinuteMetrics(); + first.sync(f.minuteKeys.sublist(0, 31), hr.sublist(0, 31), + restingHr: 54, maxHr: 186, profile: p, quietHrr: .12); + final state = IncrementalMinuteMetrics.fromJson(checkpoint(first.toJson())); + hr[17] += 5.123456789123; + minuteClose( + state.sync(f.minuteKeys, hr, + restingHr: 54, maxHr: 186, profile: p, quietHrr: .12), + f.minuteKeys, + hr, + profile: p); + expect(state.processedMinutes, 62); + }); + test( + 'minute real capture means and sparse keys match batch load and calories', + () { + final f = realIncrementalFixture(); + expect(f.hr.length, 15); + final p = energyProfiles['male']!; + final state = IncrementalMinuteMetrics(); + for (var n = 0; n <= f.hr.length; n++) { + final k = f.minuteKeys.sublist(0, n), h = f.hr.sublist(0, n); + minuteClose( + state.sync(k, h, + restingHr: 54, maxHr: 186, profile: p, quietHrr: .12), + k, + h, + profile: p); + } + }); +} diff --git a/test/onehz/incremental/minute_summary_test.dart b/test/onehz/incremental/minute_summary_test.dart new file mode 100644 index 00000000..504b0aca --- /dev/null +++ b/test/onehz/incremental/minute_summary_test.dart @@ -0,0 +1,281 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/energy_day.dart'; +import '../support/incremental_compare.dart'; +import '../support/incremental_fixtures.dart'; + +void _summaryClose( + MinuteMetrics actual, + List hr, { + List? cadence, + double? rhr = 54, + double? maxHr = 186, + Sex sex = Sex.male, + WorkoutUserProfile? profile, + int dayMinutes = 1440, + double? quietHrr = .12, +}) { + expect(actual.minutes, isNull, + reason: 'summary requests omit minute artifacts'); + final trimp = banisterTrimp(hr, restingHr: rhr, maxHr: maxHr, sex: sex); + final strain = strainScoreMetric(trimp.value, + wakeMinutes: hr.length.toDouble(), + quietHrr: quietHrr, + quietSettled: false, + female: sex == Sex.female); + metricEnvelope(actual.trimp, trimp); + numberClose(actual.trimp.value, trimp.value); + metricEnvelope(actual.strain, strain); + numberClose(actual.strain.value, strain.value); + final energy = profile == null || rhr == null || maxHr == null + ? null + : Calories.dailyEnergy(hr, + profile: profile, + hrmax: maxHr, + restingHr: rhr, + dayMinutes: dayMinutes, + cadenceSpmPerMin: cadence); + if (energy == null) { + expect(actual.energy, isNull); + } else { + expect(actual.energy, isNotNull); + numberClose(actual.energy!.total, energy.total); + numberClose(actual.energy!.active, energy.active); + numberClose(actual.energy!.basal, energy.basal); + numberClose(actual.energy!.walking, energy.walking); + } +} + +void main() { + for (final seed in [1, 42, 914]) { + for (final entry in energyProfiles.entries) { + for (final chunk in [1, 19]) { + test( + 'summary append/restore seed=$seed profile=${entry.key} chunk=$chunk', + () { + final f = incrementalFixture(seed: seed, minutes: 61); + final sex = entry.key == 'female' ? Sex.female : Sex.male; + var state = IncrementalMinuteMetrics(); + for (final n in prefixSizes(f.hr.length, chunk)) { + final keys = f.minuteKeys.sublist(0, n), hr = f.hr.sublist(0, n); + final cad = f.cadence.sublist(0, n); + _summaryClose( + state.sync(keys, hr, + cadenceSpm: cad, + restingHr: 54, + maxHr: 186, + sex: sex, + profile: entry.value, + quietHrr: .12, + dayMinutes: 900, + includeMinuteSeries: false), + hr, + cadence: cad, + sex: sex, + profile: entry.value, + dayMinutes: 900); + expect(state.processedMinutes, n); + if (n == 31) { + state = + IncrementalMinuteMetrics.fromJson(checkpoint(state.toJson())); + } + } + final work = state.processedMinutes; + minuteClose( + state.sync(f.minuteKeys, f.hr, + cadenceSpm: f.cadence, + restingHr: 54, + maxHr: 186, + sex: sex, + profile: entry.value, + quietHrr: .12, + dayMinutes: 900), + f.minuteKeys, + f.hr, + cadence: f.cadence, + sex: sex, + profile: entry.value, + dayMinutes: 900); + expect(state.processedMinutes, work, + reason: + 'emitting retained minute artifacts requires no repricing'); + }); + } + } + } + test( + 'summary edits/removals/reorder and artifact toggles only reprice changed keys', + () { + final f = incrementalFixture(minutes: 50); + final profile = energyProfiles['male']!; + final state = IncrementalMinuteMetrics(); + var keys = [...f.minuteKeys], hr = [...f.hr]; + var cadence = [...f.cadence]; + MinuteMetrics sync({bool series = false, bool force = false}) => + state.sync(keys, hr, + cadenceSpm: cadence, + restingHr: 54, + maxHr: 186, + profile: profile, + quietHrr: .12, + force: force, + includeMinuteSeries: series); + void check() => + _summaryClose(sync(), hr, cadence: cadence, profile: profile); + // The existing default behavior remains the full minute series. + minuteClose(sync(series: true), keys, hr, + cadence: cadence, profile: profile); + expect(state.processedMinutes, 50); + check(); + expect(state.processedMinutes, 50); + hr[11] += 29; + check(); + expect(state.processedMinutes, 51); + cadence[22] = 123; + check(); + expect(state.processedMinutes, 52); + keys.insert(5, 27999999); + hr.insert(5, 95); + cadence.insert(5, 120); + check(); + expect(state.processedMinutes, 53); + keys.removeAt(8); + hr.removeAt(8); + cadence.removeAt(8); + check(); + expect(state.processedMinutes, 53); + keys = keys.reversed.toList(); + hr = hr.reversed.toList(); + cadence = cadence.reversed.toList(); + check(); + expect(state.processedMinutes, 53); + minuteClose(sync(series: true), keys, hr, + cadence: cadence, profile: profile); + expect(state.processedMinutes, 53); + _summaryClose(sync(force: true), hr, cadence: cadence, profile: profile); + expect(state.processedMinutes, 53 + keys.length); + keys.clear(); + hr.clear(); + cadence.clear(); + check(); + final work = state.processedMinutes; + minuteClose(sync(series: true), keys, hr, + cadence: cadence, profile: profile); + expect(state.processedMinutes, work); + }); + test( + 'summary anchor/profile/sex changes and day duration follow batch exactly', + () { + final f = incrementalFixture(minutes: 31); + var state = IncrementalMinuteMetrics(); + for (final p in energyProfiles.values) { + for (final anchors in <(double?, double?)>[ + (54, 186), + (65, 197), + (null, 186), + (54, null), + (70, 60), + (double.nan, 186) + ]) { + for (final sex in Sex.values) { + for (final duration in [0, 900, 1440]) { + _summaryClose( + state.sync(f.minuteKeys, f.hr, + cadenceSpm: f.cadence, + restingHr: anchors.$1, + maxHr: anchors.$2, + sex: sex, + profile: p, + quietHrr: .18, + dayMinutes: duration, + includeMinuteSeries: false), + f.hr, + cadence: f.cadence, + rhr: anchors.$1, + maxHr: anchors.$2, + sex: sex, + profile: p, + quietHrr: .18, + dayMinutes: duration); + state = + IncrementalMinuteMetrics.fromJson(checkpoint(state.toJson())); + } + } + } + } + }); + test('summary nonfinite HR/cadence preserves abstentions and finite energy', + () { + final f = energyDay(n: 91); + final keys = List.generate(f.hr.length, (i) => 28000000 + i * 2); + final p = energyProfiles['nonbinary']!; + final state = IncrementalMinuteMetrics(); + for (final n in prefixSizes(f.hr.length, 17)) { + final k = keys.sublist(0, n), + h = f.hr.sublist(0, n), + c = f.cadence.sublist(0, n); + _summaryClose( + state.sync(k, h, + cadenceSpm: c, + restingHr: 54, + maxHr: 186, + profile: p, + quietHrr: .12, + includeMinuteSeries: false), + h, + cadence: c, + profile: p); + } + }); + test('summary without profile still calculates clinical metrics', () { + final f = incrementalFixture(minutes: 31); + final state = IncrementalMinuteMetrics(); + _summaryClose( + state.sync(f.minuteKeys, f.hr, + restingHr: 54, + maxHr: 186, + quietHrr: .12, + includeMinuteSeries: false), + f.hr); + final work = state.processedMinutes; + minuteClose( + state.sync(f.minuteKeys, f.hr, + restingHr: 54, maxHr: 186, quietHrr: .12), + f.minuteKeys, + f.hr); + expect(state.processedMinutes, work); + final p = energyProfiles['male']!; + _summaryClose( + state.sync(f.minuteKeys, f.hr, + restingHr: 54, + maxHr: 186, + profile: p, + quietHrr: .12, + includeMinuteSeries: false), + f.hr, + profile: p); + }); + test('day duration reprices only basal, across short, DST and long days', () { + final f = incrementalFixture(seed: 5, minutes: 47); + final p = energyProfiles.values.first; + final state = IncrementalMinuteMetrics(); + MinuteMetrics sync(int dayMinutes) => state.sync(f.minuteKeys, f.hr, + cadenceSpm: f.cadence, + restingHr: 54, + maxHr: 186, + profile: p, + quietHrr: .12, + dayMinutes: dayMinutes, + includeMinuteSeries: false); + sync(1440); + final work = state.processedMinutes; + // Empty, one minute, an hour, both DST days, a whole and a double day, and + // a negative count (batch multiplies it through; parity, not validation). + for (final minutes in [0, 1, 60, 1380, 1439, 1440, 1441, 1500, 2880, -1]) { + _summaryClose(sync(minutes), f.hr, + cadence: f.cadence, profile: p, dayMinutes: minutes); + expect(state.processedMinutes, work, + reason: 'basal is a scalar over the day; no minute is repriced'); + } + }); +} diff --git a/test/onehz/minute_energy_test.dart b/test/onehz/minute_energy_test.dart new file mode 100644 index 00000000..b8e1e50f --- /dev/null +++ b/test/onehz/minute_energy_test.dart @@ -0,0 +1,360 @@ +// Per-minute energy: the SAME computation `Calories.dailyEnergy` does, exposed +// minute by minute. Three things are pinned here: +// +// 1. dailyEnergy's outputs did not move (GOLDEN literals captured from the +// implementation BEFORE the refactor; `==`, not closeTo). The consuming app +// versions every derived day on these numbers. +// 2. the minute series folds back to dailyEnergy BIT-FOR-BIT. +// 3. a minute with nothing measured abstains — null and a reason, never a +// filled or interpolated value. + +import 'package:test/test.dart'; +import 'package:openstrap_analytics/src/onehz/workout/calories.dart'; +import 'support/energy_day.dart'; + +const _hrmax = 186.0; +const _rhr = 54.0; // gate = 54 + 0.40 * (186 - 54) = 106.8 bpm + +// Captured from dailyEnergy at 7fe67a7 (before minuteEnergy existed), over +// `energyDay()` with hrmax 186 / restingHr 54 / dayMinutes 1440. +// (total, active, basal, walking) +const _golden = { + 'male false': (8251.412972070117, 6460.162972070118, 1791.25, 0.0), + 'male true': ( + 8854.354873246793, + 7063.104873246794, + 1791.25, + 602.9419011766767 + ), + 'female false': (5546.635132551552, 4258.885132551552, 1287.75, 0.0), + 'female true': ( + 5980.096923613808, + 4692.346923613808, + 1287.75, + 433.4617910622563 + ), + 'nonbinary false': ( + 6563.416347695368, + 5008.291347695368, + 1555.1249999999998, + 0.0 + ), + 'nonbinary true': ( + 7086.877631194254, + 5531.752631194254, + 1555.1249999999998, + 523.4612834988859 + ), +}; +// First 700 minutes, dayMinutes 900, male, with cadence. +const _goldenPartial = + (4497.553602919252, 3378.0223529192513, 1119.53125, 311.26779559338775); + +void main() { + final day = energyDay(); + + group('dailyEnergy is unchanged (golden)', () { + for (final k in energyProfiles.keys) { + for (final withCad in [false, true]) { + test('$k cadence=$withCad', () { + final e = Calories.dailyEnergy(day.hr, + profile: energyProfiles[k]!, + hrmax: _hrmax, + restingHr: _rhr, + cadenceSpmPerMin: withCad ? day.cadence : null)!; + final g = _golden['$k $withCad']!; + expect(e.total, g.$1); + expect(e.active, g.$2); + expect(e.basal, g.$3); + expect(e.walking, g.$4); + }); + } + } + test('partial day, dayMinutes pro-rates basal', () { + final e = Calories.dailyEnergy(day.hr.sublist(0, 700), + profile: energyProfiles['male']!, + hrmax: _hrmax, + restingHr: _rhr, + dayMinutes: 900, + cadenceSpmPerMin: day.cadence.sublist(0, 700))!; + expect(e.total, _goldenPartial.$1); + expect(e.active, _goldenPartial.$2); + expect(e.basal, _goldenPartial.$3); + expect(e.walking, _goldenPartial.$4); + }); + }); + + group('minuteEnergy folds back to dailyEnergy exactly', () { + for (final k in energyProfiles.keys) { + for (final withCad in [false, true]) { + test('$k cadence=$withCad', () { + final cad = withCad ? day.cadence : null; + final d = Calories.dailyEnergy(day.hr, + profile: energyProfiles[k]!, + hrmax: _hrmax, + restingHr: _rhr, + cadenceSpmPerMin: cad)!; + final s = Calories.minuteEnergy(day.hr, + profile: energyProfiles[k]!, + hrmax: _hrmax, + restingHr: _rhr, + cadenceSpmPerMin: cad)!; + expect(s.minutes.length, day.hr.length); + expect(s.active, d.active); + expect(s.walking, d.walking); + // basal is a rate × minutes, not a sum of 1440 additions. + expect(s.basalKcalPerMin * 1440, d.basal); + expect(s.basalKcalPerMin * 1440 + s.active, d.total); + }); + } + } + + test('the app path: zero-HR minutes dropped (with their cadence) before ' + 'dailyEnergy equals the aligned series with that cadence masked', () { + // lib/compute/derivation_engine.dart wakeDayEnergy compacts the series. + final hr = []; + final cad = []; + for (var i = 0; i < day.hr.length; i++) { + if (day.hr[i] <= 0) continue; + hr.add(day.hr[i]); + cad.add(day.cadence[i]); + } + final compact = Calories.dailyEnergy(hr, + profile: energyProfiles['female']!, + hrmax: _hrmax, + restingHr: _rhr, + cadenceSpmPerMin: cad)!; + final s = Calories.minuteEnergy(day.hr, + profile: energyProfiles['female']!, + hrmax: _hrmax, + restingHr: _rhr, + cadenceSpmPerMin: [ + for (var i = 0; i < day.hr.length; i++) + day.hr[i] <= 0 ? null : day.cadence[i] + ])!; + expect(s.active, compact.active); + expect(s.walking, compact.walking); + }); + + test('partial series', () { + final s = Calories.minuteEnergy(day.hr.sublist(0, 700), + profile: energyProfiles['male']!, + hrmax: _hrmax, + restingHr: _rhr, + cadenceSpmPerMin: day.cadence.sublist(0, 700))!; + expect(s.active, _goldenPartial.$2); + expect(s.walking, _goldenPartial.$4); + expect(s.basalKcalPerMin * 900, _goldenPartial.$3); + }); + }); + + group('per-minute rules', () { + const p = WorkoutUserProfile( + weightKg: 80, heightCm: 180, age: 30, sex: 'male'); + final basal = Calories.mifflinBmrKcalDay(80, 180, 30, 'male') / 1440.0; + + MinuteEnergySeries run(List hr, {List? cad, List? at}) => + Calories.minuteEnergy(hr, + profile: p, + hrmax: _hrmax, + restingHr: _rhr, + cadenceSpmPerMin: cad, + epochMinutes: at)!; + + test('resting HR: measured rest, basal only', () { + final m = run([70.0]).minutes.single; + expect(m.source, MinuteEnergySource.rest); + expect(m.abstained, isNull); + expect(m.basal, basal); + expect(m.active, 0.0); + expect(m.walking, isNull); + expect(m.total, basal); + }); + + test('HR over the gate: Keytel surplus over basal', () { + final coeffs = Calories.resolveCoeffs('male'); + final surplus = Calories.activeKcalPerS(coeffs, 150.0, _hrmax, 80, 30) * + 60.0 - + basal; + final m = run([150.0]).minutes.single; + expect(m.source, MinuteEnergySource.hr); + expect(m.active, surplus); + expect(m.walking, isNull); + expect(m.total, basal + surplus); + }); + + test('no usable HR and no cadence abstains: null everywhere + reason', () { + for (final bad in [0.0, -3.0, double.nan, double.infinity]) { + final m = run([bad]).minutes.single; + expect(m.abstained, MinuteEnergyAbstain.noHr, reason: '$bad'); + expect(m.source, isNull); + expect(m.basal, isNull); + expect(m.active, isNull); + expect(m.walking, isNull); + expect(m.total, isNull); + } + }); + + test('a minute is computed from its own inputs only (no fill, no smoothing)', + () { + final a = run([70.0, 0.0, 150.0]).minutes; + final b = run([70.0, 0.0, 120.0]).minutes; + expect(a[1].total, isNull); + expect(b[1].total, isNull); + expect(a[0].total, b[0].total); + expect(a[2].total, isNot(b[2].total)); + }); + + test('cadence on a sub-flex minute bills walking surplus', () { + final m = run([80.0], cad: [120.0]).minutes.single; // 5 METs + expect(m.source, MinuteEnergySource.cadence); + expect(m.walking, (5.0 - 1.0) * basal); + expect(m.active, m.walking); + expect(m.total, basal + m.walking!); + }); + + test('cadence on an HR-billed minute is not billed twice', () { + final m = run([150.0], cad: [120.0]).minutes.single; + expect(m.source, MinuteEnergySource.hr); + expect(m.walking, isNull); + }); + + test('cadence below the CADENCE-Adults floor bills nothing', () { + expect(run([80.0], cad: [95.0]).minutes.single.source, + MinuteEnergySource.rest); + expect(run([0.0], cad: [95.0]).minutes.single.abstained, + MinuteEnergyAbstain.noHr); + }); + + test('cadence with no HR is still measured gait (same as dailyEnergy)', () { + final m = run([0.0], cad: [110.0]).minutes.single; + expect(m.source, MinuteEnergySource.cadence); + expect(m.walking, (4.0 - 1.0) * basal); // 110 spm = 4 METs + }); + + test('minute carries its epoch minute when given, else its index', () { + expect(run([70.0, 70.0]).minutes.map((m) => m.minute), [0, 1]); + expect(run([70.0, 70.0], at: [29000000, 29000007]).minutes + .map((m) => m.minute), [29000000, 29000007]); + }); + + test('coverage counters', () { + final s = run([70.0, 0.0, 150.0, double.nan]); + expect(s.coveredMinutes, 2); + expect(s.abstainedMinutes, 2); + }); + }); + + group('input contract', () { + const p = WorkoutUserProfile(); + test('unusable anchors → null, like dailyEnergy', () { + for (final (mx, rest) in [ + (186.0, 0.0), + (186.0, 190.0), + (double.nan, 54.0), + (186.0, double.nan), + ]) { + expect( + Calories.minuteEnergy([100.0], + profile: p, hrmax: mx, restingHr: rest), + isNull); + } + }); + test('misaligned cadence / epochMinutes throw', () { + expect( + () => Calories.minuteEnergy([100.0, 100.0], + profile: p, + hrmax: _hrmax, + restingHr: _rhr, + cadenceSpmPerMin: [null]), + throwsArgumentError); + expect( + () => Calories.minuteEnergy([100.0, 100.0], + profile: p, + hrmax: _hrmax, + restingHr: _rhr, + epochMinutes: [1]), + throwsArgumentError); + }); + test('empty input → empty series, zero everything', () { + final s = Calories.minuteEnergy(const [], + profile: p, hrmax: _hrmax, restingHr: _rhr)!; + expect(s.minutes, isEmpty); + expect(s.active, 0.0); + }); + }); + + group('hourlyRollup', () { + final s = Calories.minuteEnergy(day.hr, + profile: energyProfiles['male']!, + hrmax: _hrmax, + restingHr: _rhr, + cadenceSpmPerMin: day.cadence)!; + + test('24 hourly buckets over a full index-aligned day', () { + final h = Calories.hourlyRollup(s.minutes); + expect(h.length, 24); + expect([for (final b in h) b.hour], List.generate(24, (i) => i)); + }); + + test('hour sums are the sums of its covered minutes, with coverage', () { + final h = Calories.hourlyRollup(s.minutes); + for (final b in h) { + var active = 0.0, total = 0.0, cov = 0; + for (var m = b.hour * 60; m < b.hour * 60 + 60; m++) { + final r = s.minutes[m]; + if (r.abstained != null) continue; + active += r.active!; + total += r.total!; + cov++; + } + expect(b.coveredMinutes, cov); + expect(b.coverage, cov / 60.0); + expect(b.active, active); + expect(b.total, total); + } + expect(h.fold(0, (a, b) => a + b.active!), + closeTo(s.active, 1e-9)); + }); + + test('an hour with no covered minute is null, not zero', () { + final hr = [for (var i = 0; i < 120; i++) i < 60 ? 0.0 : 70.0]; + final h = Calories.hourlyRollup(Calories.minuteEnergy(hr, + profile: energyProfiles['male']!, + hrmax: _hrmax, + restingHr: _rhr)! + .minutes); + expect(h[0].coveredMinutes, 0); + expect(h[0].coverage, 0.0); + expect(h[0].total, isNull); + expect(h[0].active, isNull); + expect(h[0].basal, isNull); + expect(h[1].coveredMinutes, 60); + expect(h[1].total, isNotNull); + }); + + test('partial hour is a partial sum, flagged by coverage, not scaled', () { + final r = Calories.minuteEnergy([for (var i = 0; i < 30; i++) 70.0], + profile: energyProfiles['male']!, hrmax: _hrmax, restingHr: _rhr)!; + final h = Calories.hourlyRollup(r.minutes); + expect(h.single.coveredMinutes, 30); + expect(h.single.coverage, 0.5); + expect(h.single.basal, closeTo(30 * r.basalKcalPerMin, 1e-9)); // not 60 + }); + + test('epoch minutes bucket by local hour via minuteOffset', () { + // epoch minute 600 = 10:00 UTC. +330 (UTC+5:30) → 15:30. + final at = [600, 601, 629, 630]; + final r = Calories.minuteEnergy([70.0, 70.0, 70.0, 70.0], + profile: energyProfiles['male']!, + hrmax: _hrmax, + restingHr: _rhr, + epochMinutes: at)!; + final utc = Calories.hourlyRollup(r.minutes); + expect([for (final b in utc) b.hour], [10]); + final ist = Calories.hourlyRollup(r.minutes, minuteOffset: 330); + expect([for (final b in ist) b.hour], [15, 16]); + expect([for (final b in ist) b.coveredMinutes], [3, 1]); + }); + }); +} diff --git a/test/onehz/nonfinite_beat_times_test.dart b/test/onehz/nonfinite_beat_times_test.dart new file mode 100644 index 00000000..fa090422 --- /dev/null +++ b/test/onehz/nonfinite_beat_times_test.dart @@ -0,0 +1,135 @@ +// A non-finite beat time used to make rsaRespRate and hrvFreq walk their +// sub-windows forever (an infinite last time is never reached), which ran a +// test isolate out of memory on the dev box. They abstain now, and the +// streams refuse such a fold and answer a snapshot with the batch. +// cvhrApneaScreen / CvhrStream follow the same rule: CvhrStream took a NaN +// time and its snapshot then threw where the batch answered. +// hrvTime (and nocturnalRmssd, sleepSessionRmssdDetail, hrvTimeAggregator, +// IncrementalHrvTime) threw on one: "Infinity or NaN toInt". +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void main() { + final nn = [for (var i = 0; i < 1500; i++) 900.0 + 40 * (i % 7)]; + final ts = []; + var t = 0.0; + for (final v in nn) { + t += v; + ts.add(t); + } + List withLast(double bad) => [...ts]..[ts.length - 1] = bad; + + for (final bad in [double.infinity, double.nan, double.negativeInfinity]) { + test('batch abstains on a last beat time of $bad', () { + final r = rsaRespRate(nn, withLast(bad), artifactFraction: 0); + expect(r.present, isFalse); + expect(r.note, 'non-finite beat time'); + final h = hrvFreq(nn, withLast(bad), artifactFraction: 0); + expect(h.present, isFalse); + expect(h.note, 'non-finite beat time'); + final c = cvhrApneaScreen(nn, withLast(bad), artifactFraction: 0); + expect(c.present, isFalse); + expect(c.note, 'non-finite beat time'); + }); + + test('streams refuse the fold, unchanged, on $bad', () { + final r = RsaRespStream()..fold(nn.sublist(0, 700), ts.sublist(0, 700)); + final h = HrvFreqStream()..fold(nn.sublist(0, 700), ts.sublist(0, 700)); + final c = CvhrStream()..fold(nn.sublist(0, 700), ts.sublist(0, 700)); + final rBefore = r.toJson(), hBefore = h.toJson(), cBefore = c.toJson(); + final badTs = [...ts.sublist(700, 800)]..[50] = bad; + expect(() => r.fold(nn.sublist(700, 800), badTs), throwsArgumentError); + expect(() => h.fold(nn.sublist(700, 800), badTs), throwsArgumentError); + expect(() => c.fold(nn.sublist(700, 800), badTs), throwsArgumentError); + expect(r.toJson(), rBefore); + expect(h.toJson(), hBefore); + expect(c.toJson(), cBefore); + }); + } + + test('a snapshot with an infinite tail time is the batch answer', () { + final r = RsaRespStream()..fold(nn.sublist(0, 1400), ts.sublist(0, 1400)); + final h = HrvFreqStream()..fold(nn.sublist(0, 1400), ts.sublist(0, 1400)); + final tailTs = withLast(double.infinity).sublist(1400); + final tailNn = nn.sublist(1400); + expect( + r.snapshot(tailNnMs: tailNn, tailNnTimesMs: tailTs, artifactFraction: 0) + .note, + 'non-finite beat time'); + expect( + h.snapshot(tailNnMs: tailNn, tailNnTimesMs: tailTs, artifactFraction: 0) + .note, + 'non-finite beat time'); + final c = CvhrStream()..fold(nn.sublist(0, 1400), ts.sublist(0, 1400)); + for (final bad in [double.infinity, double.nan]) { + final tail = withLast(bad).sublist(1400); + final got = + c.snapshot(tailNnMs: tailNn, tailNnTimesMs: tail, artifactFraction: 0); + expect(got.note, 'non-finite beat time'); + expect(got.note, + cvhrApneaScreen(nn, withLast(bad), artifactFraction: 0).note); + } + }); + + group('hrvTime family', () { + for (final bad in [double.infinity, double.nan, double.negativeInfinity]) { + test('hrvTime abstains on a beat time of $bad', () { + for (final at in [ts.length - 1, 0, 700]) { + final t = [...ts]..[at] = bad; + final m = hrvTime(nn, nnTimesMs: t); + expect(m.present, isFalse); + expect(m.note, 'non-finite beat time'); + } + // Without times nothing about them is read. + expect(hrvTime(nn).present, isTrue); + }); + + test('nocturnalRmssd abstains on a beat time of $bad', () { + final m = nocturnalRmssd(nn, withLast(bad)); + expect(m.present, isFalse); + expect(m.note, 'non-finite beat time'); + }); + + test('sleepSessionRmssdDetail abstains on a beat time of $bad', () { + final start = 1700000000; + final abs = [for (final x in ts) start * 1000.0 + x]; + abs[abs.length - 1] = bad; + final m = sleepSessionRmssdDetail(nn, abs, + startSec: start, endSec: start + 3 * 3600); + expect(m.present, isFalse); + expect(m.note, 'non-finite beat time'); + }); + + test('hrvTimeAggregator refuses the slice, state unchanged, on $bad', + () { + final agg = hrvTimeAggregator(originMs: ts.first); + Beats beats(List t) => Beats( + Float64List.fromList(nn.sublist(0, t.length)), + Float64List.fromList(t)); + final state = agg.fold(beats(ts.sublist(0, 700))); + final before = agg.encodeState(state); + final badTs = [...ts.sublist(0, 100)]..[50] = bad; + expect(() => agg.addSlice(state, beats(badTs), 0, badTs.length), + throwsArgumentError); + expect(agg.encodeState(state), before); + expect(() => agg.fold(beats(badTs)), throwsArgumentError); + // Times are ignored when not gap-aware, so NaN stays fine there. + final plain = hrvTimeAggregator(gapAware: false); + expect(() => plain.fold(beats(badTs)), returnsNormally); + }); + + test('IncrementalHrvTime snapshot with a tail time of $bad is the batch', + () { + final inc = IncrementalHrvTime(); + inc.sync(nn.sublist(0, 1400), nnTimesMs: ts.sublist(0, 1400)); + final got = inc.sync(nn, nnTimesMs: withLast(bad)); + expect(got.present, isFalse); + expect(got.note, 'non-finite beat time'); + expect(got.note, hrvTime(nn, nnTimesMs: withLast(bad)).note); + }); + } + }); +} diff --git a/test/onehz/phase0_parity_test.dart b/test/onehz/phase0_parity_test.dart new file mode 100644 index 00000000..aa786f03 --- /dev/null +++ b/test/onehz/phase0_parity_test.dart @@ -0,0 +1,1047 @@ +// PHASE 0 PARITY (round 15) - characterization tests. +// +// The `_ref*` functions below are FROZEN verbatim copies of the code as it +// stood before round 15 (CVHR baseline loop, 1 g reference loop, Welch segment +// scan, and the callers around them). The production code is being refactored +// for speed with a bit-identical contract, so every comparison here is exact: +// doubles with `identical`-style equality (== or both NaN), every result field. +// +// Do NOT "fix" or tidy the reference copies. If production output changes, this +// test is supposed to fail. +// +// Only deviation from a straight copy: the cardio stager reference calls the +// PUBLIC `cleanRrBeatsBetween` / `remFeaturesFromBeats` / `classifyCardioEpochs` +// where the original called private wrappers around them (same bodies). + +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +// ───────────────────────── frozen CVHR reference ───────────────────────────── + +/// [p25, p50, p75] of [xs], or null when empty. Same linear-interpolated +/// percentile the rest of the package uses. +List? _refQuartiles(List xs) => xs.isEmpty + ? null + : [percentile(xs, 25)!, percentile(xs, 50)!, percentile(xs, 75)!]; + +Metric refCvhrApneaScreen( + List nnMs, + List nnTimesMs, { + required double artifactFraction, + double minWidthSec = 10, + double maxWidthSec = 120, + double depthWidthRatioMin = 0.7, // ms per second + double envWindowSec = 130, + double maxArtifact = 0.30, + double maxGapSec = 30, +}) { + const inputs = ['rr_cleaned', 'beat_times']; + if (nnMs.length < 60 || nnTimesMs.length != nnMs.length) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'too few beats for a CVHR screen (need ≥60)', + ); + } + if (artifactFraction > maxArtifact) { + return Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'artifact fraction ${round6(artifactFraction)} > gate ' + '— CVHR dip detection unreliable', + ); + } + + final tSec = [for (final t in nnTimesMs) t / 1000.0]; + if (tSec.last - tSec.first <= 0) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'degenerate beat times', + ); + } + + // SEGMENT AT GAPS. Beats more than [maxGapSec] apart are two recordings, not + // one: interpolating a straight line across the hole invents a flat stretch + // with no cycles in it, and charging the hole to `analyzedHours` divides a + // real cycle count by time we never observed. Both push the index DOWN, so a + // charging break used to read as an improvement. + final segStart = [0]; + for (var i = 1; i < tSec.length; i++) { + if (tSec[i] - tSec[i - 1] > maxGapSec) segStart.add(i); + } + + var cycleCount = 0; + var analyzedHours = 0.0; + final depths = []; // ms + final widths = []; // s + for (var s = 0; s < segStart.length; s++) { + final lo = segStart[s]; + final hi = (s + 1 < segStart.length ? segStart[s + 1] : tSec.length) - 1; + final segT = tSec.sublist(lo, hi + 1); + final segNn = nnMs.sublist(lo, hi + 1); + final spanSec = segT.last - segT.first; + // Too short to hold even one cycle plus its baseline window — it also can't + // be scored, so it contributes neither cycles nor hours. + if (segT.length < 60 || spanSec < 60) continue; + analyzedHours += spanSec / 3600.0; + _refScoreSegment( + segT, + segNn, + minWidthSec: minWidthSec, + maxWidthSec: maxWidthSec, + depthWidthRatioMin: depthWidthRatioMin, + envWindowSec: envWindowSec, + onCycle: (depth, width) { + cycleCount++; + depths.add(depth); + widths.add(width); + }, + ); + } + + if (analyzedHours <= 0) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'no gap-free stretch long enough for a CVHR screen (<60 s)', + ); + } + + final perHour = cycleCount / analyzedHours; + // Target main 9db83ce scales confidence and tier by observed duration. + final coverage = (analyzedHours / minAnalyzedHoursPerNight).clamp(0.0, 1.0); + final conf = ((1 - artifactFraction) * 0.85 * coverage).clamp(0.01, 0.85); + final tier = analyzedHours < minAnalyzedHoursPerNight ? Tier.estimate : Tier.high; + return Metric( + value: CvhrResult( + cycleCount: cycleCount, + cvhrPerHour: perHour, + analyzedHours: analyzedHours, + meanDepthMs: depths.isEmpty ? null : mean(depths), + meanWidthSec: widths.isEmpty ? null : mean(widths), + depthQuartilesMs: _refQuartiles(depths), + widthQuartilesSec: _refQuartiles(widths), + ), + confidence: conf, + tier: tier, + inputs_used: inputs, + note: 'CVHR/ACAT (Hayano) apnea SCREEN — NOT a diagnosis, NOT an AHI; ' + 'single-night CVHR has substantial night-to-night variability, ' + 'interpret as a trend over multiple nights', + ); +} + +/// Score ONE gap-free stretch: resample → smooth → sliding-median baseline → +/// positive-excursion runs → width + steepness gates. Calls [onCycle] with +/// (depthMs, widthSec) per surviving cycle. [t] in seconds, [nn] in ms. +void _refScoreSegment( + List t, + List nn, { + required double minWidthSec, + required double maxWidthSec, + required double depthWidthRatioMin, + required double envWindowSec, + required void Function(double depthMs, double widthSec) onCycle, +}) { + // Resample NN onto a uniform 1 Hz grid by piecewise-linear interpolation of + // the tachogram (NN vs beat time). This gives evenly-spaced points for the + // sliding-window baseline and the polynomial smoother. Only ever called on a + // stretch with no gap larger than the caller's tolerance, so no interpolation + // here spans a hole. + final t0 = t.first; + final nGrid = (t.last - t0).floor() + 1; + final grid = List.generate(nGrid, (i) => t0 + i); + final nnGrid = _refResampleLinear(t, nn, grid); + + // 2nd-order local-polynomial smoothing (Savitzky-Golay quadratic, ~7 s half + // window) to suppress beat jitter while keeping the cyclic shape. + final smooth = _refSavgolQuadratic(nnGrid, halfWin: 7); + + // Adaptive baseline over a ~130 s sliding median (spans ≥2 apnea cycles, so + // it tracks the eupneic level without being pulled into individual dips). + // The CVHR bradycardia shows up as a local NN MAXIMUM (NN ↑ ⇔ HR ↓) standing + // proud of this baseline; the following tachycardia is the recovery valley. + final halfEnv = (envWindowSec / 2).round(); + final base = List.filled(nGrid, 0); + for (var j = 0; j < nGrid; j++) { + final lo = math.max(0, j - halfEnv); + final hi = math.min(nGrid - 1, j + halfEnv); + base[j] = median(smooth.sublist(lo, hi + 1)) ?? smooth[j]; + } + // Baseline-subtracted bradycardic excursion (clip negatives — we only score + // the NN-up bradycardia, not the tachycardia trough). + final exc = [ + for (var j = 0; j < nGrid; j++) math.max(0.0, smooth[j] - base[j]) + ]; + + // Prominence threshold: a dip must clear a robust noise floor AND a fraction + // of the typical excursion amplitude. We anchor it BELOW the median positive + // excursion (Hayano's adaptive amplitude criterion is permissive enough to + // capture the whole bradycardia run, not just its tip) — a threshold at the + // peak would clip the run width to a few seconds and miss the cycle. + final posExc = [ + for (final e in exc) + if (e > 0) e + ]; + final excP75 = (posExc.isNotEmpty ? percentile(posExc, 75) : null) ?? 0; + // half of the upper-quartile excursion: well inside each genuine dip's run. + final prom = math.max(0.5 * excP75, 5.0); // ms + + // Find peaks of `exc`: local maxima exceeding `prom`, each isolated to one + // contiguous super-threshold run (so one bradycardia = one cycle). Measure + // the run width and the peak depth; apply the width + depth/width gates. + var i = 0; + while (i < nGrid) { + if (exc[i] <= prom) { + i++; + continue; + } + final start = i; + var peakDepth = 0.0; + while (i < nGrid && exc[i] > prom) { + if (exc[i] > peakDepth) peakDepth = exc[i]; + i++; + } + final widthSec = (i - start).toDouble(); // 1 Hz grid => samples == seconds + if (widthSec < minWidthSec || widthSec > maxWidthSec) continue; + // Depth-to-width steepness gate (ms per second): the bradycardia-tachycardia + // swing is steep; slow baseline drift is shallow per second. + if (peakDepth / widthSec < depthWidthRatioMin) continue; + onCycle(peakDepth, widthSec); + } +} + +/// Piecewise-linear resample of (t,y) onto a sorted [grid] of times (same unit). +/// Out-of-range grid points clamp to the nearest endpoint value. +List _refResampleLinear( + List t, List y, List grid) { + final out = List.filled(grid.length, 0); + var j = 0; + for (var k = 0; k < grid.length; k++) { + final g = grid[k]; + if (g <= t.first) { + out[k] = y.first; + continue; + } + if (g >= t.last) { + out[k] = y.last; + continue; + } + while (j < t.length - 1 && t[j + 1] < g) { + j++; + } + final t0 = t[j], t1 = t[j + 1]; + final span = t1 - t0; + final frac = span == 0 ? 0.0 : (g - t0) / span; + out[k] = y[j] + (y[j + 1] - y[j]) * frac; + } + return out; +} + +/// 2nd-order Savitzky-Golay smoothing via a local quadratic least-squares fit +/// over a symmetric window of half-width [halfWin]. Evaluated at the centre. +List _refSavgolQuadratic(List x, {int halfWin = 7}) { + final n = x.length; + final out = List.filled(n, 0); + for (var i = 0; i < n; i++) { + final lo = math.max(0, i - halfWin); + final hi = math.min(n - 1, i + halfWin); + final m = hi - lo + 1; + if (m < 3) { + out[i] = x[i]; + continue; + } + // Fit y = a + b·u + c·u² with u centred at i; return a (value at u=0). + var s0 = 0.0, s1 = 0.0, s2 = 0.0, s3 = 0.0, s4 = 0.0; + var sy = 0.0, suy = 0.0, su2y = 0.0; + for (var k = lo; k <= hi; k++) { + final u = (k - i).toDouble(); + final u2 = u * u; + s0 += 1; + s1 += u; + s2 += u2; + s3 += u2 * u; + s4 += u2 * u2; + sy += x[k]; + suy += u * x[k]; + su2y += u2 * x[k]; + } + // Solve the 3x3 normal equations for [a,b,c]; we only need a. + final a = _refSolve3( + [s0, s1, s2, s1, s2, s3, s2, s3, s4], + [sy, suy, su2y], + ); + out[i] = a == null ? x[i] : a[0]; + } + return out; +} + +/// Solve a 3x3 linear system (row-major A, rhs b) via Cramer's rule. Null if +/// singular. +List? _refSolve3(List a, List b) { + double det3(double a0, double a1, double a2, double a3, double a4, double a5, + double a6, double a7, double a8) => + a0 * (a4 * a8 - a5 * a7) - + a1 * (a3 * a8 - a5 * a6) + + a2 * (a3 * a7 - a4 * a6); + final d = det3(a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8]); + if (d == 0) return null; + final dx = det3(b[0], a[1], a[2], b[1], a[4], a[5], b[2], a[7], a[8]); + final dy = det3(a[0], b[0], a[2], a[3], b[1], a[5], a[6], b[2], a[8]); + final dz = det3(a[0], a[1], b[0], a[3], a[4], b[1], a[6], a[7], b[2]); + return [dx / d, dy / d, dz / d]; +} + +// ───────────────────────── frozen hrvFreq / Welch reference ────────────────── + +Metric refHrvFreq( + List nnMs, + List nnTimesMs, { + required double artifactFraction, + double hfArtifactGate = 0.15, + double oversample = 4.0, +}) { + const inputs = ['rr_cleaned', 'beat_times']; + if (nnMs.length < 16 || nnTimesMs.length != nnMs.length) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'too few beats for a spectral estimate', + ); + } + + // Times in seconds for Hz output; tachogram value = NN (ms). + final tSec = [for (final t in nnTimesMs) t / 1000.0]; + final spanSec = tSec.last - tSec.first; + if (spanSec <= 0) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'degenerate beat times', + ); + } + + // Each band gets its own segment length and its own resolution-matched grid. + // A band whose lowest frequency the record is too short to resolve returns + // null — absent, not a leakage-filled 0.0 (ULF used to be emitted as exactly + // 0.0 for any session over 333 s, which is not a 24-h record by any reading). + final ulf = + _refWelchBandPower(tSec, nnMs, 0.0003, 0.003, oversample: oversample); + final vlf = _refWelchBandPower(tSec, nnMs, 0.003, 0.04, oversample: oversample); + final lf = _refWelchBandPower(tSec, nnMs, 0.04, 0.15, oversample: oversample); + final hfRaw = _refWelchBandPower(tSec, nnMs, 0.15, 0.40, oversample: oversample); + if (lf == null && hfRaw == null) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'record too short to resolve any HRV band', + ); + } + + final hfGated = artifactFraction > hfArtifactGate; + final hf = hfGated ? null : hfRaw; + + double? lfhf, nuLf, nuHf; + if (lf != null && hf != null && (lf + hf) > 0) { + lfhf = hf == 0 ? null : lf / hf; + nuLf = 100.0 * lf / (lf + hf); + nuHf = 100.0 * hf / (lf + hf); + } + // TOTAL POWER is by definition the sum over ALL bands (Task Force 1996). + // When HF is suppressed we do not have a trustworthy HF term, so a "total" + // is not computable: silently summing the withheld hfRaw back in republished + // exactly the quantity the gate withheld (the gated and ungated totals came + // out bit-identical), and dropping HF from the sum would republish a + // different quantity under the same name. Either way it would be dishonest — + // so total is WITHHELD alongside HF. + // + // A band the record could not RESOLVE is a different case, and it used to be + // handled with `?? 0` — which is exactly the absence-as-zero this package + // forbids, and it is not hypothetical: ULF needs 33 333 s of span (10 cycles + // at 0.0003 Hz) and a night is ~28 700 s, so ULF resolved on precisely no + // nights and the published total was silently the ULF-less sum. Withholding + // total instead would delete a rendered number on every night forever. So it + // is published with its composition NAMED in [totalBands]: the sum is over + // those bands and no others. + final bands = { + 'ulf': ulf, + 'vlf': vlf, + 'lf': lf, + 'hf': hfRaw + }; + final resolved = [ + for (final e in bands.entries) + if (e.value != null) e.key + ]; + final total = (hfGated || lf == null || hfRaw == null) + ? null + : [for (final b in resolved) bands[b]!].reduce((a, b) => a + b); + + // Confidence: penalize artifacts heavily; low-band-only reads still HIGH-ish. + final conf = ((1 - artifactFraction) * (hfGated ? 0.6 : 0.9)).clamp(0.2, 0.9); + return Metric( + value: HrvFreq( + ulf: ulf, + vlf: vlf, + lf: lf, + hf: hf, + total: total, + totalBands: total == null ? null : resolved, + lfhf: lfhf, + nuLf: nuLf, + nuHf: nuHf, + hfGated: hfGated, + ), + confidence: conf, + tier: Tier.high, + inputs_used: inputs, + note: hfGated + ? 'HF suppressed: artifact fraction ${round6(artifactFraction)} ' + '> gate — LF/VLF reported, HF/LF-HF/nu/total withheld' + : 'PRV spectrum; HF band quantization-limited at 1 Hz', + ); +} + +double? _refWelchBandPower( + List tSec, + List y, + double loHz, + double hiHz, { + double cyclesPerSegment = 10.0, + double oversample = 4.0, + int minPointsPerSegment = 16, +}) { + if (loHz <= 0 || hiHz <= loHz || tSec.length < minPointsPerSegment) + return null; + final span = tSec.last - tSec.first; + final segSec = cyclesPerSegment / loHz; + if (span < segSec) return null; // band not resolvable in this record + + final df = 1.0 / segSec / oversample; + final nGrid = ((hiHz - loHz) / df).ceil() + 1; + final grid = freqGrid(loHz, hiHz, nGrid); + + var sum = 0.0; + var k = 0; + final step = segSec / 2; // 50 % overlap, the Welch default + for (var start = tSec.first; start + segSec <= tSec.last; start += step) { + final end = start + segSec; + final ts = []; + final ys = []; + for (var i = 0; i < tSec.length; i++) { + if (tSec[i] < start) continue; + if (tSec[i] >= end) break; + ts.add(tSec[i]); + ys.add(y[i]); + } + // A segment has to be BOTH beat-dense and time-complete: a dropout in + // the middle leaves few beats spanning the full window, and its + // periodogram is a window function, not a spectrum. Endpoint span alone + // misses an internal dropout that still leaves beats near both edges, so + // also reject on the single largest gap between consecutive beats. + if (ts.length < minPointsPerSegment || ts.last - ts.first < segSec * 0.8) { + continue; + } + var maxGap = 0.0; + for (var i = 1; i < ts.length; i++) { + final g = ts[i] - ts[i - 1]; + if (g > maxGap) maxGap = g; + } + if (maxGap > segSec * 0.2) continue; + final ls = lombScargle(ts, ys, grid); + if (ls == null) continue; + final p = ls.bandPower(loHz, hiHz); + if (!p.isFinite) continue; + sum += p; + k++; + } + return k == 0 ? null : sum / k; +} + +// ───────────────────────── frozen cardio stager reference ──────────────────── + +CardioStagerResult refCardioStager( + List hr1hz, + List accel, { + List rrMs = const [], + List rrTsMs = const [], + int epochSec = 30, + SleepUserProfile? userProfile, +}) { + // Explicit arg wins (unit tests); else the ambient profile the edge set for + // this staging pass; else null ⇒ pure per-night-local (cold-start behavior). + final profile = userProfile ?? cardioUserProfile; + // `_cleanBeatsInWindow` binary-searches this, so it has to be non-decreasing. + // The producer guarantees it — beats inherit their record's second and the + // records are sorted, and the one path that folds in loose live beats + // re-sorts the pair explicitly. An `assert` rather than a linear fallback: + // a fallback would pay the scan back on every window to insure against a + // state the caller cannot reach, whereas this is free in release and loud in + // every test the moment somebody builds an unsorted series. + assert(() { + for (var i = 1; i < rrTsMs.length; i++) { + if (rrTsMs[i] < rrTsMs[i - 1]) return false; + } + return true; + }(), 'rrTsMs must be non-decreasing — the beat window is binary-searched'); + final n = math.min(hr1hz.length, accel.length); + // ── REAL-TIME epoch grid ────────────────────────────────────────────────── + // `n ~/ epochSec` conflated ARRAY POSITIONS with SECONDS. At 1 Hz they are + // the same number and everything below is unchanged; off 1 Hz they are not, + // and the failure was not an accuracy one — a 300 s band gave + // `96 ~/ 30 = 3` "epochs" of 2.5 REAL HOURS each, which the 30-s-tuned rules + // labelled NREM end to end and published as `wakePct = 0.0`: a night this + // stager could not read, reported as zero wake. No new parameter is needed + // for the fix — `accel[i].tsMs` is already the clock `_cleanBeatsInWindow` + // binary-searches, so the cadence is measurable from the input we have. + final cadenceSec = sampleCadenceSeconds( + [for (var i = 0; i < n; i++) accel[i].tsMs / 1000.0]); + // Coarser than one sample per epoch ⇒ there is no 30-s epoch to score, and + // the Webster/Cole-Kripke continuity rules below are specified at this epoch + // length. Stretching `epochSec` to fit the device would make a number appear + // where the evidence for it does not; abstain instead. + if (cadenceSec == null || cadenceSec > epochSec) return _refAbstain(epochSec); + final perEpoch = math.max(1, (epochSec / cadenceSec).round()); + // `perEpoch` samples at this cadence must actually SPAN `epochSec` — a + // cadence that does not divide it evenly (4 s against a 30 s epoch rounds + // `perEpoch` to 8, an 8 * 4 = 32 s epoch) drifts the real grid against the + // reported one. `epochSec` is not just a label: it is what the result + // reports (`CardioStagerResult.epochSec`) and what `consolidateSleepStages` + // / `_websterRescore` / `_mergeShortDeep` derive the Webster and bout + // thresholds from below, and the drift accumulates over the whole night. + // Abstain rather than publish a grid whose real spacing does not match its + // own label. + if ((perEpoch * cadenceSec - epochSec).abs() > 1e-6) + return _refAbstain(epochSec); + final nEpoch = n ~/ perEpoch; + if (nEpoch < 3) return _refAbstain(epochSec); + + // ── per-second ENMO (motion) against a LOCALLY-ADAPTIVE 1 g reference ────── + // A single whole-night gravity-magnitude reference (the old approach) is + // wrong on real WHOOP-4 units: the decoded gravity vector's magnitude is NOT + // perfectly orientation-invariant (per-axis gain/calibration isn't exact), + // so different STATIC sleep postures can read meaningfully apart in |accel| + // even though nothing is moving. Verified on a real overnight capture: a + // person lying rock-still for 30+ min (within-epoch stddev of |accel| < + // 0.0003 g) read |accel| = 1.0512 g against that whole-night's calibrated + // reference of 1.0348 g — a 0.0167 g "motion" score, ~3x the bigMove + // threshold, purely from holding a different (but equally static) posture + // than whichever one the night happened to calibrate against. Across that + // night, 389 of 421 "big move" epochs (92%) were this exact artifact (tiny + // within-epoch variance, large offset from the single global reference), + // not real movement — and the resulting misclassified WAKE blocks could not + // be fully bridged back by Webster rescore below. Recomputing the reference + // locally (a window wide enough to not react to real short movement bouts, + // narrow enough to track a genuine posture change within a few minutes) + // absorbs each posture as its own baseline instead of misreading it as + // sustained motion. + final mag = List.filled(n, 0); + for (var i = 0; i < n; i++) { + final a = accel[i]; + mag[i] = math.sqrt(a.x * a.x + a.y * a.y + a.z * a.z); + } + const int _gRefWinSec = 300; // 5 min window, centered per epoch + // …in SAMPLES at the measured cadence. 300 at 1 Hz, unchanged. + final gRefHalf = math.max(1, (_gRefWinSec / cadenceSec).round()) ~/ 2; + final gRefByEpoch = List.filled(nEpoch, 1.0); + for (var e = 0; e < nEpoch; e++) { + final es = e * perEpoch; + final lo = math.max(0, es - gRefHalf); + final hi = math.min(n, es + perEpoch + gRefHalf); + gRefByEpoch[e] = median(mag.sublist(lo, hi)) ?? 1.0; + } + + // ── per-epoch features ───────────────────────────────────────────────────── + final motion = List.filled(nEpoch, 0); + final hr = List.filled(nEpoch, double.nan); + final hrSd = List.filled(nEpoch, 0); + final rmssd = List.filled(nEpoch, double.nan); + final sdnn = List.filled(nEpoch, double.nan); + // REM autonomic features (P1): LF/HF from the RR spectrum + R(k) = rolling + // mean |ΔIHR|. REM's signature at 1 Hz beat-timing is a SYMPATHETIC shift — + // LF/HF rises, RR variability drops (low RMSSD), and instantaneous HR gets + // "jittery" (elevated |ΔIHR|). The old REM rule leaned only on the RMSSD drop + // and under-called REM; these two independent axes recover it (OR-combined + // below), gated by atonia + an HR floor so a plain arousal is not miscalled. + final lfhf = List.filled(nEpoch, double.nan); + final rk = List.filled(nEpoch, double.nan); + + for (var e = 0; e < nEpoch; e++) { + final s = e * perEpoch, t = math.min(s + perEpoch, n); + // motion = mean ENMO over the epoch, against THIS epoch's local reference. + final gRefE = gRefByEpoch[e]; + var ms = 0.0; + for (var i = s; i < t; i++) { + final d = mag[i] - gRefE; + ms += d > 0 ? d : 0.0; + } + motion[e] = ms / (t - s); + // hr mean/sd over valid (>0) seconds + final hv = [ + for (var i = s; i < t; i++) + if (hr1hz[i] > 0) hr1hz[i] + ]; + if (hv.isNotEmpty) hr[e] = mean(hv)!; + if (hv.length >= 2) hrSd[e] = stddev(hv) ?? 0; + // rmssd over RR beats within a ±2.5-min window centred on the epoch + rmssd[e] = _refWindowRmssd(rrMs, rrTsMs, accel, s, t, epochSec); + sdnn[e] = _refWindowSdnn(rrMs, rrTsMs, accel, s, t, epochSec); + // LF/HF + R(k) over a ±90-s RR window centred on the epoch. + final rem = _refWindowRemFeatures(rrMs, rrTsMs, accel, s, t, epochSec); + if (rem.lfhf != null) lfhf[e] = rem.lfhf!; + if (rem.rk != null) rk[e] = rem.rk!; + } + + return classifyCardioEpochs( + CardioEpochFeatures( + motion: motion, + hr: hr, + hrSd: hrSd, + rmssd: rmssd, + lfhf: lfhf, + rk: rk, + sdnn: sdnn, + ), + epochSec: epochSec, + userProfile: profile, + ); +} + +CardioStagerResult _refAbstain(int epochSec) => CardioStagerResult( + StagerResult( + stages: const [], + epochSec: epochSec, + wakePct: 0, + nremPct: 0, + remPct: 0, + ), + const [], + 0, + ); + +({List beats, List tsSec, List seam}) + _refCleanBeatsInWindow( + List rrMs, + List rrTsMs, + List accel, + int s, + int t, { + int halfWinMs = 150 * 1000, +}) { + const empty = (beats: [], tsSec: [], seam: []); + if (rrMs.isEmpty || rrTsMs.length != rrMs.length) return empty; + final mid = (s + t) ~/ 2; + if (mid >= accel.length) return empty; + final centreMs = accel[mid].tsMs; + return cleanRrBeatsBetween( + rrMs, rrTsMs, centreMs - halfWinMs, centreMs + halfWinMs); +} + +double _refWindowRmssd(List rrMs, List rrTsMs, + List accel, int s, int t, int epochSec) { + // Upstream's seam rule (merged 2026-10-18): skip seam pairs, gate on pairs. + final win = _refCleanBeatsInWindow(rrMs, rrTsMs, accel, s, t); + final beats = win.beats; + if (beats.length < 5) return double.nan; + var ss = 0.0; + var pairs = 0; + for (var i = 1; i < beats.length; i++) { + if (win.seam[i]) continue; + final d = beats[i] - beats[i - 1]; + ss += d * d; + pairs++; + } + return pairs >= 4 ? math.sqrt(ss / pairs) : double.nan; +} + +double _refWindowSdnn(List rrMs, List rrTsMs, + List accel, int s, int t, int epochSec) { + final beats = _refCleanBeatsInWindow(rrMs, rrTsMs, accel, s, t).beats; + if (beats.length < 5) return double.nan; + return stddev(beats) ?? double.nan; +} + +({double? lfhf, double? rk}) _refWindowRemFeatures(List rrMs, + List rrTsMs, List accel, int s, int t, int epochSec) { + final win = + _refCleanBeatsInWindow(rrMs, rrTsMs, accel, s, t, halfWinMs: 90 * 1000); + return remFeaturesFromBeats(win.beats, win.tsSec, win.seam); +} + +// ───────────────────────────── helpers ─────────────────────────────────────── + +bool sameD(double? a, double? b) { + if (a == null || b == null) return a == null && b == null; + if (a.isNaN || b.isNaN) return a.isNaN && b.isNaN; + return a == b && a.isNegative == b.isNegative; +} + +void expectSameD(double? a, double? b, String what) { + expect(sameD(a, b), isTrue, reason: '$what: $a vs $b'); +} + +void expectSameList(List? a, List? b, String what) { + if (a == null || b == null) { + expect(a == null && b == null, isTrue, reason: '$what nullness'); + return; + } + expect(a.length, b.length, reason: '$what length'); + for (var i = 0; i < a.length; i++) { + if (!sameD(a[i], b[i])) fail('$what[$i]: ${a[i]} vs ${b[i]}'); + } +} + +void expectSameMetric(Metric a, Metric b) { + expect(a.value == null, b.value == null, reason: 'value presence'); + expectSameD(a.confidence, b.confidence, 'confidence'); + expect(a.tier, b.tier); + expect(a.inputs_used, b.inputs_used); + expect(a.note, b.note); + expect(a.drivers == null, b.drivers == null); +} + +void expectSameCvhr(Metric a, Metric b) { + expectSameMetric(a, b); + final x = a.value, y = b.value; + if (x == null || y == null) return; + expect(x.cycleCount, y.cycleCount); + expectSameD(x.cvhrPerHour, y.cvhrPerHour, 'cvhrPerHour'); + expectSameD(x.analyzedHours, y.analyzedHours, 'analyzedHours'); + expectSameD(x.meanDepthMs, y.meanDepthMs, 'meanDepthMs'); + expectSameD(x.meanWidthSec, y.meanWidthSec, 'meanWidthSec'); + expectSameList(x.depthQuartilesMs, y.depthQuartilesMs, 'depthQuartiles'); + expectSameList(x.widthQuartilesSec, y.widthQuartilesSec, 'widthQuartiles'); +} + +void expectSameHrv(Metric a, Metric b) { + expectSameMetric(a, b); + final x = a.value, y = b.value; + if (x == null || y == null) return; + expectSameD(x.ulf, y.ulf, 'ulf'); + expectSameD(x.vlf, y.vlf, 'vlf'); + expectSameD(x.lf, y.lf, 'lf'); + expectSameD(x.hf, y.hf, 'hf'); + expectSameD(x.total, y.total, 'total'); + expect(x.totalBands, y.totalBands); + expectSameD(x.lfhf, y.lfhf, 'lfhf'); + expectSameD(x.nuLf, y.nuLf, 'nuLf'); + expectSameD(x.nuHf, y.nuHf, 'nuHf'); + expect(x.hfGated, y.hfGated); +} + +void expectSameStager(CardioStagerResult a, CardioStagerResult b) { + expect(a.base.stages, b.base.stages); + expect(a.base.epochSec, b.base.epochSec); + expectSameD(a.base.wakePct, b.base.wakePct, 'wakePct'); + expectSameD(a.base.nremPct, b.base.nremPct, 'nremPct'); + expectSameD(a.base.remPct, b.base.remPct, 'remPct'); + expect(a.deepFlag, b.deepFlag); + expectSameD(a.confidence, b.confidence, 'confidence'); +} + +// ───────────────────────────── generators ──────────────────────────────────── + +/// Beat series: NN with a ~[period]-s cyclic dip pattern plus noise. +/// gapEvery: mean beats between >30 s holes (0 = none) +/// nanFrac / dupFrac: fraction of NaN NN values / repeated timestamps +({List nn, List tMs}) genBeats( + int seed, + int n, { + double period = 55, + double amp = 60, + int gapEvery = 0, + double nanFrac = 0, + double dupFrac = 0, + double meanNn = 900, +}) { + final r = math.Random(seed); + final nn = []; + final t = []; + var tMs = 1000.0 * r.nextInt(1000); + for (var i = 0; i < n; i++) { + final ph = 2 * math.pi * (tMs / 1000.0) / period; + var v = meanNn + + amp * math.max(0.0, math.sin(ph)) - + 0.3 * amp * math.min(0.0, math.sin(ph)) + + (r.nextDouble() - 0.5) * 40; + if (nanFrac > 0 && r.nextDouble() < nanFrac) v = double.nan; + nn.add(v); + final isDup = dupFrac > 0 && i > 0 && r.nextDouble() < dupFrac; + if (!isDup) { + var step = v.isNaN ? meanNn : v; + if (gapEvery > 0 && r.nextInt(gapEvery) == 0) { + step += 31000 + r.nextInt(90000); + } + tMs += step; + } + t.add(tMs); + } + return (nn: nn, tMs: t); +} + +({List hr, List accel, List rr, List rrTs}) + genNight( + int seed, + int seconds, { + int cadenceSec = 1, + bool nanAccel = false, + bool hrDropouts = false, + bool rrOutliers = false, + bool withRr = true, +}) { + final r = math.Random(seed); + final n = seconds ~/ cadenceSec; + final hr = []; + final accel = []; + final rr = []; + final rrTs = []; + var posture = 1.0; + var moveLeft = 0; + var hrBase = 62.0; + for (var i = 0; i < n; i++) { + final ts = i * cadenceSec * 1000.0; + if (r.nextInt(900) == 0) posture = 0.95 + r.nextDouble() * 0.1; + if (moveLeft == 0 && r.nextInt(500) == 0) moveLeft = 5 + r.nextInt(40); + var m = posture + (r.nextDouble() - 0.5) * 0.002; + if (moveLeft > 0) { + m += r.nextDouble() * 0.3; + moveLeft--; + } + if (r.nextInt(2000) == 0) hrBase = 52 + r.nextDouble() * 25; + final cyc = 4 * math.sin(2 * math.pi * i * cadenceSec / 5400.0); + var h = hrBase + cyc + (r.nextDouble() - 0.5) * 4; + if (moveLeft > 0) h += 12; + if (hrDropouts && (i ~/ 200) % 7 == 3) h = 0; + if (hrDropouts && r.nextInt(300) == 0) h = double.nan; + hr.add(h); + var gx = m * 0.6; + final gy = m * 0.64, gz = m * 0.48; + if (nanAccel && r.nextInt(400) == 0) gx = double.nan; + accel.add(AccelSample(ts, gx, gy, gz)); + if (withRr) { + // beats inherit their record second => tied timestamps, ascending + final nb = h.isNaN || h <= 0 ? 0 : (h * cadenceSec / 60).round(); + for (var b = 0; b < nb; b++) { + var v = 60000.0 / h + + (r.nextDouble() - 0.5) * 60 + + 40 * math.sin(2 * math.pi * i * cadenceSec / 25.0); + if (rrOutliers && r.nextInt(60) == 0) { + v = r.nextBool() ? 150 : 2600; + } else if (rrOutliers && r.nextInt(80) == 0) { + v += 350; + } + rr.add(v); + rrTs.add(ts); + } + } + } + return (hr: hr, accel: accel, rr: rr, rrTs: rrTs); +} + +// ───────────────────────────── tests ───────────────────────────────────────── + +void main() { + group('CVHR baseline parity (frozen cvhrApneaScreen)', () { + void check(String name, int seed, int n, + {int gapEvery = 0, + double nanFrac = 0, + double dupFrac = 0, + double artifact = 0.05, + double period = 55}) { + test(name, () { + final b = genBeats(seed, n, + gapEvery: gapEvery, + nanFrac: nanFrac, + dupFrac: dupFrac, + period: period); + expectSameCvhr( + cvhrApneaScreen(b.nn, b.tMs, artifactFraction: artifact), + refCvhrApneaScreen(b.nn, b.tMs, artifactFraction: artifact), + ); + }); + } + + check('clean 1 h', 1, 3800); + check('clean 2 h, other period', 2, 7600, period: 83); + check('gaps', 3, 4000, gapEvery: 700); + check('many gaps (short segments dropped)', 4, 2500, gapEvery: 90); + // NaN NN values are deliberately NOT exercised here: a NaN reaching the run + // detection (`exc[i] <= prom` is false for NaN, and the inner + // `while (exc[i] > prom)` is false too) never advances `i` - an infinite loop + // in today's code, so no frozen output exists to compare against. + check('duplicate timestamps', 6, 3600, dupFrac: 0.05); + check('gaps + duplicates', 7, 4200, gapEvery: 600, dupFrac: 0.03); + check('exactly 60 beats', 8, 60); + check('59 beats (absent)', 9, 59); + check('short, 75 beats', 10, 75); + check('artifact gate (absent)', 11, 3600, artifact: 0.5); + + test('mismatched lengths and degenerate times', () { + final b = genBeats(12, 100); + expectSameCvhr( + cvhrApneaScreen(b.nn, b.tMs.sublist(1), artifactFraction: 0.0), + refCvhrApneaScreen(b.nn, b.tMs.sublist(1), artifactFraction: 0.0), + ); + final flat = List.filled(100, 5000.0); + expectSameCvhr( + cvhrApneaScreen(b.nn, flat, artifactFraction: 0.0), + refCvhrApneaScreen(b.nn, flat, artifactFraction: 0.0), + ); + }); + + test('non-default tunables', () { + final b = genBeats(13, 3600); + expectSameCvhr( + cvhrApneaScreen(b.nn, b.tMs, + artifactFraction: 0.1, + envWindowSec: 91, + minWidthSec: 8, + maxWidthSec: 100, + depthWidthRatioMin: 0.5, + maxGapSec: 20), + refCvhrApneaScreen(b.nn, b.tMs, + artifactFraction: 0.1, + envWindowSec: 91, + minWidthSec: 8, + maxWidthSec: 100, + depthWidthRatioMin: 0.5, + maxGapSec: 20), + ); + }); + + test('full 8 h night with gaps', () { + final b = genBeats(14, 30000, gapEvery: 6000, amp: 45); + final a = cvhrApneaScreen(b.nn, b.tMs, artifactFraction: 0.04); + expectSameCvhr( + a, refCvhrApneaScreen(b.nn, b.tMs, artifactFraction: 0.04)); + // Not vacuous: the synthetic dips must actually be scored. + expect(a.value!.cycleCount, greaterThan(0)); + }); + }); + + group('hrvFreq / Welch parity (frozen _welchBandPower)', () { + void check(String name, int seed, int n, + {int gapEvery = 0, + double nanFrac = 0, + double dupFrac = 0, + double artifact = 0.05, + double meanNn = 900}) { + test(name, () { + final b = genBeats(seed, n, + gapEvery: gapEvery, + nanFrac: nanFrac, + dupFrac: dupFrac, + meanNn: meanNn); + expectSameHrv( + hrvFreq(b.nn, b.tMs, artifactFraction: artifact), + refHrvFreq(b.nn, b.tMs, artifactFraction: artifact), + ); + }); + } + + check('clean 1.2 h', 21, 4500); + check('clean 1.5 h, faster heart', 22, 6000, meanNn: 700); + check('beat gaps reject segments', 23, 5000, gapEvery: 400); + check('NaN NN values', 24, 4000, nanFrac: 0.002); + check('duplicate timestamps', 25, 4000, dupFrac: 0.05); + check('gaps + NaN + duplicates', 26, 5000, + gapEvery: 900, nanFrac: 0.001, dupFrac: 0.02); + check('HF gated by artifact fraction', 27, 4000, artifact: 0.3); + check('short record, VLF unresolved', 28, 400); + check('16 beats', 29, 16); + check('15 beats (absent)', 30, 15); + check('very short span (nothing resolves)', 31, 50, meanNn: 400); + // 10 h of sparse beats so the ULF band (33 333 s segment) resolves too. + check('ULF resolves (10 h sparse)', 32, 6000, meanNn: 6000); + check('ULF with gaps', 33, 6000, meanNn: 6000, gapEvery: 1500); + + test('non-default oversample', () { + final b = genBeats(34, 3500); + expectSameHrv( + hrvFreq(b.nn, b.tMs, artifactFraction: 0.0, oversample: 2.5), + refHrvFreq(b.nn, b.tMs, artifactFraction: 0.0, oversample: 2.5), + ); + }); + + test('degenerate times and mismatched lengths', () { + final b = genBeats(35, 40); + expectSameHrv( + hrvFreq(b.nn, List.filled(40, 7000.0), artifactFraction: 0.0), + refHrvFreq(b.nn, List.filled(40, 7000.0), artifactFraction: 0.0), + ); + expectSameHrv( + hrvFreq(b.nn, b.tMs.sublist(2), artifactFraction: 0.0), + refHrvFreq(b.nn, b.tMs.sublist(2), artifactFraction: 0.0), + ); + }); + }); + + group('cardio stager parity (frozen 1 g reference loop + entry point)', () { + void check(String name, int seed, int seconds, + {int cadenceSec = 1, + bool nanAccel = false, + bool hrDropouts = false, + bool rrOutliers = false, + bool withRr = true}) { + test(name, () { + final g = genNight(seed, seconds, + cadenceSec: cadenceSec, + nanAccel: nanAccel, + hrDropouts: hrDropouts, + rrOutliers: rrOutliers, + withRr: withRr); + final a = cardioStager(g.hr, g.accel, rrMs: g.rr, rrTsMs: g.rrTs); + final b = refCardioStager(g.hr, g.accel, rrMs: g.rr, rrTsMs: g.rrTs); + expectSameStager(a, b); + // Not vacuous: normal inputs must actually stage something. + if (name.startsWith('staged')) { + expect(a.base.stages, isNotEmpty); + } + }); + } + + check('staged 40 min with RR', 41, 2400); + check('staged 50 min, RR outliers + HR dropouts', 42, 3000, + rrOutliers: true, hrDropouts: true); + check('staged 40 min, NaN accel magnitudes', 43, 2400, nanAccel: true); + check('staged 1 h without RR', 44, 3600, withRr: false); + check('staged 10 s cadence', 45, 7200, cadenceSec: 10); + check('5 s cadence (abstains: 30/5 grid)', 46, 7200, cadenceSec: 5); + check('60 s cadence (abstains)', 47, 14400, cadenceSec: 60); + check('short: 2 epochs (abstains)', 48, 70); + check('short: 3 epochs', 49, 95); + check('short: 100 s partial last epoch', 50, 100); + + test('mismatched hr/accel lengths and empty input', () { + final g = genNight(51, 1800); + expectSameStager( + cardioStager(g.hr.sublist(0, 1500), g.accel, + rrMs: g.rr, rrTsMs: g.rrTs), + refCardioStager(g.hr.sublist(0, 1500), g.accel, + rrMs: g.rr, rrTsMs: g.rrTs), + ); + expectSameStager(cardioStager(const [], const []), + refCardioStager(const [], const [])); + }); + + test('custom epochSec', () { + final g = genNight(52, 2400); + expectSameStager( + cardioStager(g.hr, g.accel, rrMs: g.rr, rrTsMs: g.rrTs, epochSec: 60), + refCardioStager(g.hr, g.accel, + rrMs: g.rr, rrTsMs: g.rrTs, epochSec: 60), + ); + }); + + test('long night (3 h)', () { + final g = genNight(53, 3 * 3600, hrDropouts: true, rrOutliers: true); + expectSameStager( + cardioStager(g.hr, g.accel, rrMs: g.rr, rrTsMs: g.rrTs), + refCardioStager(g.hr, g.accel, rrMs: g.rr, rrTsMs: g.rrTs), + ); + }); + }); +} diff --git a/test/onehz/support/energy_day.dart b/test/onehz/support/energy_day.dart new file mode 100644 index 00000000..5b6dfd22 --- /dev/null +++ b/test/onehz/support/energy_day.dart @@ -0,0 +1,56 @@ +// Deterministic synthetic day for the per-minute energy tests: no randomness, +// no clock. Mixes every case dailyEnergy distinguishes — resting HR, HR over +// the flex gate, dropped (0) and non-finite HR, cadence on sub-flex minutes, +// cadence on HR-billed minutes, cadence on no-HR minutes. + +import 'package:openstrap_analytics/src/onehz/workout/calories.dart'; + +class EnergyDay { + final List hr; + final List cadence; + const EnergyDay(this.hr, this.cadence); +} + +/// [n] minutes. Same LCG as the rest of the suite's synthetic data. +EnergyDay energyDay({int n = 1440, int seed = 4242}) { + var s = seed; + double rnd() { + s = (s * 1103515245 + 12345) & 0x7fffffff; + return s / 0x7fffffff; + } + + final hr = []; + final cad = []; + for (var i = 0; i < n; i++) { + final r = rnd(); + double h; + if (r < 0.05) { + h = 0.0; // off-skin sentinel + } else if (r < 0.07) { + h = double.nan; + } else if (r < 0.60) { + h = 52.0 + rnd() * 40.0; // resting / light, mostly under the gate + } else { + h = 100.0 + rnd() * 75.0; // straddles the gate, tops out near hrmax + } + hr.add(h); + final c = rnd(); + if (c < 0.15) { + cad.add(90.0 + rnd() * 60.0); // below, inside and past the 100-130 range + } else if (c < 0.17) { + cad.add(double.nan); + } else { + cad.add(null); + } + } + return EnergyDay(hr, cad); +} + +const energyProfiles = { + 'male': WorkoutUserProfile( + weightKg: 82.5, heightCm: 181.0, age: 34, sex: 'male'), + 'female': WorkoutUserProfile( + weightKg: 61.0, heightCm: 167.0, age: 41, sex: 'female'), + 'nonbinary': WorkoutUserProfile( + weightKg: 70.0, heightCm: 172.5, age: 29, sex: 'nonbinary'), +}; diff --git a/test/onehz/support/incremental_compare.dart b/test/onehz/support/incremental_compare.dart index a969d77c..7d59b5f7 100644 --- a/test/onehz/support/incremental_compare.dart +++ b/test/onehz/support/incremental_compare.dart @@ -74,5 +74,73 @@ void enmoClose(EnmoResult actual, EnmoResult expected) { } } +void minuteClose(MinuteMetrics actual, List keys, List hr, + {List? cadence, + double? rhr = 54, + double? maxHr = 186, + Sex sex = Sex.male, + WorkoutUserProfile? profile, + int dayMinutes = 1440, + double? quietHrr = .12, + bool quietSettled = false}) { + final trimp = banisterTrimp(hr, restingHr: rhr, maxHr: maxHr, sex: sex); + final strain = strainScoreMetric(trimp.value, + wakeMinutes: hr.length.toDouble(), + quietHrr: quietHrr, + quietSettled: quietSettled, + female: sex == Sex.female); + metricEnvelope(actual.trimp, trimp); + numberClose(actual.trimp.value, trimp.value); + metricEnvelope(actual.strain, strain); + numberClose(actual.strain.value, strain.value); + if (profile == null || rhr == null || maxHr == null) { + expect(actual.energy, isNull); + expect(actual.minutes, isNull); + return; + } + final e = Calories.dailyEnergy(hr, + profile: profile, + hrmax: maxHr, + restingHr: rhr, + dayMinutes: dayMinutes, + cadenceSpmPerMin: cadence); + final series = Calories.minuteEnergy(hr, + profile: profile, + hrmax: maxHr, + restingHr: rhr, + epochMinutes: keys, + cadenceSpmPerMin: cadence); + if (e == null) { + expect(actual.energy, isNull); + } else { + expect(actual.energy, isNotNull); + numberClose(actual.energy!.total, e.total); + numberClose(actual.energy!.active, e.active); + numberClose(actual.energy!.basal, e.basal); + numberClose(actual.energy!.walking, e.walking); + } + if (series == null) { + expect(actual.minutes, isNull); + return; + } + final a = actual.minutes!; + expect(a.coveredMinutes, series.coveredMinutes); + expect(a.abstainedMinutes, series.abstainedMinutes); + numberClose(a.basalKcalPerMin, series.basalKcalPerMin); + numberClose(a.active, series.active); + numberClose(a.walking, series.walking); + expect(a.minutes.length, series.minutes.length); + for (var i = 0; i < series.minutes.length; i++) { + final am = a.minutes[i], em = series.minutes[i]; + expect(am.minute, em.minute); + expect(am.source, em.source); + expect(am.abstained, em.abstained); + numberClose(am.basal, em.basal); + numberClose(am.active, em.active); + numberClose(am.total, em.total); + numberClose(am.walking, em.walking); + } +} + Map checkpoint(Map json) => jsonDecode(jsonEncode(json)) as Map; diff --git a/test/properties/incremental_laws_test.dart b/test/properties/incremental_laws_test.dart new file mode 100644 index 00000000..b9e2f9b5 --- /dev/null +++ b/test/properties/incremental_laws_test.dart @@ -0,0 +1,3172 @@ +// LAWS of the incremental calculations (design 05, pilot cluster C2A, analytics +// side): the state objects behind `lib/src/onehz/incremental*.dart` -- +// `IncrementalHrvTime`, `IncrementalEnmoSeries`, `IncrementalLombScargle`, +// `IncrementalMinuteMetrics` (all four are fed the WHOLE series so far on every +// `sync` and keep sums for the prefix they have already seen), and the two +// mergeable summaries under them, `RunningMoments` and `IntHistogram`, plus the +// `CalculationCache` that guards the results. +// +// L1 chunk invariance, stated for a sync: syncing growing prefixes of a +// series in ANY chunking (empty prefixes and a save/restore through real +// JSON text between syncs included) gives the checkpoint of one sync of +// the whole series -- and, because a state that cannot continue starts +// over, the same holds after any DETOUR (an edited series, a dropped +// prefix, a forced rebuild, another artifact fraction / reference) as +// long as the last sync is of the same series. The checkpoint is +// compared as text without the work counters (`processedPoints`, +// `processedMinutes`: performance contracts, kept as examples). The +// checkpoint of the minute metrics lists its bills in the order the keys +// were first seen, so after a detour that REORDERS keys it is compared +// by key; chunking alone gives the same text. +// L1b streamed vs batch PARITY: after every checked sync the output is what +// the production batch function gives for the same series (`hrvTime`, +// `enmoSeries`, `lombScargle`, `banisterTrimp` / `strainScoreMetric` / +// `Calories`), counts and abstentions exact, numbers within +// max(1e-9, 1e-8 relative) (running sums against a two-pass batch). +// These batch functions are NOT independent oracles: they share helpers +// and constants with the incremental state (`hrvTime` is what the state +// delegates to at the refusal boundary, `kNnDiffAcf1Floor`, the Banister +// weight, the energy pricer, ...), so a fault in a shared helper passes +// this law. What it pins is that the streamed path agrees with the batch +// path; the shared helpers have their own tests. Only `RunningMoments` +// and `IntHistogram` are checked against references written here. +// L2 checkpoint round trip: `toJson` -> real JSON text -> `fromJson` -> +// `toJson` is the same text, and the restored state goes on exactly as +// the live one does (same outputs, same text). +// L3 a checkpoint of another type or version, a missing or mistyped part, +// parts that disagree (counts against the data they count), hostile +// sizes: refused whole with a FormatException, never another error type. +// A change the reader cannot tell from a legal checkpoint (one sum +// altered) may be accepted, but then the state holds exactly what was +// written; an unknown key is ignored, nothing else moves. +// L4 identity: syncing the series it already holds changes nothing (outputs +// and checkpoint), an empty first sync leaves a state that takes the +// series as a fresh one would, and the other way round the parameters +// belong to the CALL: handing other parameters rebuilds the state as a +// fresh sync of them would (there is nothing to refuse), a refused call +// (misaligned arguments, duplicate keys) leaves the state unchanged. +// L5 conservation: every beat / sample / minute is counted exactly once in +// the sums a state keeps (pairs, bins, trigonometric sums, bills), +// checked against counts made here from the INPUT, never from the code. +// M merge (`RunningMoments`, `IntHistogram`): merging the summaries of two +// parts is the summary of the concatenation (exact for the histogram, +// within float tolerance for the moments), associative, commutative, +// with the empty summary as identity -- the nearest thing to a monoid +// law the analytics side has. +// +// The laws describe the code at analytics 0fc57682. A failing law is first +// checked against the module's contract; only a violation of the intended +// contract is a bug, and a wrong expectation is fixed here, never in lib/. +// +// Replay a failure with the command in its report, e.g. +// PROPERTY_SEED= PROPERTY_CASE= TZ=UTC dart test \ +// test/properties/incremental_laws_test.dart --plain-name '' + +import 'dart:convert'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +import '../onehz/support/energy_day.dart'; +import '../onehz/support/incremental_compare.dart'; +import '../support/fold_law_support.dart'; +import '../support/law_registry.dart'; +import '../support/property.dart'; + +final _laws = LawSet(); + +typedef Json = Map; + +// ── the shape every sync module shares ────────────────────────────────────── + +/// One module under the sync laws. [In] is a whole series PLUS the parameters +/// of the call; everything else is how to drive it and how to look at it. +abstract class Ops { + String get name; + + /// The series of a recipe (a handful of ints; cached by the caller). + In expand(List recipe); + int length(In input); + + /// The first [k] items with the same parameters. + In prefix(In input, int k); + + /// A different series with the same parameters (a replaced item, a dropped + /// prefix, a shrunk tail, ...), chosen by [kind]. + In edit(In input, int kind, int salt); + + /// The same series under other parameters ([alt] picks), or null when the + /// module has none (the Lomb grid belongs to the constructor). + In? reconfigure(In input, int alt); + + /// A new state for (the parameters of) [input]. + dynamic fresh(In input); + dynamic fromJson(Json j); + Json toJson(dynamic state); + Object? sync(dynamic state, In input, {bool force = false}); + + /// The output as exact text. + String show(Object? out); + + /// Fails unless [out] is what the batch gives for [input]. + void close(Object? out, In input); + + /// Fails unless two outputs of the same series are the same. Exact by + /// default; a module that rebuilds a running sum on restore (the HRV bins) + /// says "within the batch tolerance" instead. + void sameOutput(Object? a, Object? b, String why) => + expect(show(a), show(b), reason: why); + + /// Keys of the checkpoint that count work, not state. + List get counters; + + /// The checkpoint as text, without the counters. + String norm(Json j) { + final c = Map.of(j); + for (final k in counters) { + c.remove(k); + } + return jsonEncode(c); + } + + /// Fails unless [got], the checkpoint after a detour (series replaced, items + /// dropped, a forced rebuild, other parameters) and the same series again, is + /// the checkpoint [want] of one sync. Exact text by default; a module whose + /// checkpoint is equal in MEANING but not in text after a detour (it is + /// keyed, and the order its entries were added in is not state) projects + /// first. + void sameState(Json got, Json want, String why) => + expect(norm(got), norm(want), reason: why); + + /// Hand-written invariants of a reachable state that, broken, must be + /// refused. + List get refuseMutations; + + /// Changes the reader cannot tell from a legal checkpoint. + List get faithfulMutations; + + /// Calls that must be refused (ArgumentError) and leave the state unchanged; + /// `applies` says whether the input has something to get wrong. + List<({String name, bool Function(In) applies, void Function(dynamic, In) call})> + get refusals => const []; + + /// Counts made from the INPUT that the checkpoint must agree with. + void conserved(Json checkpoint, In input, String why); + + /// Positions where the series changes character (window and bin edges). + List get special; + + Gen> recipeGen(int maxN); + List> get forced; + void observe(List recipe, void Function(String) bump); + Map get shares; +} + +String _text(Json j) => jsonEncode(j); + +dynamic _restore(Ops ops, dynamic st) => + ops.fromJson((jsonDecode(jsonEncode(ops.toJson(st))) as Map).cast()); + +/// (splits, cut seed, restart mask, detour). +typedef Plan = (int, int, int, int); + +final Gen _planGen = G.quad(G.intIn(0, 6), G.intIn(0, 1 << 20), + G.intIn(0, 127), G.elements(const [0, 0, 1, 2, 3, 4, 5])); + +typedef Case = (List, Plan); + +Gen _caseGen(Ops ops, int maxN) => G.pair(ops.recipeGen(maxN), _planGen); + +String _tagOf(Ops ops, Case c) => '${ops.name} ${c.$1} plan=${c.$2}'; + +final Map _expandCache = {}; + +In _expand(Ops ops, List recipe) { + final key = '${ops.name}|$recipe'; + if (_expandCache.length > 500) _expandCache.clear(); + return _expandCache.putIfAbsent(key, () => ops.expand(recipe)) as In; +} + +void _observeCase(Ops ops, Case c, void Function(String) bump) { + ops.observe(c.$1, bump); + final n = ops.length(_expand(ops, c.$1)); + final (splits, cutSeed, restart, detour) = c.$2; + final b = foldBounds(n, splits, cutSeed, special: ops.special); + if (b.length >= 4) bump('chunks: three or more'); + if (hasEmptyChunk(b)) bump('an empty prefix'); + if (restart != 0 && b.length > 2) bump('a restore between syncs'); + if (detour != 0) bump('detour: $detour'); +} + +Map _lawShares(Ops ops, {Set drop = const {}}) => { + ...ops.shares, + 'chunks: three or more': .4, + 'an empty prefix': .04, + 'a restore between syncs': .3, + for (var k = 1; k <= 5; k++) 'detour: $k': .02, + }..removeWhere((k, _) => drop.contains(k)); + +// ── the laws, written once for every module ───────────────────────────────── + +void _l1(Ops ops, Case c) { + final x = _expand(ops, c.$1); + final n = ops.length(x); + final (splits, cutSeed, restart, detour) = c.$2; + final tag = _tagOf(ops, c); + final single = ops.fresh(x); + final out1 = ops.sync(single, x); + final want = ops.toJson(single); + // Growing prefixes (the last one is the whole series), a save/restore after + // some of them, and a detour before the whole series. + var st = ops.fresh(x); + final bounds = foldBounds(n, splits, cutSeed, special: ops.special); + for (var k = 1; k + 1 < bounds.length; k++) { + ops.sync(st, ops.prefix(x, bounds[k])); + if ((restart >> (k % 7)) & 1 == 1) st = _restore(ops, st); + } + if (detour != 0) { + switch (detour) { + case 1: + ops.sync(st, ops.edit(x, 0, cutSeed)); // one item replaced + case 2: + ops.sync(st, ops.edit(x, 1, cutSeed)); // the first items dropped + case 3: + ops.sync(st, x, force: true); // a forced rebuild + case 4: + final other = ops.reconfigure(x, cutSeed); + if (other != null) ops.sync(st, other); // other parameters + default: + ops.sync(st, ops.edit(x, 2, cutSeed)); // the tail dropped + } + st = _restore(ops, st); + } + final out2 = ops.sync(st, x); + ops.sameState(ops.toJson(st), want, '$tag bounds=$bounds: the checkpoint is the same'); + ops.sameOutput(out2, out1, '$tag: the output is the same'); +} + +void _l1b(Ops ops, Case c) { + final x = _expand(ops, c.$1); + final n = ops.length(x); + final (splits, cutSeed, restart, detour) = c.$2; + var st = ops.fresh(x); + final bounds = foldBounds(n, splits, cutSeed, special: ops.special); + final picks = pickSeams(bounds.length - 1); + for (var k = 1; k < bounds.length; k++) { + final part = ops.prefix(x, bounds[k]); + final out = ops.sync(st, part); + if (picks.contains(k - 1) || k == bounds.length - 1) { + ops.close(out, part); + } + if ((restart >> (k % 7)) & 1 == 1) st = _restore(ops, st); + if (k == bounds.length - 2 && detour != 0) { + // Whatever the history, the output is the batch's for the series now. + final other = detour == 4 + ? ops.reconfigure(x, cutSeed) ?? ops.edit(x, 0, cutSeed) + : ops.edit(x, detour % 3, cutSeed); + ops.close(ops.sync(st, other), other); + } + } +} + +void _l2(Ops ops, Case c) { + final x = _expand(ops, c.$1); + final n = ops.length(x); + final (splits, cutSeed, _, _) = c.$2; + final tag = _tagOf(ops, c); + final bounds = foldBounds(n, splits, cutSeed, special: ops.special); + final at = bounds.length > 2 ? bounds[1] : n ~/ 2; + final live = ops.fresh(x); + ops.sync(live, ops.prefix(x, at)); + final text = _text(ops.toJson(live)); + final map = (jsonDecode(text) as Map).cast(); + final back = ops.fromJson(map); + expect(_text(ops.toJson(back)), text, reason: '$tag: write(read(b)) == b'); + // Not aliased to the map it came from. + for (final k in map.keys.toList()) { + final v = map[k]; + if (v is List) v.clear(); + } + expect(_text(ops.toJson(back)), text, reason: '$tag: not aliased to the map'); + final restored = ops.fromJson((jsonDecode(text) as Map).cast()); + // And it goes on identically: the rest of the series, then a re-sync. + final a = ops.sync(live, x), b = ops.sync(restored, x); + ops.sameOutput(b, a, '$tag: same output after the rest'); + expect(_text(ops.toJson(restored)), _text(ops.toJson(live)), + reason: '$tag: same checkpoint after the rest (counters included)'); + ops.sameOutput(ops.sync(restored, x), a, '$tag: twice'); +} + +/// The mutated checkpoint, what it may do, and (when that is not simply "the +/// text of the mutated map") the texts a faithful reading may give. +(Json, Must, List?) _mutate(Ops ops, Json src, Mut m) { + final j = (deepCopy(src) as Map).cast(); + final (kind, a, b) = m; + final keys = j.keys.toList(); + Must refuseUnlessNoop(Json bad) => + jsonEncode(bad) == jsonEncode(src) ? Must.faithful : Must.refuse; + switch (kind % 7) { + case 0: + const versions = [0, 2, 3, -1, 99, '1', null, true, 1 << 40]; + j['version'] = versions[a % versions.length]; + return (j, Must.refuse, null); + case 1: + const types = ['RunningMoments', 'Other', null, '', 1, 'incrementalhrvtime']; + j['type'] = types[a % types.length]; + return (j, Must.refuse, null); + case 2: + // A key gone: refused, or (the readers take an absent key for null where + // a part is nullable, and rebuild a total that older checkpoints did not + // carry) read back as null, or as it was. + final k = keys[a % keys.length]; + j.remove(k); + final asNull = (deepCopy(src) as Map).cast()..[k] = null; + return ( + j, + k == 'version' || k == 'type' ? Must.refuse : Must.faithful, + [_text(asNull), _text(src)] + ); + case 3: + final k = keys[a % keys.length]; + const junk = ['x', {}, [[]]]; + j[k] = junk[b % junk.length]; + return (j, Must.refuse, null); + case 4: + if (ops.refuseMutations.isEmpty) return (j, Must.faithful, null); + ops.refuseMutations[a % ops.refuseMutations.length](j); + return (j, refuseUnlessNoop(j), null); + case 5: + if (ops.faithfulMutations.isEmpty) return (j, Must.faithful, null); + ops.faithfulMutations[a % ops.faithfulMutations.length](j); + return (j, Must.faithful, null); + default: + j['futureField${a % 3}'] = const [1, 'x', [1]][b % 3]; + return (j, Must.ignored, null); + } +} + +String _outcome(Ops ops, Json j) { + try { + final s = ops.fromJson((deepCopy(j) as Map).cast()); + return 'accepted:${_text(ops.toJson(s))}'; + } on FormatException { + return 'refused'; + } catch (e) { + return 'WRONG ERROR TYPE: ${e.runtimeType}: $e'; + } +} + +void _l3(Ops ops, (Case, Mut) arg) { + final (c, m) = arg; + final tag = '${_tagOf(ops, c)} mutation=${m.$1 % 7}(${m.$2},${m.$3})'; + final x = _expand(ops, c.$1); + final st = ops.fresh(x); + ops.sync(st, x); + final src = (jsonDecode(_text(ops.toJson(st))) as Map).cast(); + final (bad, must, expected) = _mutate(ops, src, m); + final outcome = _outcome(ops, bad); + if (expected == null) { + expectMutationOutcome(outcome, must, _text(bad), _text(src), tag); + } else { + expect(outcome, isNot(startsWith('WRONG')), reason: '$tag: only FormatException'); + if (must == Must.refuse) { + expect(outcome, 'refused', reason: '$tag: refused whole'); + } else if (outcome != 'refused') { + expect(expected.map((t) => 'accepted:$t'), contains(outcome), + reason: '$tag: accepted means a faithful reading of what was written'); + } + } +} + +void _l4(Ops ops, Case c) { + final x = _expand(ops, c.$1); + final tag = _tagOf(ops, c); + final fresh = ops.fresh(x); + final out = ops.sync(fresh, x); + final text = ops.norm(ops.toJson(fresh)); + // Syncing what it already holds: nothing moves. + expect(ops.show(ops.sync(fresh, x)), ops.show(out), reason: '$tag: same output'); + expect(ops.norm(ops.toJson(fresh)), text, reason: '$tag: sync of the same is the identity'); + // An empty first sync, then the series: as if it were a fresh state. + final late = ops.fresh(x); + ops.sync(late, ops.prefix(x, 0)); + ops.sync(late, x); + expect(ops.norm(ops.toJson(late)), text, + reason: '$tag: an empty first sync leaves a state that takes the series afresh'); + // The converse: the parameters belong to the call. Other parameters rebuild + // the state as a fresh sync of them would (nothing is refused, even for a + // state restored from a checkpoint written under the first), and handing the + // first ones back rebuilds the first. + final other = ops.reconfigure(x, c.$2.$2); + if (other != null) { + var s = ops.fresh(x); + ops.sync(s, x); + s = _restore(ops, s); + ops.sync(s, other); + final f = ops.fresh(other); + ops.sync(f, other); + expect(ops.norm(ops.toJson(s)), ops.norm(ops.toJson(f)), + reason: '$tag: other parameters rebuild, as a fresh sync of them'); + ops.sync(s, x); + expect(ops.norm(ops.toJson(s)), text, reason: '$tag: and back'); + } + // A refused call leaves the checkpoint as it was. + final before = _text(ops.toJson(fresh)); + for (final r in ops.refusals) { + if (!r.applies(x)) continue; + expect(() => r.call(fresh, x), throwsArgumentError, reason: '$tag: ${r.name}'); + expect(_text(ops.toJson(fresh)), before, + reason: '$tag: ${r.name} leaves the state unchanged'); + } +} + +void _l5(Ops ops, Case c) { + final x = _expand(ops, c.$1); + final n = ops.length(x); + final (splits, cutSeed, restart, _) = c.$2; + final tag = _tagOf(ops, c); + var st = ops.fresh(x); + final bounds = foldBounds(n, splits, cutSeed, special: ops.special); + for (var k = 1; k < bounds.length; k++) { + final part = ops.prefix(x, bounds[k]); + ops.sync(st, part); + final json = (jsonDecode(_text(ops.toJson(st))) as Map).cast(); + ops.conserved(json, part, '$tag at ${bounds[k]}'); + // The reader takes back what the writer wrote. + st = (restart >> (k % 7)) & 1 == 1 ? _restore(ops, st) : st; + } +} + +/// Registers the five laws of [ops]. +void _registerSyncLaws(Ops ops, + {int maxN = 400, + int casesL1 = 120, + int casesL1b = 60, + Set dropL1b = const {}}) { + group(ops.name, () { + final reachFor = (Set drop) => Reach( + _lawShares(ops, drop: drop), (c, bump) => _observeCase(ops, c, bump)); + _laws.law( + 'L1 ${ops.name}: any chunking of the syncs, empty prefixes, restores and ' + 'detours included, gives the checkpoint text and output of one sync', + _caseGen(ops, maxN), + (c) => _l1(ops, c), + examples: [for (final r in ops.forced) (r, (3, 5 + r.length, 85, r.length % 6))], + cases: casesL1, + reach: reachFor(const {}), + ); + _laws.law( + 'L1b ${ops.name}: after every checked sync the output is the batch ' + "function's, history notwithstanding", + _caseGen(ops, maxN), + (c) => _l1b(ops, c), + examples: [for (final r in ops.forced) (r, (4, 9 + r.length, 127, (r.length + 1) % 6))], + cases: casesL1b, + reach: reachFor(dropL1b), + ); + _laws.law( + 'L2 ${ops.name}: write(read(b)) == b through JSON text, and the restored ' + 'state goes on identically', + _caseGen(ops, maxN), + (c) => _l2(ops, c), + examples: [for (final r in ops.forced) (r, (2, 3 + r.length, 0, 0))], + cases: 100, + reach: reachFor(const {'detour: 1', 'detour: 2', 'detour: 3', 'detour: 4', 'detour: 5'}), + ); + _laws.law<(Case, Mut)>( + 'L3 ${ops.name}: a mutated checkpoint is refused whole or holds exactly ' + 'what was written', + G.pair(_caseGen(ops, 200), mutGen(7)), + (arg) => _l3(ops, arg), + examples: [ + for (var i = 0; i < ops.forced.length; i++) + for (var k = 0; k < 7; k++) + ((ops.forced[i], (3, 5, 0, 0)), (k, 3 * i + k, 5 * i + 2 * k)), + for (var a = 0; a < ops.refuseMutations.length; a++) + for (var i = 0; i < ops.forced.length; i++) + ((ops.forced[i], (3, 5, 0, 0)), (4, a, i)), + for (var a = 0; a < ops.faithfulMutations.length; a++) + for (var i = 0; i < ops.forced.length; i++) + ((ops.forced[i], (3, 5, 0, 0)), (5, a, i)), + for (var a = 0; a < 9; a++) ((ops.forced.first, (3, 5, 0, 0)), (0, a, 0)), + ], + cases: 100, + reach: Reach<(Case, Mut)>({ + for (var k = 0; k < 7; k++) 'kind: $k': .03, + }, (arg, bump) => bump('kind: ${arg.$2.$1 % 7}')), + ); + _laws.law( + 'L4 ${ops.name}: syncing what it holds is the identity, an empty first ' + 'sync records nothing that matters, the parameters belong to the call, a ' + 'refused call changes nothing', + _caseGen(ops, maxN), + (c) => _l4(ops, c), + examples: [for (final r in ops.forced) (r, (3, 7 + r.length, 0, 0))], + cases: 100, + reach: reachFor(const {'detour: 1', 'detour: 2', 'detour: 3', 'detour: 4', 'detour: 5'}), + ); + _laws.law( + 'L5 ${ops.name}: every item is counted exactly once in the sums the state ' + 'keeps (counts made from the input)', + _caseGen(ops, maxN), + (c) => _l5(ops, c), + examples: [for (final r in ops.forced) (r, (4, 11 + r.length, 127, 0))], + cases: 100, + reach: reachFor(const {'detour: 1', 'detour: 2', 'detour: 3', 'detour: 4', 'detour: 5'}), + ); + }); +} + +// ═════════════════════════════════════════════════════════════════════════════ +// HRV +// ═════════════════════════════════════════════════════════════════════════════ + +const _hrvFlavours = [ + 'smooth', // two slow sines: a high lag-1 ACF of the differences is not expected + 'white jitter', // differenced white noise: the jitter gate refuses RMSSD / pNN50 + 'af', // irregularly irregular + 'constant', // every difference zero: variance zero + 'gaps', // a 21 s dropout every 113 beats + 'big steps', // every 7th beat 90 ms off: pNN50 moves + 'two valued', // alternating 800 / 1000: ACF of the differences is -1 + 'seconds grid', // 1 s beats on a whole-second grid: the 5-minute bin edges are hit exactly + 'gap edge', // the time step exceeds the beat by exactly, under and over the half millisecond + 'fifty', // successive differences of exactly, under and over 50 ms + // ~44 bpm, breathing at 18 br/min (0.3 Hz), a drifting heart rate: the lag-1 + // ACF of the differences is far under the jitter floor, yet the spectrum + // shows a respiratory line on a low floor, so the batch publishes RMSSD + // (upstream's noise-share exemption). + 'rsa slow heart', +]; +const _hrvOrigins = [0.0, 299999.5, 1700000010123.125, 1700000000000.0]; +const _hrvAf = [0.0, .17, .8, 1.0]; + +class HrvIn { + HrvIn(this.nn, this.t, this.af); + final List nn; + final List? t; + final double af; +} + +class HrvOps extends Ops { + @override + String get name => 'IncrementalHrvTime'; + + @override + HrvIn expand(List r) { + final (flavour, n, seed, timed, originIdx, afIdx) = (r[0], r[1], r[2], r[3], r[4], r[5]); + final g = Rng(seed * 7919 + flavour + 5); + final nn = [], t = []; + var clock = _hrvOrigins[originIdx]; + var fiftyBase = 800.0; + for (var i = 0; i < n; i++) { + final v = switch (flavour) { + 1 => 800 + 160 * (g.nextDouble() - .5), + 2 => 420 + g.nextInt(700).toDouble(), + 3 => 800.0, + 6 => i.isEven ? 800.0 : 1000.0, + 7 => 900.0 + (i % 11) * 7, + 9 => fiftyBase += const [50.0, -50.0, 49.0, -49.0, 51.0, -51.0][i % 6], + 10 => 1350 + + 80 * math.sin(2 * math.pi * (clock - _hrvOrigins[originIdx]) / 3.6e6) + + 40 * math.sin(2 * math.pi * .3 * (clock - _hrvOrigins[originIdx]) / 1000) + + 3 * (g.nextDouble() - .5), + 5 => 800 + 40 * math.sin(i * .13) + (i % 7 == 6 ? 90 : 0), + _ => 800 + 75 * math.sin(i * .13 + seed * .003) + 21 * math.sin(i * .031 + seed * .007), + }; + if (flavour == 4 && i % 113 == 112) clock += 21000; + clock += flavour == 7 ? 1000 : v; + if (flavour == 8 && i % 9 == 8) clock += const [.5, .25, .75, .5, .4999, .5001][(i ~/ 9) % 6]; + nn.add(v); + t.add(clock); + } + return HrvIn(nn, timed == 0 ? t : null, _hrvAf[afIdx]); + } + + @override + int length(HrvIn i) => i.nn.length; + @override + HrvIn prefix(HrvIn i, int k) => + HrvIn(i.nn.sublist(0, k), i.t?.sublist(0, k), i.af); + @override + HrvIn edit(HrvIn i, int kind, int salt) { + final n = i.nn.length; + if (n < 2) return i; + switch (kind % 3) { + case 0: + final nn = [...i.nn]; + nn[salt % n] += 60; + return HrvIn(nn, i.t, i.af); + case 1: + final d = 1 + salt % (n - 1); + return HrvIn(i.nn.sublist(d), i.t?.sublist(d), i.af); + default: + final d = 1 + salt % (n - 1); + return HrvIn(i.nn.sublist(0, n - d), i.t?.sublist(0, n - d), i.af); + } + } + + @override + HrvIn? reconfigure(HrvIn i, int alt) => HrvIn(i.nn, i.t, _hrvAf[(alt + 1) % 4] == i.af ? .5 : _hrvAf[(alt + 1) % 4]); + + @override + dynamic fresh(HrvIn input) => IncrementalHrvTime(); + @override + dynamic fromJson(Json j) => IncrementalHrvTime.fromJson(j); + @override + Json toJson(dynamic s) => (s as IncrementalHrvTime).toJson(); + @override + Object? sync(dynamic s, HrvIn i, {bool force = false}) => (s as IncrementalHrvTime) + .sync(i.nn, nnTimesMs: i.t, artifactFraction: i.af, force: force); + + @override + String show(Object? out) { + final m = out as Metric; + final v = m.value; + return [ + m.present, + m.confidence, + m.tier, + m.note, + m.inputs_used, + if (v != null) [v.nBeats, v.rmssd, v.sdnn, v.sdann, v.sdnnIndex, v.pnn50, v.diffAcf1], + ].join('|'); + } + + @override + void sameOutput(Object? a, Object? b, String why) { + // Restoring rebuilds the running sums of the bin means and SDs from the + // stored bins (they are not in the checkpoint), so a restored state's + // SDANN / SDNN index agree with one that never stopped to ~1e-14 relative, + // not bit for bit. + final x = a as Metric, y = b as Metric; + try { + hrvClose(x, y); + } on TestFailure catch (e) { + fail('$why: ${e.message}'); + } + } + + @override + void close(Object? out, HrvIn i) => + hrvClose(out as Metric, hrvTime(i.nn, nnTimesMs: i.t, artifactFraction: i.af)); + + @override + List get counters => const ['processedPoints']; + + @override + List get special => const [0, 1, 2, 29, 30, 31, 299, 300, 301, 302, 599, 600, 601]; + + @override + List get refuseMutations => [ + (j) { + final n = (j['nn'] as List).length; + if (n > 0) j['processedPoints'] = n - 1; + }, + (j) => j['over50'] = (j['pairs'] as int) + 1, + (j) => j['lagPairs'] = (j['pairs'] as int) + 1, + (j) => j['pairs'] = math.max(0, (j['nn'] as List).length - 1) + 1, + (j) => (j['levels'] as Map)['count'] = ((j['levels'] as Map)['count'] as int) + 1, + (j) { + final bins = j['bins'] as List; + if (bins.isNotEmpty) (bins.first as Map)['count'] = ((bins.first as Map)['count'] as int) + 1; + }, + (j) => (j['nn'] as List).add(800.0), + (j) { + final t = j['times'] as List?; + if (t != null && t.isNotEmpty) t.removeLast(); + }, + (j) { + if (((j['levels'] as Map)['count'] as int) > 0) j['finite'] = false; + }, + (j) { + final nn = j['nn'] as List; + if (nn.isNotEmpty && j['finite'] == true) nn[0] = 'NaN'; + }, + (j) => (j['bins'] as List).add({ + 'version': 1, + 'type': 'RunningMoments', + 'count': 0, + 'origin': 0.0, + 'meanOffset': 0.0, + 'm2': 0.0 + }), + (j) => j['diffSquares'] = -1.0, + ]; + + @override + List get faithfulMutations => [ + (j) { + final nn = j['nn'] as List; + if (nn.isNotEmpty) nn[0] = (nn[0] as num) + 1.0; + }, + (j) => j['diffSum'] = (j['diffSum'] as num) + 1.0, + (j) => j['products'] = (j['products'] as num) + 1.0, + (j) => j['endpoints'] = (j['endpoints'] as num) + 1.0, + (j) => j['lastDiff'] = ((j['lastDiff'] as num?) ?? 0) + 1.0, + (j) => j['binIdx'] = (j['binIdx'] as int) + 1, + (j) => j['processedPoints'] = (j['processedPoints'] as int) + 5, + (j) { + final t = j['times'] as List?; + if (t != null && t.isNotEmpty) t[0] = (t[0] as num) + 1.0; + }, + ]; + + @override + void conserved(Json cp, HrvIn i, String why) { + final n = i.nn.length; + expect((cp['nn'] as List).length, n, reason: 'the series is kept $why'); + expect((cp['levels'] as Map)['count'], n, reason: 'every beat is a level, once $why'); + final timed = i.t != null; + // Pairs made here: neighbours, except across a dropout (a time step longer + // than the beat itself, plus half a millisecond). + var pairs = 0, over = 0, lag = 0; + double? last; + for (var k = 1; k < n; k++) { + if (timed && i.t![k] - i.t![k - 1] > i.nn[k] + .5) { + last = null; + continue; + } + final d = i.nn[k] - i.nn[k - 1]; + pairs++; + if (d.abs() > 50) over++; + if (last != null) lag++; + last = d; + } + expect(cp['pairs'], pairs, reason: 'successive differences $why'); + expect(cp['over50'], over, reason: 'differences over 50 ms $why'); + expect(cp['lagPairs'], lag, reason: 'lag-1 pairs $why'); + final bins = cp['bins'] as List; + if (!timed) { + expect(bins, isEmpty, reason: 'no times, no bins $why'); + } else { + // Every beat is in exactly one five-minute bin; bins are runs of beats + // with the same index since the first beat. + var total = 0; + for (final b in bins) { + total += (b as Map)['count'] as int; + } + expect(total, n, reason: 'every beat is in one bin $why'); + var runs = 0; + int? prev; + for (var k = 0; k < n; k++) { + final idx = ((i.t![k] - i.t!.first) / 300000).floor(); + if (prev == null || idx != prev) runs++; + prev = idx; + } + expect(bins.length, runs, reason: 'bins are runs of the same five minutes $why'); + } + } + + @override + Gen> recipeGen(int maxN) => IntsGen([ + G.intIn(0, _hrvFlavours.length - 1), + SizeGen(maxN, pool: const [0, 1, 2, 3, 29, 30, 31, 32, 299, 300, 301, 302, 599, 600, 601, 610]), + G.intIn(0, 1 << 12), + G.elements(const [0, 0, 0, 1]), + G.intIn(0, _hrvOrigins.length - 1), + G.intIn(0, _hrvAf.length - 1), + ]); + + @override + List> get forced => [ + [0, 0, 1, 0, 0, 0], // empty + [0, 1, 1, 0, 0, 0], // one beat + [0, 2, 1, 0, 0, 0], // two + [3, 31, 1, 0, 0, 0], // constant, 30 pairs + [3, 29, 1, 1, 0, 0], // constant, no times + [1, 300, 2, 0, 0, 1], // jitter: RMSSD refused + [6, 120, 1, 0, 0, 0], // ACF of the differences exactly -1 + [0, 31, 1, 0, 0, 0], // exactly 30 pairs: the first with an ACF + [0, 32, 2, 1, 0, 0], + [7, 450, 1, 0, 0, 0], // exactly two 5-minute bins + [8, 120, 1, 0, 0, 0], // dropout edges + [9, 120, 1, 0, 0, 0], // differences of exactly 50 ms + [9, 120, 2, 1, 3, 1], + [7, 610, 1, 0, 0, 0], // whole-second grid across 5-minute edges + [7, 610, 1, 0, 1, 2], // shifted origin + [7, 610, 1, 0, 2, 3], + [4, 400, 3, 0, 0, 0], // dropouts + [5, 200, 1, 0, 3, 0], // big steps + [2, 250, 5, 1, 0, 1], // AF, no times + [0, 500, 7, 0, 3, 2], + [10, 900, 1, 0, 0, 0], // RSA at a slow heart: exempt from the jitter gate + [10, 900, 2, 1, 0, 1], // the same without times + [10, 900, 3, 0, 2, 0], + ]; + + @override + void observe(List r, void Function(String) bump) { + bump('flavour: ${_hrvFlavours[r[0]]}'); + final n = r[1]; + if (n == 0) bump('series: empty'); + if (n >= 1 && n < 30) bump('series: under 30 pairs'); + if (n >= 300) bump('series: past one 5-minute bin'); + if (r[3] == 1) bump('no times'); + if (r[5] != 0) bump('artifact fraction set'); + } + + @override + Map get shares => { + for (var f = 0; f < _hrvFlavours.length; f++) 'flavour: ${_hrvFlavours[f]}': .03, + 'series: empty': .01, + 'series: under 30 pairs': .05, + 'series: past one 5-minute bin': .15, + 'no times': .08, + 'artifact fraction set': .3, + }; +} + +// ═════════════════════════════════════════════════════════════════════════════ +// ENMO +// ═════════════════════════════════════════════════════════════════════════════ + +const _enmoFlavours = [ + 'regular', // a sample every 1/hz s, a still vector with noise + 'gaps + invalid', // a hole of 69 samples, every 47th sample invalid + 'duplicate times', // two samples per timestamp + 'unsorted', // the stream arrives newest first + 'constant vector', +]; +const _hz = [.5, 1.0, 2.0]; +const _enmoOrigins = [0.0, 1700000010000.0, 59000.0, 1700000040000.5]; + +/// (gRef, gravityWindowS, minSamplesPerMinute, expectedMinutes). A null gRef is +/// the auto-calibrated path (the batch every time); a NaN gRef the fallback. +const List<(double?, double, int, int?)> _enmoCfgs = [ + (1.013456789, defaultGravityWindowS, 30, 12), + (1.0, defaultGravityWindowS, 30, null), + (null, defaultGravityWindowS, 30, null), + (1.05, 31, 60, 100), + (1.0, 0.0, 1, null), + (1.0, 1.0, 60, null), + (1.0, 61.5, 30, 12), + (double.nan, defaultGravityWindowS, 30, null), + (double.infinity, defaultGravityWindowS, 30, null), +]; + +class EnmoIn { + EnmoIn(this.samples, this.cfg); + final List samples; + final int cfg; + (double?, double, int, int?) get c => _enmoCfgs[cfg]; +} + +class EnmoOps extends Ops { + @override + String get name => 'IncrementalEnmoSeries'; + + @override + EnmoIn expand(List r) { + final (n, hzIdx, seed, flavour, cfg, originIdx) = (r[0], r[1], r[2], r[3], r[4], r[5]); + final g = Rng(seed * 104729 + flavour + 7); + final hz = _hz[hzIdx]; + final out = []; + for (var i = 0; i < n; i++) { + if (flavour == 1 && i >= 70 && i < 139) continue; + final idx = flavour == 2 ? i ~/ 2 : i; + final ts = _enmoOrigins[originIdx] + idx * 1000 / hz; + if (flavour == 4) { + out.add(AccelSample(ts, .3, -.4, 1.0)); + } else { + out.add(AccelSample( + ts, + .22 * math.sin(i * .29) + g.nextDouble() * .007, + .11 * math.cos(i * .07), + 1.02 + .065 * math.sin(i * .23), + valid: !(flavour == 1 && i % 47 == 46))); + } + } + return EnmoIn(flavour == 3 ? out.reversed.toList() : out, cfg); + } + + @override + int length(EnmoIn i) => i.samples.length; + @override + EnmoIn prefix(EnmoIn i, int k) => EnmoIn(i.samples.sublist(0, k), i.cfg); + @override + EnmoIn edit(EnmoIn i, int kind, int salt) { + final n = i.samples.length; + if (n < 2) return i; + switch (kind % 3) { + case 0: + final a = [...i.samples]; + final o = a[salt % n]; + a[salt % n] = AccelSample(o.tsMs, o.x + .07, o.y, o.z, valid: o.valid); + return EnmoIn(a, i.cfg); + case 1: + return EnmoIn(i.samples.sublist(1 + salt % (n - 1)), i.cfg); + default: + return EnmoIn(i.samples.sublist(0, n - 1 - salt % (n - 1)), i.cfg); + } + } + + @override + EnmoIn? reconfigure(EnmoIn i, int alt) => + EnmoIn(i.samples, (i.cfg + 1 + alt % (_enmoCfgs.length - 1)) % _enmoCfgs.length); + + @override + dynamic fresh(EnmoIn input) => IncrementalEnmoSeries(); + @override + dynamic fromJson(Json j) => IncrementalEnmoSeries.fromJson(j); + @override + Json toJson(dynamic s) => (s as IncrementalEnmoSeries).toJson(); + @override + Object? sync(dynamic s, EnmoIn i, {bool force = false}) => + (s as IncrementalEnmoSeries).sync(i.samples, + gRef: i.c.$1, + gravityWindowS: i.c.$2, + minSamplesPerMinute: i.c.$3, + expectedMinutes: i.c.$4, + force: force); + + @override + String show(Object? out) { + final r = out as EnmoResult; + return [ + r.gRef, + r.coverage, + for (final m in r.minutes) + [m.tsMinStartMs, m.nSamples, m.enmo, m.mad, m.meanMag, m.dynAmp], + ].join('|'); + } + + @override + void close(Object? out, EnmoIn i) => enmoClose( + out as EnmoResult, + enmoSeries(i.samples, + gRef: i.c.$1, + gravityWindowS: i.c.$2, + minSamplesPerMinute: i.c.$3, + expectedMinutes: i.c.$4)); + + @override + List get counters => const ['processedPoints']; + + @override + List get special => const [0, 1, 2, 29, 30, 31, 59, 60, 61, 69, 70, 120, 121]; + + @override + List get refuseMutations => [ + (j) { + final n = (j['valid'] as List).length; + if (n > 0) j['processedPoints'] = n - 1; + }, + (j) => j['lo'] = (j['valid'] as List).length + 1, + (j) { + final bins = j['bins'] as List; + if (bins.isNotEmpty && ((bins.first as Map)['mags'] as List).isNotEmpty) { + ((bins.first as Map)['mags'] as List).removeLast(); + } + }, + (j) => (j['valid'] as List).add([1.0, 0.0, 0.0, 1.0]), + (j) { + if ((j['valid'] as List).isNotEmpty) j['gRef'] = null; + }, + (j) { + final v = j['valid'] as List; + if (v.length >= 2) (v[0] as List)[0] = ((v[1] as List)[0] as num) + 1.0; + }, + (j) { + final v = j['valid'] as List; + if (v.isNotEmpty) (v[0] as List).removeLast(); + }, + (j) => (j['bins'] as List).add({ + 'key': -1, + 'mags': [], + 'magSum': 0.0, + 'enmoSum': 0.0, + 'dynSum': 0.0 + }), + (j) { + final bins = j['bins'] as List; + if (bins.isNotEmpty) bins.add(deepCopy(bins.first)); + }, + (j) { + final bins = j['bins'] as List; + if (bins.isNotEmpty) (bins.first as Map)['enmoSum'] = -1.0; + }, + (j) => j['windowS'] = 'x', + ]; + + @override + List get faithfulMutations => [ + (j) => j['sx'] = (j['sx'] as num) + 1.0, + (j) => j['sy'] = (j['sy'] as num) + 1.0, + (j) => j['sz'] = (j['sz'] as num) + 1.0, + (j) => j['windowS'] = (j['windowS'] as num) + 1.0, + (j) => j['processedPoints'] = (j['processedPoints'] as int) + 5, + (j) { + final bins = j['bins'] as List; + if (bins.isNotEmpty) (bins.first as Map)['magSum'] = ((bins.first as Map)['magSum'] as num) + 1.0; + }, + (j) { + final bins = j['bins'] as List; + if (bins.isNotEmpty) (bins.first as Map)['dynSum'] = ((bins.first as Map)['dynSum'] as num) + 1.0; + }, + (j) { + final v = j['valid'] as List; + if (v.isNotEmpty) (v[0] as List)[1] = ((v[0] as List)[1] as num) + .01; + }, + ]; + + @override + void conserved(Json cp, EnmoIn i, String why) { + // Every valid sample is in exactly one minute bin (the minute its time falls + // in); a state with a non-finite reference or parameter keeps nothing. + final c = i.c; + final usable = c.$1 != null && c.$1!.isFinite && c.$2.isFinite; + final valid = i.samples.where((s) => s.valid).toList(); + if (!usable) { + expect((cp['valid'] as List), isEmpty, reason: 'nothing is kept $why'); + expect((cp['bins'] as List), isEmpty, reason: 'no bins $why'); + return; + } + expect((cp['valid'] as List).length, valid.length, reason: 'valid samples kept $why'); + final want = {}; + for (final s in valid) { + want.update((s.tsMs / 60000).floor(), (v) => v + 1, ifAbsent: () => 1); + } + final got = { + for (final b in cp['bins'] as List) + (b as Map)['key'] as int: (b['mags'] as List).length + }; + expect(got, want, reason: 'every valid sample is in the bin of its minute, once $why'); + } + + @override + Gen> recipeGen(int maxN) => IntsGen([ + SizeGen(maxN, pool: const [0, 1, 2, 3, 29, 30, 31, 59, 60, 61, 69, 70, 120, 121]), + G.intIn(0, _hz.length - 1), + G.intIn(0, 1 << 12), + G.intIn(0, _enmoFlavours.length - 1), + G.intIn(0, _enmoCfgs.length - 1), + G.intIn(0, _enmoOrigins.length - 1), + ]); + + @override + List> get forced => [ + [0, 1, 1, 0, 0, 0], // empty + [1, 1, 1, 0, 0, 0], // one sample + [31, 1, 1, 0, 1, 0], + [200, 1, 1, 1, 0, 1], // gaps + invalid + [300, 2, 1, 2, 3, 1], // duplicate timestamps + [150, 1, 1, 3, 1, 0], // newest first + [130, 1, 1, 4, 1, 1], // constant vector + [200, 0, 2, 0, 2, 1], // auto-calibrated + [200, 1, 2, 0, 7, 0], // NaN reference: the fallback + [200, 1, 2, 0, 8, 2], // infinite reference + [200, 1, 3, 0, 4, 2], // window of zero seconds + [240, 2, 4, 0, 6, 3], // wide window + [180, 1, 5, 0, 5, 2], + ]; + + @override + void observe(List r, void Function(String) bump) { + bump('flavour: ${_enmoFlavours[r[3]]}'); + final n = r[0]; + if (n == 0) bump('series: empty'); + if (n >= 1 && n < 30) bump('series: under a minute of samples'); + if (n >= 120) bump('series: two minutes or more'); + final c = _enmoCfgs[r[4]]; + if (c.$1 == null) bump('reference: auto'); + if (c.$1 != null && !c.$1!.isFinite) bump('reference: not finite'); + if (c.$1 != null && c.$1!.isFinite) bump('reference: given'); + if (c.$2 != defaultGravityWindowS) bump('window: not the default'); + } + + @override + Map get shares => { + for (final f in _enmoFlavours) 'flavour: $f': .03, + 'series: empty': .01, + 'series: under a minute of samples': .05, + 'series: two minutes or more': .2, + 'reference: auto': .05, + 'reference: not finite': .05, + 'reference: given': .4, + 'window: not the default': .2, + }; +} + +// ═════════════════════════════════════════════════════════════════════════════ +// Lomb-Scargle +// ═════════════════════════════════════════════════════════════════════════════ + +const _lombFlavours = [ + 'regular', // beats at ~0.8 s with slow modulation + 'irregular', // uneven gaps, dropouts + 'constant y', // nothing to analyse: abstains + 'collapsed t', // every time the same: abstains + 'duplicate t', // pairs of equal times + 'NaN y', // a non-finite value halfway: the batch decides + 'infinite t', // a non-finite time halfway + 'length mismatch', // one value fewer than times +]; +final _lombGrids = >[ + [0.0, .0033, .01, .04, .07, .1, .15, .25, .4], + [], + [0.0], + [.04, .04, .13], + [.1, .07, .4], + [for (var k = 0; k <= 40; k++) .0033 + k * .01], +]; +const _lombOrigins = [0.0, 12345.125, 1791028730.0]; + +class LombIn { + LombIn(this.t, this.y, this.grid); + final List t, y; + final int grid; + List get freqs => _lombGrids[grid]; +} + +class LombOps extends Ops { + @override + String get name => 'IncrementalLombScargle'; + + @override + LombIn expand(List r) { + final (n, seed, flavour, grid, originIdx) = (r[0], r[1], r[2], r[3], r[4]); + final g = Rng(seed * 6007 + flavour + 13); + var clock = _lombOrigins[originIdx]; + final t = [], y = []; + for (var i = 0; i < n; i++) { + final rr = 800 + 75 * math.sin(i * .13 + seed * .003) + 21 * math.sin(i * .031) + (flavour == 1 ? g.nextDouble() * 400 : 0); + if (flavour == 4) { + if (i.isEven) clock += rr / 1000; + } else if (flavour == 3) { + // time does not move + } else { + clock += rr / 1000; + } + if (flavour == 1 && i % 61 == 60) clock += 21; + t.add(clock); + y.add(flavour == 2 ? 800.0 : rr); + } + if (flavour == 5 && n > 4) y[n ~/ 2] = double.nan; + if (flavour == 6 && n > 4) t[n ~/ 2] = double.infinity; + if (flavour == 7 && n > 0) y.removeLast(); + return LombIn(t, y, grid); + } + + @override + int length(LombIn i) => i.t.length; + @override + LombIn prefix(LombIn i, int k) => + LombIn(i.t.sublist(0, k), i.y.sublist(0, math.min(k, i.y.length)), i.grid); + @override + LombIn edit(LombIn i, int kind, int salt) { + final n = i.t.length; + if (n < 2 || i.y.length != n) return i; + switch (kind % 3) { + case 0: + final y = [...i.y]; + y[salt % n] += 79; + return LombIn(i.t, y, i.grid); + case 1: + final d = 1 + salt % (n - 1); + return LombIn(i.t.sublist(d), i.y.sublist(d), i.grid); + default: + final d = 1 + salt % (n - 1); + return LombIn(i.t.sublist(0, n - d), i.y.sublist(0, n - d), i.grid); + } + } + + // The grid belongs to the constructor (and travels in the checkpoint). + @override + LombIn? reconfigure(LombIn i, int alt) => null; + + @override + dynamic fresh(LombIn input) => IncrementalLombScargle(_lombGrids[input.grid]); + @override + dynamic fromJson(Json j) => IncrementalLombScargle.fromJson(j); + @override + Json toJson(dynamic s) => (s as IncrementalLombScargle).toJson(); + @override + Object? sync(dynamic s, LombIn i, {bool force = false}) => + (s as IncrementalLombScargle).sync(i.t, i.y, force: force); + + @override + String show(Object? out) { + final r = out as LombScargle?; + if (r == null) return 'null'; + return [for (final p in r.spectrum) '${p.freqHz}:${p.psd}'].join('|'); + } + + /// The production batch function on times shifted by the first one (a + /// parity check, not an independent oracle): the periodogram is + /// shift-invariant, the shift is exact, and on raw epoch seconds the batch's + /// own trigonometric arguments lose ~1e-6 relative (drift_test.dart). + LombScargle? _batchOf(LombIn i) { + final ok = i.t.length == i.y.length && + i.t.isNotEmpty && + i.t.every((x) => x.isFinite) && + i.y.every((x) => x.isFinite); + if (!ok) return lombScargle(i.t, i.y, i.freqs); + final first = i.t.first; + return lombScargle([for (final x in i.t) x - first], i.y, i.freqs); + } + + @override + void close(Object? out, LombIn i) => spectrumClose(out as LombScargle?, _batchOf(i)); + + @override + void sameOutput(Object? a, Object? b, String why) { + try { + spectrumClose(a as LombScargle?, b as LombScargle?); + } on TestFailure catch (e) { + fail('$why: ${e.message}'); + } + } + + @override + List get counters => const ['processedPoints']; + + @override + List get special => const [0, 1, 2, 3, 4, 5, 30, 31, 60, 61, 62]; + + @override + List get refuseMutations => [ + (j) => (j['t'] as List).add(1.0), + (j) { + final y = j['y'] as List; + if (y.isNotEmpty) y.removeLast(); + }, + (j) { + final sums = j['sums'] as List; + if (sums.isNotEmpty) (sums.first as List).removeLast(); + }, + (j) { + final sums = j['sums'] as List; + if (sums.isNotEmpty) sums.removeLast(); + }, + (j) => (j['moments'] as Map)['count'] = ((j['moments'] as Map)['count'] as int) + 1, + (j) { + // (Only an incremental state ties its work counter to its data.) + final n = (j['t'] as List).length; + if (n > 0 && j['incremental'] == true) j['processedPoints'] = n - 1; + }, + (j) { + if (j['incremental'] == true && (j['t'] as List).isNotEmpty) { + j['origin'] = (j['origin'] as num) + 1.0; + } + }, + (j) { + if (j['incremental'] == true && (j['t'] as List).isNotEmpty) { + j['tMax'] = (j['tMax'] as num) + 1.0; + } + }, + (j) { + if (j['incremental'] == true && (j['t'] as List).isNotEmpty) { + final sums = j['sums'] as List; + if (sums.isNotEmpty) (sums.first as List)[2] = ((sums.first as List)[2] as num) + 5.0; + } + }, + (j) { + if (((j['moments'] as Map)['count'] as int) > 0) j['incremental'] = false; + }, + (j) => (j['frequencies'] as List).add(.5), + (j) { + if (j['incremental'] == true && (j['y'] as List).isNotEmpty) { + j['yOrigin'] = (j['yOrigin'] as num) + 1.0; + } + }, + ]; + + @override + List get faithfulMutations => [ + (j) { + final sums = j['sums'] as List; + if (sums.isNotEmpty) (sums.first as List)[0] = ((sums.first as List)[0] as num) + 1.0; + }, + (j) { + final sums = j['sums'] as List; + if (sums.isNotEmpty) (sums.first as List)[4] = ((sums.first as List)[4] as num) + 1.0; + }, + (j) { + final sums = j['sums'] as List; + if (sums.isNotEmpty) (sums.last as List)[5] = ((sums.last as List)[5] as num) + 1.0; + }, + (j) => j['processedPoints'] = (j['processedPoints'] as int) + 3, + (j) { + final m = j['moments'] as Map; + m['m2'] = ((m['m2'] as num) + 1.0); + }, + ]; + + @override + void conserved(Json cp, LombIn i, String why) { + final finite = i.t.length == i.y.length && + i.t.every((x) => x.isFinite) && + i.y.every((x) => x.isFinite); + if (!finite) { + // The batch decides; nothing is folded into the sums. + expect((cp['moments'] as Map)['count'], 0, reason: 'no moments $why'); + return; + } + final n = i.t.length; + expect((cp['moments'] as Map)['count'], n, reason: 'every value is a moment, once $why'); + expect((cp['t'] as List).length, n, reason: why); + // cos^2 + sin^2 = 1 at every point: the two squares sum to n per frequency. + if (n > 0) { + for (final row in cp['sums'] as List) { + final r = (row as List).cast(); + expect((r[2] + r[3] - n).abs() <= 1e-8 * n, isTrue, + reason: 'sum of cos^2 and sin^2 is the count ${r[2] + r[3]} vs $n $why'); + } + } + } + + @override + Gen> recipeGen(int maxN) => IntsGen([ + SizeGen(maxN, pool: const [0, 1, 2, 3, 4, 5, 30, 31, 60, 61, 62]), + G.intIn(0, 1 << 12), + G.intIn(0, _lombFlavours.length - 1), + G.intIn(0, _lombGrids.length - 1), + G.intIn(0, _lombOrigins.length - 1), + ]); + + @override + List> get forced => [ + [0, 1, 0, 0, 0], + [3, 1, 0, 0, 0], // under 4 points: abstains + [4, 1, 0, 0, 1], // exactly 4 + [60, 1, 0, 0, 0], + [60, 2, 0, 0, 2], // epoch-sized times + [100, 3, 1, 5, 0], // irregular, dense grid + [40, 1, 2, 0, 0], // constant y + [40, 1, 3, 0, 0], // collapsed time + [40, 1, 4, 0, 1], // duplicate times + [40, 1, 5, 0, 0], // NaN value + [40, 1, 6, 0, 0], // infinite time + [40, 1, 7, 0, 0], // lengths differ + [50, 1, 0, 1, 0], // empty grid + [50, 1, 0, 2, 0], // grid of zero only + [50, 1, 0, 3, 1], // repeated frequency + [50, 1, 0, 4, 0], // unsorted grid + ]; + + @override + void observe(List r, void Function(String) bump) { + bump('flavour: ${_lombFlavours[r[2]]}'); + final n = r[0]; + if (n == 0) bump('series: empty'); + if (n >= 1 && n < 4) bump('series: under 4 points'); + if (n >= 60) bump('series: 60 or more'); + bump('grid: ${r[3]}'); + if (r[4] == 2) bump('epoch-sized times'); + } + + @override + Map get shares => { + for (final f in _lombFlavours) 'flavour: $f': .02, + 'series: under 4 points': .03, + 'series: 60 or more': .3, + for (var g = 0; g < _lombGrids.length; g++) 'grid: $g': .03, + 'epoch-sized times': .15, + }; +} + +// ═════════════════════════════════════════════════════════════════════════════ +// Minute metrics +// ═════════════════════════════════════════════════════════════════════════════ + +const _hrBranches = [54.0, 95.0, 106.799999, 106.8, 130.0, 190.0, 0.0, double.nan, 40.0]; +const _cadBranches = [null, 99.999, 100, 110, 120, 130, 160, 0, double.nan]; + +/// (resting HR, max HR, sex, profile name, day minutes, quiet HRR). Anchors +/// that gate the TRIMP off (null, equal, inverted, NaN), a negative resting HR, +/// no profile (no energy), a short, a DST and a long day, and quiet-HRR gates. +final List<(double?, double?, Sex, String?, int, double?)> _minCfgs = [ + (54, 186, Sex.male, 'male', 1440, .12), + (65, 197, Sex.female, 'female', 900, .12), + (null, 186, Sex.male, 'male', 1440, .12), + (54, null, Sex.male, 'nonbinary', 1440, null), + (70, 60, Sex.male, 'male', 1440, .18), + (double.nan, 186, Sex.male, 'male', 0, .12), + (54, 186, Sex.female, null, 1440, .12), + (54, 186, Sex.male, 'nonbinary', 1380, 0.0), + (54, 186, Sex.male, 'female', 1441, -0.1), + (54, 186, Sex.male, 'male', 1500, double.nan), + (54, 54, Sex.male, 'male', 1440, .12), + // A resting HR below zero is accepted (finite, below the max): an HR of 0 + // (off-skin) is still billed nothing, where HR 0 would otherwise read as half + // the reserve. No profile, so the energy pricer does not take the anchors. + (-1, 1, Sex.male, null, 1440, .12), +]; + +class MinIn { + MinIn(this.keys, this.hr, this.cad, this.cfg, this.series); + final List keys; + final List hr; + final List? cad; + final int cfg; + final bool series; + WorkoutUserProfile? get profile { + final name = _minCfgs[cfg].$4; + return name == null ? null : energyProfiles[name]; + } +} + +String _metric(Metric m) => + '${m.present}/${m.value}/${m.confidence}/${m.tier}/${m.note}/${m.inputs_used}'; + +const _minTotals = ['trimpTotal', 'hrActiveTotal', 'walkingTotal']; + +class MinOps extends Ops { + @override + String get name => 'IncrementalMinuteMetrics'; + + @override + MinIn expand(List r) { + final (n, seed, cfg, cadMode, series) = (r[0], r[1], r[2], r[3], r[4]); + return MinIn( + List.generate(n, (i) => 28000000 + i + i ~/ 17), + List.generate(n, (i) => _hrBranches[(i + seed) % _hrBranches.length]), + cadMode == 0 ? null : List.generate(n, (i) => _cadBranches[(i + seed) % _cadBranches.length]), + cfg, + series == 1, + ); + } + + @override + int length(MinIn i) => i.keys.length; + @override + MinIn prefix(MinIn i, int k) => MinIn(i.keys.sublist(0, k), i.hr.sublist(0, k), + i.cad?.sublist(0, k), i.cfg, i.series); + @override + MinIn edit(MinIn i, int kind, int salt) { + final n = i.keys.length; + if (n < 2) return i; + switch (kind % 3) { + case 0: + // One minute's HR replaced, or two minutes swapped as whole items, or + // every minute in the opposite order: the keys are then not ascending + // and every key keeps its value, so the bills are reused in a new order. + switch (salt ~/ 3 % 3) { + case 0: + final hr = [...i.hr]; + hr[salt % n] += 29; + return MinIn(i.keys, hr, i.cad, i.cfg, i.series); + case 1: + final a = salt % n, b = (salt * 7 + 1) % n == a ? (a + 1) % n : (salt * 7 + 1) % n; + return _reorder(i, [ + for (var k = 0; k < n; k++) k == a ? b : (k == b ? a : k) + ]); + default: + return _reorder(i, [for (var k = n - 1; k >= 0; k--) k]); + } + case 1: + final d = 1 + salt % (n - 1); + return MinIn(i.keys.sublist(d), i.hr.sublist(d), i.cad?.sublist(d), i.cfg, i.series); + default: + final d = 1 + salt % (n - 1); + return MinIn(i.keys.sublist(0, n - d), i.hr.sublist(0, n - d), + i.cad?.sublist(0, n - d), i.cfg, i.series); + } + } + + /// The same minutes, each with its own HR and cadence, in the order [from]. + static MinIn _reorder(MinIn i, List from) => MinIn( + [for (final k in from) i.keys[k]], + [for (final k in from) i.hr[k]], + i.cad == null ? null : [for (final k in from) i.cad![k]], + i.cfg, + i.series); + + @override + MinIn? reconfigure(MinIn i, int alt) => + MinIn(i.keys, i.hr, i.cad, (i.cfg + 1 + alt % (_minCfgs.length - 1)) % _minCfgs.length, i.series); + + @override + dynamic fresh(MinIn input) => IncrementalMinuteMetrics(); + @override + dynamic fromJson(Json j) => IncrementalMinuteMetrics.fromJson(j); + @override + Json toJson(dynamic s) => (s as IncrementalMinuteMetrics).toJson(); + @override + Object? sync(dynamic s, MinIn i, {bool force = false}) { + final c = _minCfgs[i.cfg]; + return (s as IncrementalMinuteMetrics).sync(i.keys, i.hr, + cadenceSpm: i.cad, + restingHr: c.$1, + maxHr: c.$2, + sex: c.$3, + profile: i.profile, + dayMinutes: c.$5, + quietHrr: c.$6, + force: force, + includeMinuteSeries: i.series); + } + + @override + String show(Object? out) { + final m = out as MinuteMetrics; + final e = m.energy, s = m.minutes; + return [ + _metric(m.trimp), + _metric(m.strain), + if (e != null) [e.total, e.active, e.basal, e.walking], + if (s != null) ...[ + s.coveredMinutes, + s.abstainedMinutes, + s.basalKcalPerMin, + s.active, + s.walking, + for (final x in s.minutes) + [x.minute, x.source, x.abstained, x.basal, x.active, x.total, x.walking], + ], + ].join('|'); + } + + @override + void close(Object? out, MinIn i) { + final m = out as MinuteMetrics; + final c = _minCfgs[i.cfg]; + if (i.series) { + minuteClose(m, i.keys, i.hr, + cadence: i.cad, + rhr: c.$1, + maxHr: c.$2, + sex: c.$3, + profile: i.profile, + dayMinutes: c.$5, + quietHrr: c.$6); + return; + } + // A summary request omits the minute artifacts and equals the batch's + // trimp, strain and daily energy. + expect(m.minutes, isNull, reason: 'summary requests omit minute artifacts'); + final trimp = banisterTrimp(i.hr, restingHr: c.$1, maxHr: c.$2, sex: c.$3); + final strain = strainScoreMetric(trimp.value, + wakeMinutes: i.hr.length.toDouble(), + quietHrr: c.$6, + quietSettled: false, + female: c.$3 == Sex.female); + metricEnvelope(m.trimp, trimp); + numberClose(m.trimp.value, trimp.value); + metricEnvelope(m.strain, strain); + numberClose(m.strain.value, strain.value); + final p = i.profile; + final energy = p == null || c.$1 == null || c.$2 == null + ? null + : Calories.dailyEnergy(i.hr, + profile: p, + hrmax: c.$2!, + restingHr: c.$1!, + dayMinutes: c.$5, + cadenceSpmPerMin: i.cad); + if (energy == null) { + expect(m.energy, isNull); + } else { + expect(m.energy, isNotNull); + numberClose(m.energy!.total, energy.total); + numberClose(m.energy!.active, energy.active); + numberClose(m.energy!.basal, energy.basal); + numberClose(m.energy!.walking, energy.walking); + } + } + + @override + List get counters => const ['processedMinutes']; + + // After a detour the bills may have been added in another order and the totals + // carry add/remove float residue: the checkpoint is equal in meaning, not in + // text. The projection is BY KEY: every field but the bills and the three + // totals as text (the counters aside), the bills as a map from minute key to + // bill (each bill exact: a bill is computed from one minute alone), the totals + // within the reader's own tolerance, 1e-9 relative. + @override + void sameState(Json got, Json want, String why) { + Map rest(Json j) => { + for (final e in j.entries) + if (!counters.contains(e.key) && + e.key != 'bills' && + !_minTotals.contains(e.key)) + e.key: e.value + }; + expect(jsonEncode(rest(got)), jsonEncode(rest(want)), reason: '$why: parameters'); + Map byKey(Json j) { + final out = {}; + for (final b in j['bills'] as List) { + final m = (b as Map).cast(); + expect(out.containsKey(m['key']), isFalse, reason: '$why: a minute billed twice'); + out[m['key']] = jsonEncode(m); + } + return out; + } + + expect(byKey(got), byKey(want), reason: '$why: the bills, by minute key'); + for (final t in _minTotals) { + final g = (got[t] as num).toDouble(), w = (want[t] as num).toDouble(); + expect((g - w).abs() <= 1e-9 * math.max(1, w.abs()), isTrue, + reason: '$why: $t $g vs $w'); + } + } + + // The same series after a detour: its sums were added to and taken from, so + // the numbers are the batch's within the batch tolerance, not bit for bit; + // the minute series (a bill per minute, in key order) is exact. + @override + void sameOutput(Object? a, Object? b, String why) { + final x = a as MinuteMetrics, y = b as MinuteMetrics; + metricEnvelope(x.trimp, y.trimp); + numberClose(x.trimp.value, y.trimp.value, reason: '$why: trimp'); + metricEnvelope(x.strain, y.strain); + numberClose(x.strain.value, y.strain.value, reason: '$why: strain'); + expect(x.energy == null, y.energy == null, reason: '$why: energy present'); + if (x.energy != null) { + numberClose(x.energy!.total, y.energy!.total, reason: '$why: total'); + numberClose(x.energy!.active, y.energy!.active, reason: '$why: active'); + numberClose(x.energy!.basal, y.energy!.basal, reason: '$why: basal'); + numberClose(x.energy!.walking, y.energy!.walking, reason: '$why: walking'); + } + expect(x.minutes == null, y.minutes == null, reason: '$why: series present'); + if (x.minutes != null) { + final p = x.minutes!, q = y.minutes!; + expect([for (final m in p.minutes) [m.minute, m.source, m.abstained, m.basal, m.active, m.total, m.walking]], + [for (final m in q.minutes) [m.minute, m.source, m.abstained, m.basal, m.active, m.total, m.walking]], + reason: '$why: the minute series'); + expect((p.coveredMinutes, p.abstainedMinutes, p.basalKcalPerMin), + (q.coveredMinutes, q.abstainedMinutes, q.basalKcalPerMin), + reason: '$why: series counts'); + numberClose(p.active, q.active, reason: '$why: series active'); + numberClose(p.walking, q.walking, reason: '$why: series walking'); + } + } + + @override + List get special => const [0, 1, 2, 3, 29, 30, 31, 60, 61]; + + @override + List<({String name, bool Function(MinIn) applies, void Function(dynamic, MinIn) call})> + get refusals => [ + ( + name: 'one more HR than keys', + applies: (i) => true, + call: (s, i) => (s as IncrementalMinuteMetrics) + .sync(i.keys, [...i.hr, 70], cadenceSpm: i.cad) + ), + ( + name: 'one more cadence than keys', + applies: (i) => true, + call: (s, i) => (s as IncrementalMinuteMetrics).sync(i.keys, i.hr, + cadenceSpm: [...(i.cad ?? List.filled(i.hr.length, null)), 100]) + ), + ( + name: 'a key appended twice', + applies: (i) => i.keys.isNotEmpty, + call: (s, i) => (s as IncrementalMinuteMetrics) + .sync([...i.keys, i.keys.first], [...i.hr, 70]) + ), + ( + name: 'a duplicate inside a reordering', + applies: (i) => i.keys.length >= 2, + call: (s, i) => (s as IncrementalMinuteMetrics).sync( + [i.keys[1], i.keys[0], i.keys[1]], [70, 71, 72]) + ), + ]; + + Map _bill(Json j, int i) => ((j['bills'] as List)[i] as Map); + + @override + List get refuseMutations => [ + (j) { + final n = (j['bills'] as List).length; + if (n > 0) j['processedMinutes'] = n - 1; + }, + (j) => j['trimpTotal'] = (j['trimpTotal'] as num) + 5.0, + (j) { + final b = j['bills'] as List; + if (b.isNotEmpty) b.add(deepCopy(b.first)); + }, + (j) { + if ((j['bills'] as List).isNotEmpty) _bill(j, 0)['trimp'] = -1.0; + }, + (j) { + if ((j['bills'] as List).isNotEmpty) { + _bill(j, 0)['source'] = 'hr'; + _bill(j, 0)['walking'] = 5.0; + } + }, + (j) { + if ((j['bills'] as List).isNotEmpty) j['parameters'] = null; + }, + (j) { + if ((j['bills'] as List).any((b) => (b as Map)['source'] != null)) { + j['basalPerMinute'] = null; + } + }, + (j) => j['hrActiveTotal'] = (j['hrActiveTotal'] as num) + 5.0, + (j) { + final p = j['parameters'] as List?; + if (p != null) p[2] = 'bogus'; + }, + (j) { + final p = j['parameters'] as List?; + if (p != null) p.removeLast(); + }, + (j) { + if ((j['bills'] as List).isNotEmpty) _bill(j, 0)['source'] = 'nonsense'; + }, + (j) { + if ((j['bills'] as List).isNotEmpty) _bill(j, 0)['key'] = 'x'; + }, + (j) => j['bills'] = 'x', + (j) => j['walkingTotal'] = (j['walkingTotal'] as num) + 5.0, + ]; + + @override + List get faithfulMutations => [ + (j) { + if ((j['bills'] as List).isNotEmpty) _bill(j, 0)['hr'] = 111.0; + }, + (j) { + if ((j['bills'] as List).isNotEmpty) _bill(j, 0)['cadence'] = 111.0; + }, + (j) { + final p = j['parameters'] as List?; + if (p != null && p[0] != null && p[0] is num) p[0] = (p[0] as num) + 1.0; + }, + (j) => j['basalPerMinute'] = ((j['basalPerMinute'] as num?) ?? 0) + 1.0, + (j) => j['processedMinutes'] = (j['processedMinutes'] as int) + 5, + ]; + + @override + void conserved(Json cp, MinIn i, String why) { + final bills = cp['bills'] as List; + expect(bills.length, i.keys.length, reason: 'a bill for every minute, once $why'); + expect({for (final b in bills) (b as Map)['key']}, i.keys.toSet(), + reason: 'the bills are the minutes handed in $why'); + var trimp = 0.0, active = 0.0, walking = 0.0; + for (final b in bills) { + final m = b as Map; + trimp += m['trimp'] as num; + active += m['active'] as num; + walking += m['walking'] as num; + } + double tol(double v) => 1e-9 * math.max(1, v.abs()); + expect((cp['trimpTotal'] as num) - trimp, inInclusiveRange(-tol(trimp), tol(trimp)), + reason: 'the total is the sum of the bills (trimp) $why'); + expect((cp['hrActiveTotal'] as num) - active, inInclusiveRange(-tol(active), tol(active)), + reason: 'the total is the sum of the bills (active) $why'); + expect((cp['walkingTotal'] as num) - walking, inInclusiveRange(-tol(walking), tol(walking)), + reason: 'the total is the sum of the bills (walking) $why'); + // Each bill echoes the minute it prices. + final byKey = {for (final b in bills) (b as Map)['key']: b}; + for (var k = 0; k < i.keys.length; k++) { + final b = byKey[i.keys[k]]!; + final h = i.hr[k]; + expect(b['hr'], h.isNaN ? 'NaN' : h, reason: 'hr of ${i.keys[k]} $why'); + final c = i.cad?[k]; + expect(b['cadence'], c == null ? isNull : (c.isNaN ? 'NaN' : c), + reason: 'cadence of ${i.keys[k]} $why'); + } + } + + @override + Gen> recipeGen(int maxN) => IntsGen([ + SizeGen(maxN, pool: const [0, 1, 2, 3, 29, 30, 31, 60, 61]), + G.intIn(0, 1 << 12), + G.intIn(0, _minCfgs.length - 1), + G.elements(const [0, 1, 1]), + G.elements(const [0, 1]), + ]); + + @override + List> get forced => [ + [0, 1, 0, 0, 1], // no minutes + [0, 1, 6, 1, 0], + [1, 1, 0, 1, 1], // one minute + [31, 1, 0, 1, 1], + [61, 3, 1, 1, 0], + [47, 5, 2, 1, 1], // resting HR unknown + [47, 5, 3, 0, 0], // max HR unknown, quiet gate absent + [47, 5, 4, 1, 1], // anchors inverted + [47, 5, 5, 1, 0], // NaN anchor, day of zero minutes + [47, 5, 6, 1, 1], // no profile + [47, 5, 7, 1, 0], // DST-short day, quiet gate 0 + [47, 5, 8, 1, 1], // long day, negative quiet gate + [47, 5, 9, 1, 0], // NaN quiet gate + [47, 5, 10, 1, 1], // anchors equal + [47, 5, 11, 1, 1], // resting HR below zero: HR 0 is off-skin, billed nothing + [47, 5, 11, 0, 0], + [90, 914, 0, 1, 0], + ]; + + @override + void observe(List r, void Function(String) bump) { + final n = r[0]; + if (n == 0) bump('series: empty'); + if (n >= 1 && n < 30) bump('series: under 30 minutes'); + if (n >= 60) bump('series: an hour or more'); + final c = _minCfgs[r[2]]; + if (c.$1 == null || c.$2 == null || c.$1!.isNaN) bump('anchors: missing'); + if (c.$1 != null && c.$2 != null && c.$1!.isFinite && c.$1! >= c.$2!) bump('anchors: not ordered'); + if (c.$4 == null) bump('no profile'); + if (c.$1 != null && c.$1! < 0) bump('anchors: negative resting HR'); + if (c.$3 == Sex.female) bump('sex: female'); + if (r[3] == 1) bump('cadence given'); + bump(r[4] == 1 ? 'minute series requested' : 'summary only'); + } + + @override + Map get shares => { + 'series: empty': .01, + 'series: under 30 minutes': .05, + 'series: an hour or more': .1, + 'anchors: missing': .1, + 'anchors: not ordered': .03, + 'anchors: negative resting HR': .03, + 'no profile': .05, + 'sex: female': .05, + 'cadence given': .4, + 'minute series requested': .3, + 'summary only': .3, + }; +} + +// ═════════════════════════════════════════════════════════════════════════════ +// RunningMoments +// ═════════════════════════════════════════════════════════════════════════════ + +const _momFlavours = [ + 'heart rate', // whole bpm, a spread of 40 + 'large offset', // 1e9 plus a spread under 1: where sum-of-squares loses every digit + 'constant', // all equal: variance exactly zero + 'wide', // magnitudes from 1e-3 to 1e6, both signs + 'two valued', + 'small spread near zero', +]; + +/// The values of a recipe (flavour, n, seed). +List _momValues(int flavour, int n, int seed) { + final g = Rng(seed * 8191 + flavour + 3); + return [ + for (var i = 0; i < n; i++) + switch (flavour) { + 0 => 40.0 + g.nextInt(120), + 1 => 1e9 + (g.nextDouble() * .5 + (i % 7) * .25), + 2 => 42.0, + 3 => (g.nextBool() ? 1 : -1) * math.pow(10, g.nextDouble() * 9 - 3).toDouble(), + 4 => i.isEven ? 800.0 : 1000.0, + _ => (g.nextDouble() - .5) * 1e-3, + } + ]; +} + +/// Mean and sample / population SD of [x] the stable way: about the first value, +/// two passes. A reference written here (unlike the batch functions the sync +/// modules are compared with): independent of the Welford state under test, and +/// accurate where a plain sum is not (a 1e9 offset). +({double? mean, double? sd, double? pop}) _twoPass(List x) { + final n = x.length; + if (n == 0) return (mean: null, sd: null, pop: null); + final x0 = x.first; + var s = 0.0; + for (final v in x) { + s += v - x0; + } + final md = s / n; + var ss = 0.0; + for (final v in x) { + final d = v - x0 - md; + ss += d * d; + } + return ( + mean: x0 + md, + sd: n < 2 ? null : math.sqrt(ss / (n - 1)), + pop: math.sqrt(ss / n), + ); +} + +const double _eps = 2.220446049250313e-16; + +/// The moments hold what two passes say: the count exact, the mean to a few +/// ulps of its own size, the SDs to 1e-9 relative (plus the same ulps). +void _momentsMatch(RunningMoments m, List x, String why) { + final o = _twoPass(x); + expect(m.count, x.length, reason: 'count $why'); + if (x.isEmpty) { + expect(m.mean, isNull, reason: 'no values, no mean $why'); + expect(m.sampleSd, isNull, reason: why); + expect(m.populationSd, isNull, reason: why); + return; + } + final scale = x.fold(1, (a, v) => math.max(a, v.abs())); + final tol = 64 * _eps * scale; + expect((m.mean! - o.mean!).abs() <= tol + 1e-12 * (o.sd ?? 0), isTrue, + reason: 'mean ${m.mean} vs ${o.mean} (tol $tol) $why'); + void sdClose(double? got, double? want, String what) { + if (want == null) { + expect(got, isNull, reason: '$what is absent below two values $why'); + return; + } + expect(got, isNotNull, reason: '$what $why'); + final t = 1e-9 * want + tol; + expect((got! - want).abs() <= t, isTrue, + reason: '$what $got vs $want (tol $t) $why'); + } + + sdClose(m.sampleSd, o.sd, 'sample sd'); + sdClose(m.populationSd, o.pop, 'population sd'); +} + +RunningMoments _momOf(List x) { + final m = RunningMoments(); + for (final v in x) { + m.add(v); + } + return m; +} + +String _momText(RunningMoments m) => jsonEncode(m.toJson()); + +/// (flavour, n, seed, splits, cut seed). +typedef MomCase = List; + +final Gen _momGen = IntsGen([ + G.intIn(0, _momFlavours.length - 1), + SizeGen(400, pool: const [0, 1, 2, 3, 4, 5, 10]), + G.intIn(0, 1 << 12), + G.intIn(0, 6), + G.intIn(0, 1 << 20), +]); + +List> _parts(MomCase c, List x) { + final b = foldBounds(x.length, c[3], c[4]); + return [for (var k = 0; k + 1 < b.length; k++) x.sublist(b[k], b[k + 1])]; +} + +void _observeMom(MomCase c, void Function(String) bump) { + bump('flavour: ${_momFlavours[c[0]]}'); + if (c[1] == 0) bump('empty'); + if (c[1] == 1) bump('one value'); + if (c[1] >= 100) bump('100 values or more'); + final b = foldBounds(c[1], c[3], c[4]); + if (b.length >= 4) bump('three parts or more'); + if (hasEmptyChunk(b)) bump('an empty part'); +} + +final Map _momShares = { + for (final f in _momFlavours) 'flavour: $f': .03, + 'empty': .01, + 'one value': .01, + '100 values or more': .2, + 'three parts or more': .4, + 'an empty part': .05, +}; + +final List _momForced = [ + [0, 0, 1, 2, 3], // nothing + [0, 1, 1, 2, 3], // one value + [0, 2, 1, 2, 3], + [1, 300, 1, 3, 5], // large offset + [2, 80, 1, 3, 7], // constant + [3, 200, 2, 4, 11], // wide + [4, 51, 1, 6, 13], // two valued, odd + [5, 120, 4, 5, 17], + [0, 100, 3, 6, 19], // heart rate +]; + +void _momMerge(MomCase c) { + final x = _momValues(c[0], c[1], c[2]); + final parts = _parts(c, x); + final tag = 'moments $c parts=${[for (final p in parts) p.length]}'; + final ms = [for (final p in parts) _momOf(p)]; + final before = [for (final m in ms) _momText(m)]; + // Left to right: merging the summaries of the parts is the summary of the + // concatenation. + final left = RunningMoments(); + for (final m in ms) { + left.merge(m); + } + _momentsMatch(left, x, '$tag: left fold'); + for (var k = 0; k < ms.length; k++) { + expect(_momText(ms[k]), before[k], reason: '$tag: the argument is left unchanged'); + } + // Associativity: (a+b)+c and a+(b+c) read the same as the concatenation; the + // state differs only in rounding. + if (parts.length >= 3) { + final ab = _momOf(parts[0])..merge(_momOf(parts[1])); + final abc = ab..merge(_momOf([for (final p in parts.skip(2)) ...p])); + final bc = _momOf(parts[1])..merge(_momOf([for (final p in parts.skip(2)) ...p])); + final a_bc = _momOf(parts[0])..merge(bc); + _momentsMatch(abc, x, '$tag: (a+b)+c'); + _momentsMatch(a_bc, x, '$tag: a+(b+c)'); + } + // Commutativity: the order of two parts does not matter to what is read. + if (parts.length >= 2) { + final ab = _momOf(parts[0])..merge(_momOf(parts[1])); + final ba = _momOf(parts[1])..merge(_momOf(parts[0])); + final both = [...parts[0], ...parts[1]]; + _momentsMatch(ab, both, '$tag: a+b'); + _momentsMatch(ba, both, '$tag: b+a'); + } + // Identity, exactly: merging the empty summary changes nothing, and merging + // into the empty summary copies. + final whole = _momOf(x); + final text = _momText(whole); + whole.merge(RunningMoments()); + expect(_momText(whole), text, reason: '$tag: x + empty == x'); + expect(_momText(RunningMoments()..merge(whole)), text, reason: '$tag: empty + x == x'); + // A summary merged with itself is the multiset twice. + final twice = _momOf(x)..merge(whole); + _momentsMatch(twice, [...x, ...x], '$tag: x + x'); + final selfMerged = _momOf(x); + selfMerged.merge(selfMerged); + _momentsMatch(selfMerged, [...x, ...x], '$tag: merge with itself'); +} + +void _momAddRemove(MomCase c) { + final x = _momValues(c[0], c[1], c[2]); + final tag = 'moments $c'; + final g = Rng(c[4] + 5); + final m = RunningMoments(); + final held = []; + // A walk of adds and removes (of values that are held), checked against two + // passes along the way. + for (var i = 0; i < x.length; i++) { + m.add(x[i]); + held.add(x[i]); + if (held.length > 2 && g.nextBool(.4)) { + final v = held.removeAt(g.nextInt(held.length)); + m.remove(v); + } + if (i % 7 == 0 || i == x.length - 1) _momentsMatch(m, held, '$tag step $i'); + } + // add then remove the same value: back where it was. (A value inside the + // data's range: removing a downdate of a wild outlier is a numerical limit of + // every running variance, not a contract.) + final probe = _momOf(held); + final text = _momText(probe); + final extra = held.isEmpty ? 123.5 : held.last; + probe.add(extra); + probe.remove(extra); + _momentsMatch(probe, held, '$tag: add then remove'); + if (held.isEmpty) expect(_momText(probe), text, reason: '$tag: back to the empty state exactly'); + // Emptied again: exactly the fresh state. + while (held.isNotEmpty) { + m.remove(held.removeLast()); + } + expect(_momText(m), _momText(RunningMoments()), reason: '$tag: emptied is fresh'); +} + +void _momJson(MomCase c) { + final x = _momValues(c[0], c[1], c[2]); + final tag = 'moments $c'; + final m = _momOf(x); + final text = _momText(m); + final back = RunningMoments.fromJson((jsonDecode(text) as Map).cast()); + expect(_momText(back), text, reason: '$tag: write(read(b)) == b'); + _momentsMatch(back, x, '$tag: restored reads the same'); + // And goes on identically. + m.add(77); + back.add(77); + expect(_momText(back), _momText(m), reason: '$tag: same after one more'); + _momentsMatch(back, [...x, 77], tag); + // Identity and refusals leave the state unchanged. + final before = _momText(m); + expect(() => m.add(double.nan), throwsArgumentError); + expect(() => m.add(double.infinity), throwsArgumentError); + expect(() => m.remove(double.negativeInfinity), throwsArgumentError); + expect(() => RunningMoments().remove(1), throwsStateError); + expect(_momText(m), before, reason: '$tag: refused calls change nothing'); + expect(m.count, x.length + 1); + // Conservation: (n - 1) sample variance and n population variance are the + // same sum of squares. + if (m.count >= 2) { + final a = m.sampleSd! * m.sampleSd! * (m.count - 1); + final b = m.populationSd! * m.populationSd! * m.count; + expect((a - b).abs() <= 1e-9 * math.max(1, a.abs()), isTrue, + reason: '$tag: one sum of squares: $a vs $b'); + } +} + +(Json, Must) _momMutate(Json src, Mut m) { + final j = (deepCopy(src) as Map).cast(); + final (kind, a, b) = m; + switch (kind % 8) { + case 0: + const versions = [0, 2, -1, 99, '1', null, true, 1 << 40]; + j['version'] = versions[a % versions.length]; + return (j, Must.refuse); + case 1: + const types = ['IntHistogram', 'Other', null, '', 1]; + j['type'] = types[a % types.length]; + return (j, Must.refuse); + case 2: + j.remove(const ['count', 'origin', 'meanOffset', 'm2', 'version', 'type'][a % 6]); + return (j, Must.refuse); + case 3: + const junk = ['x', {}, [], null, 1.5]; + final k = const ['count', 'origin', 'meanOffset', 'm2'][a % 4]; + j[k] = k == 'count' ? junk[b % junk.length] : junk[b % 4]; + return (j, Must.refuse); + case 4: + // Contradictions a reachable state never shows. + switch (a % 5) { + case 0: + j['count'] = -1 - b % 5; + case 1: + j['m2'] = -1.0 - b % 5; + case 2: + j + ..['count'] = 0 + ..['origin'] = 1.0 + b % 7; // nothing counted, yet an origin + case 3: + j + ..['count'] = 1 + ..['m2'] = 1.0 + b % 7; // one value with a spread + default: + j + ..['count'] = 0 + ..['m2'] = 1.0; + } + return (j, jsonEncode(j) == jsonEncode(src) ? Must.faithful : Must.refuse); + case 5: + // Changes the reader cannot tell from a legal state. + switch (a % 4) { + case 0: + if ((j['count'] as int) >= 2) j['m2'] = (j['m2'] as num) + 1.0; + case 1: + if ((j['count'] as int) >= 1) j['meanOffset'] = (j['meanOffset'] as num) + 1.0; + case 2: + if ((j['count'] as int) >= 1) j['origin'] = (j['origin'] as num) + 1.0; + default: + if ((j['count'] as int) >= 2) j['count'] = (j['count'] as int) + 1 + b % 3; + } + return (j, Must.faithful); + case 6: + // Hostile size: the count claims far more than the sums can hold. + if ((j['count'] as int) >= 2) j['count'] = const [1 << 40, 1 << 62, 0x7fffffffffffffff][b % 3]; + return (j, Must.faithful); + default: + j['futureField${a % 3}'] = const [1, 'x', [1]][b % 3]; + return (j, Must.ignored); + } +} + +void _momL3((MomCase, Mut) arg) { + final (c, m) = arg; + final tag = 'moments $c mutation=${m.$1 % 8}(${m.$2},${m.$3})'; + final src = (jsonDecode(_momText(_momOf(_momValues(c[0], c[1], c[2])))) as Map).cast(); + final (bad, must) = _momMutate(src, m); + String outcome; + try { + outcome = 'accepted:${_momText(RunningMoments.fromJson((deepCopy(bad) as Map).cast()))}'; + } on FormatException { + outcome = 'refused'; + } catch (e) { + outcome = 'WRONG ERROR TYPE: ${e.runtimeType}: $e'; + } + expectMutationOutcome(outcome, must, _text(bad), _text(src), tag); +} + +// ═════════════════════════════════════════════════════════════════════════════ +// IntHistogram +// ═════════════════════════════════════════════════════════════════════════════ + +const _histFlavours = [ + 'heart rate', // 40..180 bpm + 'few values', // 3 distinct + 'one value', + 'negatives', // -100..299 + 'wide', // up to +-1e9 +]; + +List _histValues(int flavour, int n, int seed) { + final g = Rng(seed * 4099 + flavour + 1); + return [ + for (var i = 0; i < n; i++) + switch (flavour) { + 0 => 40 + g.nextInt(141), + 1 => 60 + g.nextInt(3), + 2 => 61, + 3 => g.nextInt(400) - 100, + _ => g.nextInt(2000000000) - 1000000000, + } + ]; +} + +IntHistogram _histOf(List x) { + final h = IntHistogram(); + for (final v in x) { + h.add(v); + } + return h; +} + +String _histText(IntHistogram h) => jsonEncode(h.toJson()); + +const _ps = [0, 1, 5, 25, 30, 33.3, 50, 66.6, 75, 90, 95, 99, 100]; + +/// The histogram reads exactly what the sorted expanded list reads: the very +/// double `percentileSorted` gives, count and distinct count exact, and a +/// canonical checkpoint (sorted bins, no zero counts). +void _histMatches(IntHistogram h, List x, String why) { + final sorted = [for (final v in x) v.toDouble()]..sort(); + expect(h.count, x.length, reason: 'count $why'); + expect(h.distinct, x.toSet().length, reason: 'distinct $why'); + if (x.isEmpty) { + expect(h.median, isNull, reason: 'no values, no median $why'); + for (final p in _ps) { + expect(h.percentile(p), isNull, reason: 'p$p $why'); + } + } else { + for (final p in _ps) { + expect(h.percentile(p), percentileSorted(sorted, p), reason: 'p$p $why'); + } + expect(h.median, percentileSorted(sorted, 50), reason: 'median $why'); + } + final j = h.toJson(); + expect(j['values'], ([...x.toSet()]..sort()), reason: 'sorted distinct bins $why'); + final counts = {}; + for (final v in x) { + counts.update(v, (c) => c + 1, ifAbsent: () => 1); + } + expect(j['counts'], [for (final v in j['values'] as List) counts[v]], reason: 'counts $why'); +} + +void _observeHist(MomCase c, void Function(String) bump) { + bump('flavour: ${_histFlavours[c[0]]}'); + if (c[1] == 0) bump('empty'); + if (c[1] == 1) bump('one value'); + if (c[1] >= 100) bump('100 values or more'); + final b = foldBounds(c[1], c[3], c[4]); + if (b.length >= 4) bump('three parts or more'); + if (hasEmptyChunk(b)) bump('an empty part'); +} + +final Map _histShares = { + for (final f in _histFlavours) 'flavour: $f': .03, + 'empty': .01, + 'one value': .01, + '100 values or more': .2, + 'three parts or more': .4, + 'an empty part': .05, +}; + +final Gen _histGen = IntsGen([ + G.intIn(0, _histFlavours.length - 1), + SizeGen(400, pool: const [0, 1, 2, 3, 4, 5, 10]), + G.intIn(0, 1 << 12), + G.intIn(0, 6), + G.intIn(0, 1 << 20), +]); + +final List _histForced = [ + [0, 0, 1, 2, 3], + [0, 1, 1, 2, 3], + [2, 2, 1, 2, 3], // two equal values + [1, 7, 1, 3, 5], + [0, 300, 1, 4, 7], + [3, 150, 2, 5, 9], + [4, 90, 3, 6, 11], + [0, 5, 4, 6, 13], +]; + +void _histMerge(MomCase c) { + final x = _histValues(c[0], c[1], c[2]); + final b = foldBounds(x.length, c[3], c[4]); + final parts = [for (var k = 0; k + 1 < b.length; k++) x.sublist(b[k], b[k + 1])]; + final tag = 'histogram $c parts=${[for (final p in parts) p.length]}'; + final hs = [for (final p in parts) _histOf(p)]; + final before = [for (final h in hs) _histText(h)]; + // The summary of the concatenation, exactly (bins and checkpoint included). + final left = IntHistogram(); + for (final h in hs) { + left.merge(h); + } + _histMatches(left, x, '$tag: left fold'); + expect(_histText(left), _histText(_histOf(x)), reason: '$tag: same checkpoint as one fold'); + for (var k = 0; k < hs.length; k++) { + expect(_histText(hs[k]), before[k], reason: '$tag: the argument is left unchanged'); + } + // Associative and commutative, exactly: the checkpoint is canonical. + if (parts.length >= 3) { + final rest = [for (final p in parts.skip(2)) ...p]; + final ab_c = _histOf(parts[0])..merge(_histOf(parts[1]))..merge(_histOf(rest)); + final a_bc = _histOf(parts[0])..merge(_histOf(parts[1])..merge(_histOf(rest))); + expect(_histText(ab_c), _histText(a_bc), reason: '$tag: associative'); + expect(_histText(ab_c), _histText(_histOf(x)), reason: '$tag: and the concatenation'); + } + if (parts.length >= 2) { + final ab = _histOf(parts[0])..merge(_histOf(parts[1])); + final ba = _histOf(parts[1])..merge(_histOf(parts[0])); + expect(_histText(ab), _histText(ba), reason: '$tag: commutative'); + } + // Identity. + final whole = _histOf(x); + final text = _histText(whole); + whole.merge(IntHistogram()); + expect(_histText(whole), text, reason: '$tag: x + empty == x'); + expect(_histText(IntHistogram()..merge(whole)), text, reason: '$tag: empty + x == x'); + // Itself, doubled. + final self = _histOf(x); + self.merge(self); + _histMatches(self, [...x, ...x], '$tag: merge with itself'); +} + +void _histAddRemove(MomCase c) { + final x = _histValues(c[0], c[1], c[2]); + final tag = 'histogram $c'; + final g = Rng(c[4] + 9); + final h = IntHistogram(); + final held = []; + for (var i = 0; i < x.length; i++) { + if (held.isNotEmpty && g.nextBool(.4)) { + final v = held.removeAt(g.nextInt(held.length)); + h.remove(v); + } else { + held.add(x[i]); + h.add(x[i]); + } + if (i % 5 == 0 || i == x.length - 1) _histMatches(h, held, '$tag step $i'); + } + // Refusals leave the histogram exactly as it was. + final before = _histText(h); + final absent = held.fold(0, (a, v) => math.max(a, v.abs())) + 1; + expect(() => h.remove(absent), throwsStateError, reason: tag); + expect(() => h.remove(double.nan), throwsStateError, reason: tag); + expect(() => h.remove(1.5), throwsStateError, reason: tag); + expect(() => h.add(double.nan), throwsArgumentError, reason: tag); + expect(() => h.add(double.infinity), throwsArgumentError, reason: tag); + expect(() => h.add(70.5), throwsArgumentError, reason: tag); + expect(() => h.add(70, 0), throwsArgumentError, reason: tag); + expect(() => h.add(70, -2), throwsArgumentError, reason: tag); + if (h.count > 0) { + // (An empty histogram answers null for any p; the range is checked once + // there is something to read.) + expect(() => h.percentile(-1), throwsArgumentError, reason: tag); + expect(() => h.percentile(100.5), throwsArgumentError, reason: tag); + expect(() => h.percentile(double.nan), throwsArgumentError, reason: tag); + } + expect(_histText(h), before, reason: '$tag: refused calls change nothing'); + // Repeat counts: add(v, k) is k adds of v; removing is one at a time. + final r = IntHistogram()..add(60, 3)..add(70, 2); + _histMatches(r, [60, 60, 60, 70, 70], '$tag: repeat count'); + r.remove(60); + _histMatches(r, [60, 60, 70, 70], '$tag: removing one of several'); + expect(_histText(IntHistogram()..add(5, 4)), _histText(_histOf([5, 5, 5, 5])), reason: tag); + // A whole-number double is a whole number. + final w = IntHistogram()..add(70.0); + _histMatches(w, [70], tag); + // Add then remove: where it was, no zero-count bin left behind. + final t = _histOf(held); + final text = _histText(t); + t.add(1234567); + t.remove(1234567); + expect(_histText(t), text, reason: '$tag: add then remove'); +} + +void _histJson(MomCase c) { + final x = _histValues(c[0], c[1], c[2]); + final tag = 'histogram $c'; + final h = _histOf(x); + final text = _histText(h); + final back = IntHistogram.fromJson((jsonDecode(text) as Map).cast()); + expect(_histText(back), text, reason: '$tag: write(read(b)) == b'); + _histMatches(back, x, '$tag: restored'); + back.add(55); + h.add(55); + expect(_histText(back), _histText(h), reason: '$tag: same after one more'); + _histMatches(back, [...x, 55], tag); + // Conservation: the counts add up to the count, every bin is held once. + final j = back.toJson(); + expect((j['counts'] as List).fold(0, (a, v) => a + (v as int)), back.count, reason: tag); +} + +(Json, Must) _histMutate(Json src, Mut m) { + final j = (deepCopy(src) as Map).cast(); + final (kind, a, b) = m; + List l(String k) => j[k] as List; + switch (kind % 8) { + case 0: + const versions = [0, 2, -1, 99, '1', null, true, 1 << 40]; + j['version'] = versions[a % versions.length]; + return (j, Must.refuse); + case 1: + const types = ['RunningMoments', 'Other', null, '', 1]; + j['type'] = types[a % types.length]; + return (j, Must.refuse); + case 2: + j.remove(const ['values', 'counts', 'version', 'type'][a % 4]); + return (j, Must.refuse); + case 3: + const junk = ['x', {}, null, 1.5, 3]; + j[const ['values', 'counts'][a % 2]] = junk[b % junk.length]; + return (j, Must.refuse); + case 4: + // The bins contradict themselves. + switch (a % 8) { + case 0: + l('values').add(1 << 40); + case 1: + if (l('counts').isNotEmpty) l('counts').removeLast(); + case 2: + if (l('counts').isNotEmpty) l('counts')[b % l('counts').length] = 0; + case 3: + if (l('counts').isNotEmpty) l('counts')[b % l('counts').length] = -1 - b % 4; + case 4: + if (l('values').length >= 2) { + final t = l('values')[0]; + l('values')[0] = l('values')[1]; + l('values')[1] = t; // out of order + } + case 5: + if (l('values').isNotEmpty) { + l('values').insert(0, l('values')[0]); // a repeated value + l('counts').insert(0, 1); + } + case 6: + if (l('values').isNotEmpty) l('values')[0] = 1.5; + default: + if (l('values').isNotEmpty) l('counts')[0] = 1.5; + } + return (j, jsonEncode(j) == jsonEncode(src) ? Must.faithful : Must.refuse); + case 5: + // A bin count changed: legal. + if (l('counts').isNotEmpty) l('counts')[b % l('counts').length] = (l('counts')[b % l('counts').length] as int) + 1 + a % 5; + return (j, Must.faithful); + case 6: + // Hostile counts: each bin plausible, the total past what an int can + // hold (the skipped FINDING below). + if (l('counts').isNotEmpty) l('counts')[0] = const [1 << 40, 1 << 61, 0x7fffffffffffffff][b % 3]; + return (j, Must.faithful); + default: + j['futureField${a % 3}'] = const [1, 'x', [1]][b % 3]; + return (j, Must.ignored); + } +} + +void _histL3((MomCase, Mut) arg) { + final (c, m) = arg; + final tag = 'histogram $c mutation=${m.$1 % 8}(${m.$2},${m.$3})'; + final src = (jsonDecode(_histText(_histOf(_histValues(c[0], c[1], c[2])))) as Map).cast(); + final (bad, must) = _histMutate(src, m); + String outcome; + try { + outcome = 'accepted:${_histText(IntHistogram.fromJson((deepCopy(bad) as Map).cast()))}'; + } on FormatException { + outcome = 'refused'; + } catch (e) { + outcome = 'WRONG ERROR TYPE: ${e.runtimeType}: $e'; + } + expectMutationOutcome(outcome, must, _text(bad), _text(src), tag); +} + +// ═════════════════════════════════════════════════════════════════════════════ +// CalculationCache +// ═════════════════════════════════════════════════════════════════════════════ + +/// The cache against a model written from its documented behaviour: bounded +/// least-recently-used results, each with an owned snapshot of the dependencies +/// it was computed from; a hit needs the same key and DEEP-equal dependencies +/// (NaN equals NaN) and no `full` request; a hit and a computation are counted; +/// a calculation that throws changes nothing; results are copies; the +/// dependencies are snapshotted before the calculation runs, so a calculation +/// that mutates them does not change what the entry was computed from. +class _CacheModel { + _CacheModel(this.cap); + final int cap; + final List<(String, Object?, Object?)> entries = []; // oldest first + int computations = 0, hits = 0; + + static bool eq(Object? a, Object? b) { + if (a is num && b is num) return a == b || (a.isNaN && b.isNaN); + if (a is List && b is List) { + return a.length == b.length && [for (var i = 0; i < a.length; i++) eq(a[i], b[i])].every((x) => x); + } + if (a is Map && b is Map) { + return a.length == b.length && a.keys.every((k) => b.containsKey(k) && eq(a[k], b[k])); + } + return a == b; + } + + static Object? copy(Object? v) => v is List + ? [for (final e in v) copy(e)] + : v is Map + ? {for (final e in v.entries) e.key: copy(e.value)} + : v; + + Object? evaluate(String key, Object? deps, Object? Function() calc, bool full) { + final at = entries.indexWhere((e) => e.$1 == key); + if (!full && at >= 0 && eq(entries[at].$2, deps)) { + final e = entries.removeAt(at); + entries.add(e); + hits++; + return copy(e.$3); + } + // The dependencies are snapshotted BEFORE the callback runs (the callback + // is user code and may mutate them): the entry holds what the caller passed + // in, and a later lookup with that value hits (CalculationCache.evaluate). + final snapshot = copy(deps); + final v = calc(); // may throw: nothing changes + if (at >= 0) entries.removeAt(at); + entries.add((key, snapshot, copy(v))); + while (entries.length > cap) { + entries.removeAt(0); + } + computations++; + return copy(v); + } +} + +/// A dependency: a small structure with ties, NaN, nesting and null. +Object? _dep(Rng g, int depth) { + switch (g.nextInt(depth > 0 ? 7 : 5)) { + case 0: + return g.nextInt(3); + case 1: + return null; + case 2: + return g.nextBool() ? double.nan : 1.5; + case 3: + return 'k${g.nextInt(2)}'; + case 4: + return g.nextBool(); + case 5: + return [for (var i = g.nextInt(3); i > 0; i--) _dep(g, depth - 1)]; + default: + return {for (var i = g.nextInt(3); i > 0; i--) 'a${g.nextInt(3)}': _dep(g, depth - 1)}; + } +} + +/// Dependencies a mutating calculation works on: nested lists and maps. +Object? _mutableDeps(Object? pooled) => { + 'p': _CacheModel.copy(pooled), + 'q': [1, [2, 3], 'x'], + 'r': {'s': [4, 5], 't': {'u': 6}}, + }; + +/// What a user calculation might do to the dependencies it was handed: change +/// them at depth, grow and shrink them. +void _mutateDeps(Object? d) { + final m = d as Map; + ((m['q'] as List)[1] as List)[0] = 99; + (m['q'] as List).add('added'); + ((m['r'] as Map)['s'] as List).removeLast(); + (((m['r'] as Map)['t']) as Map)['u'] = 7; + m['v'] = 1; +} + +void _cacheModelLaw(List c) { + // (steps, capacity, seed, how often the calculation mutates its dependencies) + final steps = c[0], cap = c[1] + 1, seed = c[2], mutMode = c[3]; + final g = Rng(seed * 31 + 7); + final cache = CalculationCache(maxEntries: cap); + final model = _CacheModel(cap); + final tag = 'cache steps=$steps cap=$cap seed=$seed mut=$mutMode'; + var made = 0; + // The calculations that actually ran, on each side: the cache runs the + // callback exactly when the model says it misses (or is told to recompute), + // failures included. + var cacheRuns = 0, modelRuns = 0, failures = 0; + // A small pool of dependency values, so equal dependencies recur. + final pool = [for (var i = 0; i < 4; i++) _dep(Rng(seed + i * 17), 2)]; + + /// One evaluation on both sides, each with its own copy of the dependencies + /// and its own callback (so a mutation lands on its own copy). + void both(String why, String key, Object? deps, bool full, bool throws, + bool mutates, List value) { + final depsC = _CacheModel.copy(deps), depsM = _CacheModel.copy(deps); + Object? calcC() { + cacheRuns++; + if (mutates) _mutateDeps(depsC); + if (throws) throw StateError('calculation failed'); + return [...value]; + } + + Object? calcM() { + modelRuns++; + if (mutates) _mutateDeps(depsM); + if (throws) throw StateError('calculation failed'); + return [...value]; + } + + Object? got, want; + Object? gotErr, wantErr; + try { + got = cache.evaluate(key, depsC, calcC, full: full); + } catch (e) { + gotErr = e; + failures++; + } + try { + want = model.evaluate(key, depsM, calcM, full); + } catch (e) { + wantErr = e; + } + expect(gotErr.runtimeType, wantErr.runtimeType, reason: '$tag $why: failures propagate'); + expect(got, want, reason: '$tag $why: result'); + expect(cache.computations, model.computations, reason: '$tag $why: computations'); + expect(cache.hits, model.hits, reason: '$tag $why: hits'); + expect(cacheRuns, modelRuns, reason: '$tag $why: calculations that ran'); + expect(cacheRuns, cache.computations + failures, + reason: '$tag $why: one run per computation or failure, none on a hit'); + // The result is the caller's: changing it changes no later hit. + if (got is List && got.isNotEmpty) got[0] = -99; + // The dependencies handed in are snapshotted: changing them afterwards + // does not turn a later hit into a miss (or the reverse). + if (depsC is List && depsC.isNotEmpty) depsC.add('mutated'); + if (depsM is List && depsM.isNotEmpty) depsM.add('mutated'); + } + + for (var i = 0; i < steps; i++) { + final key = 'key${g.nextInt(cap + 2)}'; + final mutates = mutMode > 0 && g.nextBool(mutMode * .2); + final picked = pool[g.nextInt(pool.length)]; + final deps = mutates ? _mutableDeps(picked) : picked; + final full = g.nextBool(.15); + final throws = g.nextBool(.1); + final value = [made++, made % 3]; + both('step $i', key, deps, full, throws, mutates, value); + if (mutates) { + // The contract: the entry was snapshotted before the callback, so a + // lookup with the value passed in (not the mutated one) is the hit, and + // the mutated value is a different dependency. + both('step $i, original value again', key, deps, false, false, false, [-1, -1]); + final after = _mutableDeps(picked); + _mutateDeps(after); + both('step $i, mutated value', key, after, false, false, false, [-2, -2]); + } + } + // Size: never more than the capacity, each key held once. + var held = 0; + for (var k = 0; k < cap + 2; k++) { + var probeCalls = 0; + final before = cache.computations; + cache.evaluate('key$k', _CacheModel.copy(null), () { + probeCalls++; + return [0]; + }); + if (cache.computations == before) held++; + expect(probeCalls <= 1, isTrue); + } + expect(held <= cap, isTrue, reason: '$tag: at most $cap entries held, held $held'); + cache.clear(); + var recomputed = false; + cache.evaluate('key0', null, () => recomputed = true); + expect(recomputed, isTrue, reason: '$tag: clear discards'); +} + +void main() { + _registerSyncLaws(HrvOps(), maxN: 800); + _registerSyncLaws(EnmoOps(), maxN: 900); + _registerSyncLaws(LombOps(), maxN: 300); + _registerSyncLaws(MinOps(), maxN: 300); + group('RunningMoments', () { + _laws.law( + 'M RunningMoments.merge: the summary of the parts merged is the summary of ' + 'the concatenation; associative, commutative, the empty summary is the ' + 'identity, a summary merged with itself is the multiset twice', + _momGen, + _momMerge, + examples: _momForced, + cases: 200, + reach: Reach(_momShares, _observeMom), + ); + _laws.law( + 'L1b RunningMoments: a walk of adds and removes reads what two passes read', + _momGen, + _momAddRemove, + examples: _momForced, + cases: 120, + reach: Reach(_momShares, _observeMom), + ); + _laws.law( + 'L2 RunningMoments: write(read(b)) == b, goes on identically, refused calls ' + 'change nothing, one sum of squares', + _momGen, + _momJson, + examples: _momForced, + cases: 120, + reach: Reach(_momShares, _observeMom), + ); + _laws.law<(MomCase, Mut)>( + 'L3 RunningMoments: a mutated checkpoint is refused whole or holds exactly ' + 'what was written', + G.pair(_momGen, mutGen(8)), + _momL3, + examples: [ + for (var k = 0; k < 8; k++) + for (var i = 0; i < 4; i++) (_momForced[3 + i], (k, 2 * k + i, 3 * k + i)), + for (var a = 0; a < 5; a++) (_momForced[3], (4, a, 1)), + for (var a = 0; a < 5; a++) (_momForced[1], (4, a, 1)), + ], + cases: 200, + reach: Reach<(MomCase, Mut)>({ + for (var k = 0; k < 8; k++) 'kind: $k': .03, + }, (arg, bump) => bump('kind: ${arg.$2.$1 % 8}')), + ); + }); + + group('IntHistogram', () { + _laws.law( + 'M IntHistogram.merge: the histogram of the parts merged is exactly the ' + 'histogram of the concatenation; associative, commutative, the empty ' + 'histogram is the identity', + _histGen, + _histMerge, + examples: _histForced, + cases: 200, + reach: Reach(_histShares, _observeHist), + ); + _laws.law( + 'L1b IntHistogram: a walk of adds and removes reads exactly what the ' + 'sorted list reads; refused calls change nothing', + _histGen, + _histAddRemove, + examples: _histForced, + cases: 150, + reach: Reach(_histShares, _observeHist), + ); + _laws.law( + 'L2 IntHistogram: write(read(b)) == b, goes on identically, the counts add ' + 'up', + _histGen, + _histJson, + examples: _histForced, + cases: 150, + reach: Reach(_histShares, _observeHist), + ); + _laws.law<(MomCase, Mut)>( + 'L3 IntHistogram: a mutated checkpoint is refused whole or holds exactly ' + 'what was written', + G.pair(_histGen, mutGen(8)), + _histL3, + examples: [ + for (var k = 0; k < 8; k++) + for (var i = 0; i < 4; i++) (_histForced[3 + i], (k, 2 * k + i, 3 * k + i)), + for (var a = 0; a < 8; a++) (_histForced[4], (4, a, 1)), + ], + cases: 200, + reach: Reach<(MomCase, Mut)>({ + for (var k = 0; k < 8; k++) 'kind: $k': .03, + }, (arg, bump) => bump('kind: ${arg.$2.$1 % 8}')), + ); + }); + + group('CalculationCache', () { + _laws.law>( + 'CalculationCache equals a model of bounded LRU results with deep-compared ' + 'dependency snapshots: results, hits, computations, failures, capacity', + IntsGen([G.intIn(0, 80), G.intIn(0, 6), G.intIn(0, 1 << 12), G.intIn(0, 3)]), + _cacheModelLaw, + examples: [ + [0, 0, 1, 0], + [1, 0, 2, 0], + [40, 0, 3, 0], // capacity 1 + [60, 2, 4, 0], // capacity 3 + [80, 6, 5, 0], // capacity 7 + [80, 1, 6, 0], + // Calculations that mutate their dependencies: always (mode 5 is a + // certainty), often, now and then. + [1, 0, 7, 5], // one step, always: the smallest case + [30, 0, 8, 5], // capacity 1, always + [60, 3, 9, 3], + [80, 6, 10, 2], + ], + cases: 200, + reach: Reach>({ + 'capacity 1': .05, + 'capacity 3 or more': .3, + 'many steps': .4, + 'calculations that mutate their dependencies': .5, + }, (c, bump) { + if (c[1] == 0) bump('capacity 1'); + if (c[1] >= 2) bump('capacity 3 or more'); + if (c[0] >= 40) bump('many steps'); + if (c[3] > 0 && c[0] > 0) bump('calculations that mutate their dependencies'); + }), + ); + }); + + test('CalculationCache snapshots the dependencies before the calculation runs, ' + 'so one that mutates them leaves the entry keyed on the original value', () { + // Written out, not from the model: the contract in evaluate() is "snapshot + // before calling user code, which may itself mutate dependencies". + Map original() => { + 'q': [1, [2, 3]], + 'r': {'s': [4, 5]}, + }; + final cache = CalculationCache(); + final deps = original(); + var runs = 0; + expect(cache.evaluate>('k', deps, () { + runs++; + ((deps['q'] as List)[1] as List)[0] = 99; // nested, in place + ((deps['r'] as Map)['s'] as List).add(6); + (deps['q'] as List).add('more'); + return [runs]; + }), [1]); + expect(deps['q'], [1, [99, 3], 'more'], reason: 'the callback did mutate them'); + expect((runs, cache.computations, cache.hits), (1, 1, 0)); + // The ORIGINAL value hits the entry computed from it: no new run. + expect(cache.evaluate>('k', original(), () { + runs++; + return [-1]; + }), [1], reason: 'the original value hits'); + expect((runs, cache.computations, cache.hits), (1, 1, 1)); + // The mutated value is a different dependency: a miss, the callback runs. + expect(cache.evaluate>('k', deps, () { + runs++; + return [2]; + }), [2], reason: 'the mutated value is a miss'); + expect((runs, cache.computations, cache.hits), (2, 2, 1)); + }); + + group('reader boundaries, written out at the edge (not drawn)', () { + double nextUp(double v) { + final bits = ByteData(8)..setFloat64(0, v); + final i = bits.getInt64(0); + bits.setInt64(0, v >= 0 ? i + 1 : i - 1); + return bits.getFloat64(0); + } + + double nextDown(double v) => -nextUp(-v); + + IncrementalMinuteMetrics readMin(Json j) => + IncrementalMinuteMetrics.fromJson( + (jsonDecode(jsonEncode(j)) as Map).cast()); + + test('IncrementalMinuteMetrics: the trimp total may differ from its bills ' + 'by 1e-9 relative -- the edge itself and just inside are read, just ' + 'outside is refused', () { + // No bills: the sum is 0 and the tolerance is exactly 1e-9 * max(1, 0). + const tol = 1e-9; + Json empty(double total) => IncrementalMinuteMetrics().toJson()..['trimpTotal'] = total; + for (final sign in [1.0, -1.0]) { + for (final total in [sign * tol, sign * nextDown(tol), sign * 0.5 * tol]) { + expect(readMin(empty(total)).toJson()['trimpTotal'], total, + reason: 'a total $total from no bills is within the tolerance'); + } + for (final total in [sign * nextUp(tol), sign * 2 * tol]) { + expect(() => readMin(empty(total)), throwsFormatException, + reason: 'a total $total from no bills is outside the tolerance'); + } + } + // With bills the tolerance scales with the sum of their trimp. + final ops = MinOps(); + final x = ops.expand([61, 3, 0, 1, 1]); + final live = ops.fresh(x) as IncrementalMinuteMetrics; + ops.sync(live, x); + final j = live.toJson(); + var sum = 0.0; + for (final b in j['bills'] as List) { + sum += (b as Map)['trimp'] as num; + } + expect(sum, greaterThan(1), reason: 'the bills carry some load'); + final scale = 1e-9 * math.max(1, sum); + expect(readMin(deepCopy(j) as Json..['trimpTotal'] = sum + 0.5 * scale) + .toJson()['trimpTotal'], sum + 0.5 * scale); + expect(readMin(deepCopy(j) as Json..['trimpTotal'] = sum - 0.5 * scale) + .toJson()['trimpTotal'], sum - 0.5 * scale); + expect(() => readMin(deepCopy(j) as Json..['trimpTotal'] = sum + 2 * scale), + throwsFormatException); + expect(() => readMin(deepCopy(j) as Json..['trimpTotal'] = sum - 2 * scale), + throwsFormatException); + }); + + test('IncrementalMinuteMetrics: an HR of 0 (off-skin) is billed nothing ' + 'whatever the anchors, a negative resting HR included', () { + final ops = MinOps(); + // Config 11: resting HR -1, max HR 1; an HR of 0 would be half the reserve. + final x = MinIn([28000000, 28000001, 28000002], [0.0, 40.0, 0.0], null, 11, true); + final out = ops.sync(ops.fresh(x), x) as MinuteMetrics; + final batch = banisterTrimp(x.hr, restingHr: -1, maxHr: 1, sex: Sex.male); + expect(batch.present, isTrue); + numberClose(out.trimp.value, batch.value); + final only = MinIn([28000000], [0.0], null, 11, true); + final none = ops.sync(ops.fresh(only), only) as MinuteMetrics; + expect(none.trimp.value, 0, reason: 'only an off-skin minute: no load'); + }); + + test('IncrementalLombScargle: a cos^2 or sin^2 sum of exactly zero is read, ' + 'a negative one is refused', () { + // The sums of squares are never negative; 0 is a legal value (the sums + // only have to add up to the number of points). A reader that refuses 0 + // refuses a checkpoint the sums permit. + final ops = LombOps(); + final x = ops.expand([40, 7, 0, 0, 0]); + final live = ops.fresh(x) as IncrementalLombScargle; + ops.sync(live, x); + final j = live.toJson(); + final rows = j['sums'] as List; + final total = ((rows[2] as List)[2] as num) + ((rows[2] as List)[3] as num); + IncrementalLombScargle read(Json m) => + IncrementalLombScargle.fromJson((jsonDecode(jsonEncode(m)) as Map).cast()); + for (final which in [2, 3]) { + final ok = deepCopy(j) as Json; + final row = (ok['sums'] as List)[2] as List; + row[which] = 0.0; + row[5 - which] = total; + expect(read(ok).toJson()['sums'], ok['sums'], + reason: 'a zero in column $which is read as written'); + // Negative, with the other column making up the total: only the sign is wrong. + final bad = deepCopy(j) as Json; + final badRow = (bad['sums'] as List)[2] as List; + badRow[which] = -1e-12; + badRow[5 - which] = total + 1e-12; + expect(() => read(bad), throwsFormatException, + reason: 'a negative sum in column $which is refused'); + } + }); + + // Accepted-checkpoint continuations. The reader accepts some checkpoints a + // writer never produces (a sum changed, still self-consistent); the state + // then goes on from exactly what was written. A sync that adds nothing must + // not quietly rebuild it from the series, so these hold the altered value + // through an identical sync. + Json viaText(Json j) => (jsonDecode(jsonEncode(j)) as Map).cast(); + + // If reader hardening ever refuses this altered state, this becomes a refusal test. + test('IncrementalHrvTime: an accepted two-beat checkpoint keeps its level ' + 'moments through an identical sync', () { + final live = IncrementalHrvTime()..sync([800.0, 900.0]); + final j = viaText(live.toJson()); + expect(((j['levels'] as Map)['m2'] as num).toDouble(), 5000.0); + (j['levels'] as Map)['m2'] = 20000.0; // positive: the reader cannot tell + final back = IncrementalHrvTime.fromJson(viaText(j)); + final out = back.sync([800.0, 900.0]); + expect(out.value!.sdnn, closeTo(math.sqrt(20000.0), 1e-9), + reason: 'the SDNN is the one the state holds, not a rebuilt 70.7'); + expect(((back.toJson()['levels'] as Map)['m2'] as num).toDouble(), 20000.0); + }); + + // If reader hardening ever refuses this altered state, this becomes a refusal test. + test('IncrementalEnmoSeries: accepted altered minute sums survive an ' + 'identical sync', () { + final a = [for (var i = 0; i < 90; i++) AccelSample(i * 1000.0, .3, -.4, 1)]; + final live = IncrementalEnmoSeries()..sync(a, gRef: 1); + final j = viaText(live.toJson()); + final bin = (j['bins'] as List)[0] as Map; + final n = (bin['mags'] as List).length; + bin['enmoSum'] = 3.0; + final back = IncrementalEnmoSeries.fromJson(viaText(j)); + final out = back.sync(a, gRef: 1); + expect(out.minutes.first.enmo, closeTo(3.0 / n, 1e-12), + reason: 'the minute reads the sum the checkpoint holds'); + expect(((back.toJson()['bins'] as List)[0] as Map)['enmoSum'], 3.0); + }); + + // If reader hardening ever refuses this altered state, this becomes a refusal test. + test('IncrementalLombScargle: accepted altered spectral sums survive an ' + 'identical sync', () { + final t = [for (var i = 0; i < 40; i++) i * .8 + (i % 3) * .05]; + final y = [for (var i = 0; i < 40; i++) 800.0 + 50 * (i % 5)]; + final live = IncrementalLombScargle([.04, .1, .25]); + final batch = live.sync(t, y)!; + final j = viaText(live.toJson()); + ((j['sums'] as List)[1] as List)[5] = 1234.5; // centred y * cos: unchecked + final back = IncrementalLombScargle.fromJson(viaText(j)); + final out = back.sync(t, y)!; + expect(out.spectrum[0].psd, batch.spectrum[0].psd, reason: 'the other rows are untouched'); + expect(out.spectrum[1].psd, isNot(closeTo(batch.spectrum[1].psd, 1e-6)), + reason: 'the altered row reads the sums the checkpoint holds'); + expect(((back.toJson()['sums'] as List)[1] as List)[5], 1234.5); + }); + + test('IncrementalMinuteMetrics: a detour through reordered minutes gives ' + 'the checkpoint (by key) and the output of one sync', () { + final ops = MinOps(); + for (final r in [ + [61, 3, 0, 1, 1], + [47, 5, 1, 0, 0], + [31, 1, 0, 1, 1], + [9, 4, 6, 1, 1], + ]) { + final x = ops.expand(r); + final single = ops.fresh(x) as IncrementalMinuteMetrics; + final want = ops.sync(single, x); + for (var salt = 3; salt < 9; salt++) { + // 3..5 swap two whole minutes, 6..8 put them all in reverse. + final other = ops.edit(x, 0, salt); + expect(other.keys.toSet(), x.keys.toSet(), reason: 'the same minutes $r $salt'); + expect(other.keys, isNot(x.keys), reason: 'in another order $r $salt'); + final st = ops.fresh(x) as IncrementalMinuteMetrics; + ops.sync(st, x); + ops.close(ops.sync(st, other), other); + final back = _restore(ops, st); + final got = ops.sync(back, x); + ops.sameState(ops.toJson(back), ops.toJson(single), 'reorder $r $salt'); + ops.sameOutput(got, want, 'reorder $r $salt'); + } + } + }); + }); + + test('CalculationCache refuses a capacity under one entry', () { + for (final bad in [0, -1, -100]) { + expect(() => CalculationCache(maxEntries: bad), throwsArgumentError, + reason: 'maxEntries: $bad'); + } + expect(CalculationCache(maxEntries: 1).maxEntries, 1); + expect(CalculationCache().maxEntries, 128); + }); + + group('FINDINGS (skipped: suspected gaps against the documented contract)', () { + test('IntHistogram public overflow checks preserve state', () { + const maxInt64 = 0x7fffffffffffffff; + Map snapshot(IntHistogram h) { + final json = h.toJson(); + return { + 'count': h.count, + 'values': json['values'], + 'counts': json['counts'], + 'toJson': json, + }; + } + + final h = IntHistogram()..add(7, maxInt64); + expect(h.count, maxInt64); + expect(h.toJson()['counts'], [maxInt64]); + final before = snapshot(h); + expect(() => h.add(7), throwsStateError); + expect(snapshot(h), before); + + final other = IntHistogram()..add(8); + expect(() => h.merge(other), throwsStateError); + expect(snapshot(h), before); + + expect(() => h.merge(h), throwsStateError); + expect(snapshot(h), before); + }); + + test('IntHistogram.fromJson refuses bin counts whose total does not fit an ' + 'int', () { + final bad = >[ + {'version': 1, 'type': 'IntHistogram', 'values': [1, 2, 3], 'counts': [1 << 62, 1 << 62, 1 << 62]}, + {'version': 1, 'type': 'IntHistogram', 'values': [1, 2], 'counts': [0x7fffffffffffffff, 1]}, + ]; + for (final j in bad) { + // Today: accepted, `count` wraps negative, and a later `percentile` + // throws StateError('Order statistic out of range'), not the + // FormatException a malformed checkpoint is documented to raise. + expect(() => IntHistogram.fromJson(j), throwsFormatException, reason: '$j'); + } + }); + }); + + group('the forced scenarios say what they claim', () { + test('HRV: empty, jitter gate, constant, dropouts, several 5-minute bins, no ' + 'times, an artifact fraction', () { + final ops = HrvOps(); + var absent = false, gated = false, multiBin = false, noTimes = false; + var constant = false, gaps = false, af = false; + for (final r in ops.forced) { + final x = ops.expand(r); + final st = ops.fresh(x); + final out = ops.sync(st, x) as Metric; + final j = ops.toJson(st); + if (!out.present) absent = true; + if (out.present && out.value!.rmssd == null && out.note!.startsWith('rmssd_refused')) { + gated = true; + } + if ((j['bins'] as List).length >= 2) multiBin = true; + if (x.t == null && x.nn.length >= 2) noTimes = true; + if (r[0] == 3 && x.nn.length >= 31) constant = true; + if (x.t != null) { + for (var i = 1; i < x.nn.length; i++) { + if (x.t![i] - x.t![i - 1] > x.nn[i] + .5) gaps = true; + } + } + if (x.af > 0) af = true; + } + expect([absent, gated, multiBin, noTimes, constant, gaps, af], + everyElement(isTrue)); + }); + + test('ENMO: empty, gaps, invalid samples, duplicate and unsorted times, a ' + 'given, an auto and a non-finite reference', () { + final ops = EnmoOps(); + var empty = false, invalid = false, dup = false, unsorted = false; + var given = false, auto = false, nonFinite = false, multiMinute = false; + for (final r in ops.forced) { + final x = ops.expand(r); + final out = ops.sync(ops.fresh(x), x) as EnmoResult; + if (x.samples.isEmpty) empty = true; + if (x.samples.any((s) => !s.valid)) invalid = true; + for (var i = 1; i < x.samples.length; i++) { + if (x.samples[i].tsMs == x.samples[i - 1].tsMs) dup = true; + if (x.samples[i].tsMs < x.samples[i - 1].tsMs) unsorted = true; + } + final g = x.c.$1; + if (g == null) auto = true; + if (g != null && !g.isFinite) nonFinite = true; + if (g != null && g.isFinite) given = true; + if (out.minutes.length >= 2) multiMinute = true; + } + expect([empty, invalid, dup, unsorted, given, auto, nonFinite, multiMinute], + everyElement(isTrue)); + }); + + test('Lomb: abstentions, a spectrum, a non-finite value, a mismatch, ' + 'epoch-sized times, an empty and a repeated grid', () { + final ops = LombOps(); + var abstains = false, spectrum = false, nonFinite = false; + var mismatch = false, epoch = false, emptyGrid = false, repeated = false; + for (final r in ops.forced) { + final x = ops.expand(r); + final out = ops.sync(ops.fresh(x), x) as LombScargle?; + if (out == null) abstains = true; + if (out != null && out.spectrum.isNotEmpty) spectrum = true; + if (x.t.any((v) => !v.isFinite) || x.y.any((v) => !v.isFinite)) nonFinite = true; + if (x.t.length != x.y.length) mismatch = true; + if (x.t.isNotEmpty && x.t.first > 1e9) epoch = true; + if (x.freqs.isEmpty) emptyGrid = true; + if (x.freqs.toSet().length < x.freqs.length) repeated = true; + } + expect([abstains, spectrum, nonFinite, mismatch, epoch, emptyGrid, repeated], + everyElement(isTrue)); + }); + + test('minute metrics: no minutes, missing / inverted / NaN anchors, no ' + 'profile, a short, a DST and a long day, both artifact modes', () { + final ops = MinOps(); + var empty = false, missing = false, inverted = false, nan = false; + var noProfile = false, shortDay = false, dst = false, longDay = false; + var series = false, summary = false, energy = false, negative = false; + for (final r in ops.forced) { + final x = ops.expand(r); + final out = ops.sync(ops.fresh(x), x) as MinuteMetrics; + final c = _minCfgs[x.cfg]; + if (x.keys.isEmpty) empty = true; + if (c.$1 == null || c.$2 == null) missing = true; + if (c.$1 != null && c.$2 != null && c.$1! >= c.$2!) inverted = true; + if (c.$1 != null && c.$1!.isNaN) nan = true; + if (c.$4 == null) noProfile = true; + if (c.$5 < 1440) shortDay = true; + if (c.$5 == 1380) dst = true; + if (c.$5 > 1440) longDay = true; + if (x.series) series = true; + if (!x.series) summary = true; + if (out.energy != null) energy = true; + if (c.$1 != null && c.$1! < 0 && out.trimp.present) negative = true; + } + expect([empty, missing, inverted, nan, noProfile, shortDay, dst, longDay, series, summary, energy, negative], + everyElement(isTrue)); + }); + + test('merge: empty, one value, two, constant, large offset, wide range, ' + 'odd and even counts', () { + var empty = false, one = false, two = false, constant = false; + var offset = false, wide = false, odd = false, even = false; + for (final c in _momForced) { + if (c[1] == 0) empty = true; + if (c[1] == 1) one = true; + if (c[1] == 2) two = true; + if (c[0] == 2 && c[1] > 2) constant = true; + if (c[0] == 1) offset = true; + if (c[0] == 3) wide = true; + if (c[1] > 2 && c[1].isOdd) odd = true; + if (c[1] > 2 && c[1].isEven) even = true; + } + expect([empty, one, two, constant, offset, wide, odd, even], everyElement(isTrue)); + }); + }); + + _laws.registerReachTest(); +} diff --git a/test/registry_closure_test.dart b/test/registry_closure_test.dart new file mode 100644 index 00000000..857d4beb --- /dev/null +++ b/test/registry_closure_test.dart @@ -0,0 +1,414 @@ +// Design 06 phase 1b: the group registry is complete and honest. +// +// RED until `kAnalyticsGroupRegistry` declares the real groups. The registry +// is a hand-written claim ("this group's output depends on these files"), so +// the test re-derives the truth from the source and compares: +// +// (a) the registry validates (ids, deps, cycles, <=3 members, counter >= 1); +// (b) every member is a real public top-level name of the package; +// (c) the import closure of each group's member files stays inside the +// group's own `files` plus those of the foundations it depends on; +// (d) every lib/src file reachable from any exported function belongs to +// some group. +// +// HOW (b) KNOWS THE PUBLIC API. The package has no runtime or analyzer +// dependency (pure Dart, zero deps, by design), so the test does not link the +// analyzer. It follows `export` directives from lib/onehz.dart (honouring +// `show`/`hide`, which this package uses) and reads top-level declarations at +// column 0 after stripping comments. That reads the same files the compiler +// does, needs no hand-maintained map to rot, and a rename or a removed export +// fails here. Its one weakness is an unusual top-level layout the regex does +// not see; the sanity test below pins known names so a scanner regression is +// loud rather than silently permissive. +// +// HOW (c)/(d) WALK. Directives `import`, `export`, `part`, `part of` that name +// a lib/src file (relative, or package:openstrap_analytics/src/...) are edges. +// A `part` file is the same compilation unit, so it is an edge both ways. +import 'dart:io'; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +const _registryFile = 'lib/src/onehz/registry/group_registry.dart'; +const _pkg = 'package:openstrap_analytics/'; + +final _root = Directory.current.path; + +String _read(String rel) => File('$_root/$rel').readAsStringSync(); + +String _stripComments(String s) => s + .replaceAll(RegExp(r'/\*[\s\S]*?\*/'), '') + .replaceAll(RegExp(r'^\s*//.*$', multiLine: true), ''); + +String _norm(String from, String target) { + if (target.startsWith(_pkg)) return 'lib/${target.substring(_pkg.length)}'; + final base = from.substring(0, from.lastIndexOf('/')); + final parts = [...base.split('/')]; + for (final seg in target.split('/')) { + if (seg == '..') { + parts.removeLast(); + } else if (seg != '.' && seg.isNotEmpty) { + parts.add(seg); + } + } + return parts.join('/'); +} + +/// One whole directive statement; its URIs are pulled out separately because a +/// conditional import/export names several (`'a.dart' if (cond) 'b.dart'`). +final _directive = RegExp( + r'^(import|export|part of|part)\s+([^;]*);', + multiLine: true); +final _quoted = RegExp(r'''(['"])(.*?)\1'''); + +typedef _Directive = ({ + String kind, + String path, + List show, + List hide +}); + +/// lib/src files named by directive source [src] (the text of [file]), with +/// their kind and filters. Every URI of a conditional directive is an edge +/// (the closure must cover each platform's alternative), and both quote +/// styles are read. `show`/`hide` follow the last URI. +List<_Directive> _parseDirectives(String file, String src) { + final out = <_Directive>[]; + for (final m in _directive.allMatches(_stripComments(src))) { + final body = m.group(2)!; + final uris = _quoted.allMatches(body).toList(); + if (uris.isEmpty) continue; // `part of name;` + final rest = body.substring(uris.last.end); + List names(String kw) { + final mm = RegExp('$kw\\s+([\\w\\s,]+)').firstMatch(rest); + if (mm == null) return const []; + return mm.group(1)!.split(',').map((e) => e.trim()).where((e) => e.isNotEmpty).toList(); + } + + for (final u in uris) { + final target = u.group(2)!; + if (target.startsWith('dart:')) continue; + if (target.startsWith('package:') && !target.startsWith(_pkg)) continue; + final path = _norm(file, target); + if (!path.startsWith('lib/src/')) continue; + out.add((kind: m.group(1)!, path: path, show: names('show'), hide: names('hide'))); + } + } + return out; +} + +List<_Directive> _directives(String file) => + _parseDirectives(file, _read(file)); + +final _typeDecl = RegExp( + r'^(?:(?:abstract|final|base|sealed|interface|mixin)\s+)*(?:class|mixin|enum|typedef|extension type)\s+(\w+)', + multiLine: true); +final _fnDecl = RegExp( + r'^(?!(?:import|export|part|library|class|enum|typedef|const|final|var|late|abstract|mixin|extension|static|return|if|for|while|switch|else|sealed|base|interface)\b)[A-Za-z_][\w<>,?\[\]. ]*?\s(\w+)\s*(?:<[^>(]*>)?\(', + multiLine: true); + +/// Public top-level names declared in [file] itself, split by kind. +({Set fns, Set types}) _declared(String file) { + final src = _stripComments(_read(file)); + final types = {for (final m in _typeDecl.allMatches(src)) m.group(1)!}; + final fns = {for (final m in _fnDecl.allMatches(src)) m.group(1)!} + ..removeWhere((n) => n.startsWith('_') || types.contains(n)); + types.removeWhere((n) => n.startsWith('_')); + return (fns: fns, types: types); +} + +/// name -> declaring file, for everything lib/onehz.dart exports (public API). +/// [fnsOnly] keeps top-level functions only (constructors/types excluded). +Map _publicApi( + {bool fnsOnly = false, List libs = const ['lib/onehz.dart']}) { + final api = {}; + final seen = {}; + // Returns the names [file] exposes to an importer. + Map expose(String file) { + final out = {}; + final d = _declared(file); + for (final n in d.fns) { + out[n] = file; + } + if (!fnsOnly) { + for (final n in d.types) { + out[n] = file; + } + } + for (final e in _directives(file).where((e) => e.kind == 'export')) { + final sub = expose(e.path); + for (final en in sub.entries) { + if (e.show.isNotEmpty && !e.show.contains(en.key)) continue; + if (e.hide.contains(en.key)) continue; + out[en.key] = en.value; + } + } + return out; + } + + // The libraries are not under lib/src, so read their exports directly. + for (final lib in libs) { + for (final e in _directives(lib).where((e) => e.kind == 'export')) { + if (!seen.add(e.path)) continue; + for (final en in expose(e.path).entries) { + if (e.show.isNotEmpty && !e.show.contains(en.key)) continue; + if (e.hide.contains(en.key)) continue; + api[en.key] = en.value; + } + } + } + return api; +} + +/// Every lib/src file reachable from [start] over imports/exports/parts +/// (parts count both ways: a `part of` file belongs to its library). +Set _closure(Iterable start) { + final seen = {}; + final queue = [...start]; + while (queue.isNotEmpty) { + final f = queue.removeLast(); + if (!seen.add(f)) continue; + for (final d in _directives(f)) { + queue.add(d.path); + } + } + return seen; +} + +/// Edge also calls `prsaAggregator`, exported by the second public library +/// (lib/aggregate.dart), so members and api entries may name either. +const _publicLibs = ['lib/onehz.dart', 'lib/aggregate.dart']; + +/// Declaring file of member [name] (`fn`, `Type`, or `Type.method`), or null. +String? _memberFile(Map api, String name) { + final dot = name.indexOf('.'); + if (dot < 0) return api[name]; + final file = api[name.substring(0, dot)]; + if (file == null) return null; + final method = name.substring(dot + 1); + return _stripComments(_read(file)).contains(RegExp('\\b$method\\s*[(<]')) + ? file + : null; +} + +/// `name -> group ids` for every name owned by more than one slot: a name in +/// two groups, or in one group's members and api (or twice in either). +Map> _duplicateOwners(GroupRegistry r) { + final owners = >{}; + for (final g in r.groups) { + for (final m in [...g.members, ...g.api]) { + (owners[m] ??= []).add(g.id); + } + } + return owners..removeWhere((_, v) => v.length < 2); +} + +Set _transitiveDeps(GroupRegistry r, GroupDecl g) { + final byId = {for (final x in r.groups) x.id: x}; + final out = {}; + final q = [...g.deps]; + while (q.isNotEmpty) { + final d = q.removeLast(); + if (d == g.id || !out.add(d)) continue; + q.addAll(byId[d]?.deps ?? const []); + } + return out; +} + +const _expectedGroups = [ + 'sleep.stage', + 'hrv.freq', + 'night.rhythm', + 'day.blocks', + 'readiness.stress_illness', + 'activity.load', + 'body.spo2_temp_coverage', +]; + +void main() { + const reg = kAnalyticsGroupRegistry; + + group('scanner sanity (guards the guard)', () { + test('finds known public functions and types', () { + final api = _publicApi(); + expect(api['hrvFreq'], isNotNull); + expect(api['dispersionBelowQuantum'], isNotNull); + expect(api['RunningMoments'] ?? api['RrCorrector'], isNotNull); + expect(_publicApi(fnsOnly: true).length, greaterThan(100)); + }); + test('closure walk reaches transitive imports', () { + final f = _publicApi()['hrvFreq']!; + final c = _closure([f]); + expect(c, contains(f)); + expect(c.length, greaterThan(1)); + expect(_closure(_publicApi(fnsOnly: true).values.toSet()).length, + greaterThan(60)); + }); + test('hide filters are honoured', () { + expect(_publicApi().containsKey('readinessLnRmssdFromSuffix'), isFalse); + }); + }); + + group('directive parser (guards the guard)', () { + const f = 'lib/src/onehz/a.dart'; + List paths(String src) => + _parseDirectives(f, src).map((d) => d.path).toList(); + test('reads double-quoted directives', () { + expect(paths('import "b.dart";\nexport "c/d.dart" show X;'), + ['lib/src/onehz/b.dart', 'lib/src/onehz/c/d.dart']); + }); + test('reads every URI of a conditional import', () { + expect( + paths("import 'stub.dart'\n if (dart.library.io) 'io.dart'\n" + ' if (dart.library.js_interop) "web.dart" as impl;'), + ['lib/src/onehz/stub.dart', 'lib/src/onehz/io.dart', 'lib/src/onehz/web.dart']); + }); + test('conditional export keeps show/hide for every alternative', () { + final d = _parseDirectives( + f, "export 'a1.dart' if (dart.library.io) 'a2.dart' show Foo, Bar;"); + expect(d.map((e) => e.path), + ['lib/src/onehz/a1.dart', 'lib/src/onehz/a2.dart']); + for (final e in d) { + expect(e.kind, 'export'); + expect(e.show, ['Foo', 'Bar']); + } + }); + test('part, part of by URI, package URIs; skips dart: and foreign packages', () { + expect( + paths("part 'p.dart';\npart of \"q.dart\";\n" + "import 'package:openstrap_analytics/src/onehz/z.dart';\n" + "import 'dart:io';\nimport 'package:other/x.dart';\npart of lib;"), + ['lib/src/onehz/p.dart', 'lib/src/onehz/q.dart', 'lib/src/onehz/z.dart']); + }); + test('commented-out directives are ignored', () { + expect(paths("// import 'gone.dart';\n/* export 'also.dart'; */"), isEmpty); + }); + }); + + group('registry declares the analytics groups', () { + test('declares every expected algorithm group and >= 1 foundation', () { + final ids = reg.groups.map((g) => g.id).toSet(); + expect(ids, containsAll(_expectedGroups)); + expect(ids.where((i) => i.startsWith('found.')), isNotEmpty); + }); + + test('(a) validates: unique ids, deps resolve, acyclic, <=3 members, counter>=1', () { + final problems = []; + final byId = {}; + for (final g in reg.groups) { + if (byId.containsKey(g.id)) problems.add('${g.id}: duplicate id'); + byId[g.id] = g; + if (g.members.isEmpty) problems.add('${g.id}: no members'); + if (g.members.length > 3) problems.add('${g.id}: >3 members'); + if (g.counter < 1) problems.add('${g.id}: counter < 1'); + } + for (final g in reg.groups) { + for (final d in g.deps) { + if (!byId.containsKey(d)) problems.add('${g.id}: dep $d undeclared'); + } + } + final state = {}; + void visit(String id) { + state[id] = 1; + for (final d in byId[id]!.deps) { + if (!byId.containsKey(d)) continue; + if (state[d] == 1) problems.add('$id: cycle through $d'); + if (state[d] == null) visit(d); + } + state[id] = 2; + } + + for (final id in byId.keys) { + if (state[id] == null) visit(id); + } + expect(reg.groups, isNotEmpty, reason: 'registry is empty'); + expect(problems, isEmpty, reason: problems.join('\n')); + }); + + test('every listed file exists under lib/src', () { + final bad = []; + for (final g in reg.groups) { + for (final f in g.files) { + if (!f.startsWith('lib/src/') || !File('$_root/$f').existsSync()) { + bad.add('${g.id}: $f'); + } + } + } + expect(reg.groups, isNotEmpty, reason: 'registry is empty'); + expect(bad, isEmpty, reason: bad.join('\n')); + }); + }); + + test('a function is a member or api entry of exactly one group', () { + final dup = [ + for (final e in _duplicateOwners(reg).entries) + '${e.key}: ${e.value.join(', ')}', + ]; + expect(reg.groups, isNotEmpty, reason: 'registry is empty'); + expect(dup, isEmpty, reason: dup.join('\n')); + }); + + group('duplicate-owner check (guards the guard)', () { + GroupDecl g(String id, List m, [List api = const []]) => + GroupDecl(id: id, kind: GroupKind.algorithm, counter: 1, members: m, api: api); + test('flags one name in two groups', () { + final r = GroupRegistry(1, [g('a', ['f']), g('b', [], ['f'])]); + expect(_duplicateOwners(r).keys, ['f']); + }); + test('flags a name in one group\'s members and api', () { + final r = GroupRegistry(1, [g('a', ['f'], ['f'])]); + expect(_duplicateOwners(r).keys, ['f']); + }); + test('accepts distinct names', () { + final r = GroupRegistry(1, [g('a', ['f']), g('b', ['h'], ['i'])]); + expect(_duplicateOwners(r), isEmpty); + }); + }); + + test('(b) every member is a real public top-level name', () { + final api = _publicApi(libs: _publicLibs); + final bad = [ + for (final g in reg.groups) + for (final m in [...g.members, ...g.api]) + if (_memberFile(api, m) == null) '${g.id}: $m', + ]; + expect(reg.groups, isNotEmpty, reason: 'registry is empty'); + expect(bad, isEmpty, reason: 'not exported: ${bad.join(', ')}'); + }); + + test('(c) a group\'s import closure stays inside its files + foundations', () { + final api = _publicApi(libs: _publicLibs); + final byId = {for (final g in reg.groups) g.id: g}; + final leaks = []; + for (final g in reg.groups) { + final allowed = {...g.files}; + for (final d in _transitiveDeps(reg, g)) { + final dg = byId[d]; + if (dg != null && dg.kind == GroupKind.foundation) { + allowed.addAll(dg.files); + } + } + final start = [ + for (final m in [...g.members, ...g.api]) + if (_memberFile(api, m) != null) _memberFile(api, m)!, + ]; + final missing = _closure(start).difference(allowed) + ..remove(_registryFile); + for (final f in missing.toList()..sort()) { + leaks.add('${g.id} reaches unlisted $f'); + } + } + expect(reg.groups, isNotEmpty, reason: 'registry is empty'); + expect(leaks, isEmpty, reason: leaks.join('\n')); + }); + + test('(d) every file reachable from an exported function is in some group', () { + final api = _publicApi(fnsOnly: true); + final owned = {for (final g in reg.groups) ...g.files}; + final reach = _closure(api.values.toSet())..remove(_registryFile); + final orphans = reach.difference(owned).toList()..sort(); + expect(reg.groups, isNotEmpty, reason: 'registry is empty'); + expect(orphans, isEmpty, + reason: '${orphans.length} unowned: ${orphans.take(20).join(', ')}'); + }); +} diff --git a/tool/bench_aggregate.dart b/tool/bench_aggregate.dart new file mode 100644 index 00000000..d14e88d6 --- /dev/null +++ b/tool/bench_aggregate.dart @@ -0,0 +1,215 @@ +// Run with: dart compile exe tool/bench_aggregate.dart -o /tmp/bench_aggregate +// /tmp/bench_aggregate +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart'; + +Object? _sink; + +void main() { + final values = DoubleColumn(Float64List.fromList( + [for (var i = 0; i < 86400; i++) ((i * 7919) % 2003 - 1001).toDouble()])); + final summary = both( + both(Sum((batch) => batch.values), const Count()), + both(Min((batch) => batch.values), + Max((batch) => batch.values)), + ); + final beats = _beats(30000); + final origin = beats.timeMs.first; + final composed = hrvTimeAggregator(originMs: origin); + + final rows = <(String, String, int)>[ + ('86,400 doubles', 'hand sum/count/min/max', + _medianMicros(() => _doubleHandLoop(values))), + ('86,400 doubles', 'both blocks, one slice', + _medianMicros(() => summary.run(values))), + ('86,400 doubles', 'both blocks, 60-row slices', + _medianMicros(() => _sliced(summary, values, 60))), + ('30,000 beats', 'hrvTime() batch', _medianMicros( + () => hrvTime(beats.nnMs.toList(), nnTimesMs: beats.timeMs.toList()))), + ('30,000 beats', 'hand HRV loop', _medianMicros(() => _handHrv(beats))), + ('30,000 beats', 'composed, one slice', + _medianMicros(() => composed.run(beats))), + ('30,000 beats', 'composed, 24 slices + combine', + _medianMicros(() => _combined(composed, beats, 24))), + ('30,000 beats', 'composed, 1-row slices', + _medianMicros(() => _sliced(composed, beats, 1))), + ]; + + final reference = hrvTime(beats.nnMs.toList(), nnTimesMs: beats.timeMs.toList()) + .value!; + final result = composed.run(beats); + final parity = _hrvParity(result, reference); + final handUs = rows[4].$3; + final oneSliceUs = rows[5].$3; + final chunksUs = rows[6].$3; + + print('Median microseconds (7 timed runs after 3 warm-up runs)'); + print('${'workload'.padRight(16)} ${'operation'.padRight(36)} us'); + for (final row in rows) { + print('${row.$1.padRight(16)} ${row.$2.padRight(36)} ${row.$3}'); + } + print('parity composed vs hrvTime (1e-9): $parity'); + print('HRV ratios: one slice ${(oneSliceUs / handUs).toStringAsFixed(2)}x; ' + '24 slices ${(chunksUs / handUs).toStringAsFixed(2)}x (target <= 2.00x)'); + if (!parity || _sink == null) { + throw StateError('Benchmark parity or result observability failed'); + } +} + +(double, int, double, double) _doubleHandLoop(DoubleColumn batch) { + var sum = 0.0; + var count = 0; + var min = double.infinity; + var max = -double.infinity; + for (final value in batch.values) { + if (value.isNaN) continue; + sum += value; + count++; + if (value < min) min = value; + if (value > max) max = value; + } + return (sum, count, min, max); +} + +Beats _beats(int count) { + final random = math.Random(0xA661E6A7); + final nn = Float64List(count); + final times = Float64List(count); + var time = 1000000.0; + var interval = 900.0; + for (var i = 0; i < count; i++) { + interval = (interval + (random.nextDouble() - .5) * 24) + .clamp(800.0, 1100.0) + .toDouble(); + final gap = i > 0 && random.nextDouble() < .0007 + ? 1000 + random.nextDouble() * 30000 + : 0.0; + time += interval + gap; + nn[i] = interval; + times[i] = time; + } + return Beats(nn, times); +} + +R _combined(Aggregator aggregator, Beats batch, + int chunks) { + dynamic state = aggregator.empty(); + for (var chunk = 0; chunk < chunks; chunk++) { + final start = batch.length * chunk ~/ chunks; + final end = batch.length * (chunk + 1) ~/ chunks; + state = aggregator.combine(state, aggregator.fold(batch, start, end)); + } + return aggregator.finish(state); +} + +R _sliced, R>(Aggregator aggregator, B batch, + int sliceSize) { + var state = aggregator.empty(); + for (var start = 0; start < batch.length; start += sliceSize) { + final end = math.min(start + sliceSize, batch.length); + state = aggregator.addSlice(state, batch, start, end); + } + return aggregator.finish(state); +} + +int _medianMicros(Object? Function() action) { + for (var warm = 0; warm < 3; warm++) { + _sink = action(); + } + final samples = []; + for (var run = 0; run < 7; run++) { + final watch = Stopwatch()..start(); + _sink = action(); + watch.stop(); + samples.add(watch.elapsedMicroseconds); + } + samples.sort(); + return samples[samples.length ~/ 2]; +} + +bool _hrvParity(HrvTime a, HrvTime b) => + a.nBeats == b.nBeats && + _same(a.rmssd, b.rmssd) && + _same(a.sdnn, b.sdnn) && + _same(a.sdann, b.sdann) && + _same(a.sdnnIndex, b.sdnnIndex) && + _same(a.pnn50, b.pnn50) && + _same(a.diffAcf1, b.diffAcf1); + +bool _same(double? a, double? b) { + if (a == null || b == null) return a == b; + return (a - b).abs() <= 1e-9 + 1e-9 * b.abs(); +} + +class _Moments { + var count = 0; + var mean = 0.0; + var m2 = 0.0; + + void add(double value) { + count++; + final delta = value - mean; + mean += delta / count; + m2 += delta * (value - mean); + } + + double? get sampleSd => count < 2 ? null : math.sqrt(m2 / (count - 1)); +} + +/// Direct baseline for the same metrics, deliberately independent of the +/// aggregate graph. +HrvTime _handHrv(Beats beats) { + final levels = _Moments(); + final bins = {}; + var pairs = 0, over50 = 0, lagPairs = 0; + var sum = 0.0, squares = 0.0, products = 0.0, endpoints = 0.0; + for (var i = 0; i < beats.length; i++) { + final nn = beats.nnMs[i]; + levels.add(nn); + final key = ((beats.timeMs[i] - beats.timeMs.first) / 300000.0).floor(); + (bins[key] ??= _Moments()).add(nn); + if (i == 0 || beats.timeMs[i] - beats.timeMs[i - 1] > nn + 0.5) continue; + final diff = nn - beats.nnMs[i - 1]; + pairs++; + sum += diff; + squares += diff * diff; + if (diff.abs() > 50) over50++; + if (i >= 2 && beats.timeMs[i - 1] - beats.timeMs[i - 2] <= + beats.nnMs[i - 1] + 0.5) { + final previous = beats.nnMs[i - 1] - beats.nnMs[i - 2]; + lagPairs++; + products += previous * diff; + endpoints += previous + diff; + } + } + if (levels.count < 2) return HrvTime(nBeats: levels.count); + double? acf1; + if (pairs >= 30) { + final mean = sum / pairs; + final variance = squares - sum * mean; + if (variance > 0) { + acf1 = (products - mean * endpoints + lagPairs * mean * mean) / variance; + } + } + final jittery = acf1 != null && acf1 < kNnDiffAcf1Floor; + final means = _Moments(); + var sdSum = 0.0, binCount = 0; + for (final bin in bins.values) { + if (bin.count < 2) continue; + means.add(bin.mean); + sdSum += bin.sampleSd!; + binCount++; + } + return HrvTime( + nBeats: levels.count, + rmssd: pairs > 0 && !jittery ? math.sqrt(squares / pairs) : null, + pnn50: pairs > 0 && !jittery ? 100.0 * over50 / pairs : null, + sdnn: levels.sampleSd, + sdann: binCount >= 2 ? means.sampleSd : null, + sdnnIndex: binCount >= 2 ? sdSum / binCount : null, + diffAcf1: acf1, + ); +} diff --git a/tool/bench_night_bins.dart b/tool/bench_night_bins.dart new file mode 100644 index 00000000..1132f81f --- /dev/null +++ b/tool/bench_night_bins.dart @@ -0,0 +1,173 @@ +// Before/after for two night metrics edge re-runs on every pass: the 30-min +// RSA bins behind breathing-rate variability (edge `_respPerWindow`, copied +// here) and PRSA deceleration/acceleration capacity. 32 passes of 15 min; each +// pass folds the newly settled beats and asks with a ~100-beat provisional +// tail. "After" keeps closed bins' rates and folds PRSA with prsaAggregator. + +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart'; +import '../test/onehz/phase0_parity_test.dart' as frozen; + +const _windowMs = 1800000.0; + +/// Edge's `_respPerWindow`, verbatim apart from the name. +List batchBins(List nn, List nnTimes) { + if (nn.isEmpty || nn.length != nnTimes.length) return const []; + final t0 = nnTimes.first; + final binsNn = >{}; + final binsTs = >{}; + for (var i = 0; i < nn.length; i++) { + final idx = ((nnTimes[i] - t0) / _windowMs).floor(); + (binsNn[idx] ??= []).add(nn[i]); + (binsTs[idx] ??= []).add(nnTimes[i]); + } + final out = []; + final idxs = binsNn.keys.toList()..sort(); + for (final idx in idxs) { + final segNn = binsNn[idx]!; + if (segNn.length < 60) continue; + final r = rsaRespRate(segNn, binsTs[idx]!, artifactFraction: 0.0); + final b = r.present ? r.value!.brpm : null; + if (b != null) out.add(b); + } + return out; +} + +/// Closed bins (a settled beat exists in a later bin) are scored once. +class BinStream { + final closed = []; + final _openNn = [], _openTs = []; + double? _t0; + int _openIdx = 0; + + int _idx(double t) => ((t - _t0!) / _windowMs).floor(); + + void _close() { + if (_openNn.isNotEmpty) closed.addAll(batchBinsFrom(_openNn, _openTs)); + _openNn.clear(); + _openTs.clear(); + } + + // One bin's rate, through the batch on that bin alone. + List batchBinsFrom(List nn, List ts) { + if (nn.length < 60) return const []; + final r = rsaRespRate(nn, ts, artifactFraction: 0.0); + return r.present && r.value!.brpm != null ? [r.value!.brpm!] : const []; + } + + void fold(List nn, List ts) { + for (var i = 0; i < nn.length; i++) { + _t0 ??= ts[i]; + final k = _idx(ts[i]); + if (k != _openIdx) { + _close(); + _openIdx = k; + } + _openNn.add(nn[i]); + _openTs.add(ts[i]); + } + } + + List snapshot(List tailNn, List tailTs) { + if (_t0 == null) return batchBins(tailNn, tailTs); + final out = [...closed]; + var nn = [..._openNn], ts = [..._openTs]; + var k = _openIdx; + for (var i = 0; i < tailNn.length; i++) { + final j = _idx(tailTs[i]); + if (j != k) { + out.addAll(batchBinsFrom(nn, ts)); + nn = []; + ts = []; + k = j; + } + nn.add(tailNn[i]); + ts.add(tailTs[i]); + } + out.addAll(batchBinsFrom(nn, ts)); + return out; + } +} + +bool _sameList(List a, List b) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (a[i] != b[i]) return false; + } + return true; +} + +bool _samePrsa(Metric a, Metric b) { + if (a.present != b.present || a.note != b.note) return false; + if (!a.present) return true; + final x = a.value!, y = b.value!; + return x.capacity == y.capacity && + x.anchors == y.anchors && + _sameList(x.profile, y.profile) && + a.confidence == b.confidence; +} + +void main() { + final night = frozen.genBeats(9031, 33000, period: 55, amp: 60); + final bins = BinStream(); + final dcAgg = prsaAggregator(deceleration: true); + final acAgg = prsaAggregator(deceleration: false); + Object? dc = dcAgg.empty(), ac = acAgg.empty(); + var from = 0, parity = true; + var binsAfter = 0, binsBefore = 0, prsaAfter = 0, prsaBefore = 0; + final start = night.tMs.first; + print('pass bins after/before us prsa after/before us parity'); + for (var pass = 1; pass <= 32; pass++) { + final until = start + pass * 15 * 60 * 1000.0; + var end = from; + while (end < night.nn.length && night.tMs[end] <= until) { + end++; + } + if (end == from) continue; + final tailEnd = (end + 100).clamp(0, night.nn.length); + final tailNn = night.nn.sublist(end, tailEnd); + final tailTs = night.tMs.sublist(end, tailEnd); + final allNn = night.nn.sublist(0, tailEnd); + final allTs = night.tMs.sublist(0, tailEnd); + + var w = Stopwatch()..start(); + bins.fold(night.nn.sublist(from, end), night.tMs.sublist(from, end)); + final b1 = bins.snapshot(tailNn, tailTs); + final bA = w.elapsedMicroseconds; + w = Stopwatch()..start(); + final b0 = batchBins(allNn, allTs); + final bB = w.elapsedMicroseconds; + + w = Stopwatch()..start(); + final settled = DoubleColumn(Float64List.fromList(night.nn.sublist(from, end))); + dc = dcAgg.addSlice(dc, settled, 0, settled.length); + ac = acAgg.addSlice(ac, settled, 0, settled.length); + final tail = DoubleColumn(Float64List.fromList(tailNn)); + Metric snap( + Aggregator> agg, Object? s) { + final copy = agg.decodeState(agg.encodeState(s)); + return agg.finish(agg.addSlice(copy, tail, 0, tail.length)); + } + + final dc1 = snap(dcAgg, dc), ac1 = snap(acAgg, ac); + final pA = w.elapsedMicroseconds; + w = Stopwatch()..start(); + final dc0 = decelerationCapacity(allNn), ac0 = accelerationCapacity(allNn); + final pB = w.elapsedMicroseconds; + from = end; + + final same = _sameList(b1, b0) && _samePrsa(dc1, dc0) && _samePrsa(ac1, ac0); + parity = parity && same; + binsAfter += bA; + binsBefore += bB; + prsaAfter += pA; + prsaBefore += pB; + print('${pass.toString().padLeft(4)} ${'$bA/$bB'.padLeft(18)} ' + '${'$pA/$pB'.padLeft(18)} $same'); + } + print('total bins after $binsAfter us, before $binsBefore us; ' + 'prsa after $prsaAfter us, before $prsaBefore us; parity $parity'); + if (!parity) throw StateError('parity failed'); +} diff --git a/tool/bench_night_nn.dart b/tool/bench_night_nn.dart new file mode 100644 index 00000000..54009606 --- /dev/null +++ b/tool/bench_night_nn.dart @@ -0,0 +1,350 @@ +// Measurement tool: three ways to get the night's NN on every overnight derive +// pass. Not a test; no lib/ changes. AOT-friendly, seeded, no I/O except print. +// +// today : correctRr(sleep-window prefix) every pass (O(n) per pass) +// (a) : per-night RrCorrector, fold only the new beats, snapshot each pass +// (claimed exact: settled ++ snapshot().tail == correctRr(prefix)) +// (b) : correct the WHOLE day once, slice the result to the sleep window +// (differs near the window edges: correctRr clips windows at slice ends) +// +// Day layout: 8 h wake, 8 h sleep, 8 h wake (both window edges are interior to +// the day). 32 passes, one per 15 min of the night; the prefix grows each pass. +// +// dart compile exe tool/bench_night_nn.dart -o /tmp/bench_night_nn +// /tmp/bench_night_nn [seed ...] (default seeds 1 2 3) +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; + +const _startMs = 1.7e12; +const _hMs = 3600e3; +const _sleepStartMs = _startMs + 8 * _hMs; +const _sleepEndMs = _startMs + 16 * _hMs; +const _passMs = 15 * 60e3; +const _passes = 32; + +class Day { + final List rr = [], ts = []; +} + +Day synth(int seed) { + final rnd = math.Random(seed); + final d = Day(); + final wakeRate = 0.02 + 0.01 * rnd.nextDouble(); // 2..3 % of beats + final sleepRate = 0.005 + 0.01 * rnd.nextDouble(); // 0.5..1.5 % + var t = 0.0; // ms since start + final phase = rnd.nextDouble() * 6.28; + void emit(double v) { + v = v.roundToDouble(); + t += v; + d.rr.add(v); + d.ts.add(((_startMs + t) / 1000).floorToDouble() * 1000); + } + + // Multi-beat garbage runs (4..8 beats) and one 2-min dropout inside the sleep. + final runAt = [ + for (var i = 0; i < 3; i++) 8 * 3600e3 + (1 + 6 * rnd.nextDouble()) * _hMs + ]..sort(); + final dropAt = 8 * 3600e3 + (2 + 4 * rnd.nextDouble()) * _hMs; + var runIdx = 0; + var dropped = false; + const endT = 24 * 3600e3; + while (t < endT) { + final sleep = t >= 8 * 3600e3 && t < 16 * 3600e3; + final s = t / 1000; + final bpm = sleep + ? 57 + 7 * math.sin(s / 5400 + seed) + : 85 + 13 * math.sin(s / 1900 + seed); + final base = 60000 / bpm; + final v = base + + (sleep ? 28 : 14) * math.sin(2 * math.pi * (sleep ? .24 : .27) * s + phase) + + 18 * math.sin(2 * math.pi * 0.1 * s) + + (sleep ? 6 : 8) * (rnd.nextDouble() - .5) * 2; + if (sleep && !dropped && t >= dropAt) { + dropped = true; + t += 120e3; + continue; + } + if (sleep && runIdx < runAt.length && t >= runAt[runIdx]) { + runIdx++; + final len = 4 + rnd.nextInt(5); + for (var k = 0; k < len; k++) { + emit(250 + rnd.nextInt(2100).toDouble()); + } + continue; + } + if (rnd.nextDouble() < (sleep ? sleepRate : wakeRate)) { + final u = rnd.nextDouble(); + if (u < 0.4) { + emit(v * (0.55 + 0.15 * rnd.nextDouble())); // ectopic + pause + emit(v * (1.3 + 0.2 * rnd.nextDouble())); + } else if (u < 0.6) { + emit(v * (1.9 + 0.2 * rnd.nextDouble())); // missed beat + } else if (u < 0.85) { + final f = 0.35 + 0.3 * rnd.nextDouble(); // extra beat + emit(v * f); + emit(v * (1 - f)); + } else { + emit(250 + rnd.nextInt(2100).toDouble()); // lone garbage beat + } + continue; + } + emit(v); + } + return d; +} + +/// Replica of correctRr's private clock, so whole-day output can be mapped back +/// to input beat indices (the kept beats' times are tAll[i] bit for bit). +List beatTimes(List rr, List ts) { + final t = List.filled(rr.length, 0); + final origin = ts[0] - rr[0]; + var cur = 0.0; + for (var i = 0; i < rr.length; i++) { + final dropout = i > 0 && (ts[i] - ts[i - 1]) - rr[i] > 1000; + final anchored = ts[i] - origin; + cur = (dropout && anchored > cur) ? anchored : cur + rr[i]; + t[i] = cur; + } + return t; +} + +int lowerBound(List a, double x, [int lo = 0]) { + var hi = a.length; + while (lo < hi) { + final m = (lo + hi) >> 1; + if (a[m] < x) { + lo = m + 1; + } else { + hi = m; + } + } + return lo; +} + +/// beat index -> NN value for one correctRr result (kept beats only). +Map byBeat(RrCorrectionResult r, List tAll, int offset) { + final m = {}; + var j = 0; + for (var k = 0; k < r.nn.length; k++) { + j = lowerBound(tAll, r.nnTimesMs[k], j); + if (tAll[j] != r.nnTimesMs[k]) throw StateError('beat map broke'); + m[j + offset] = r.nn[k]; + } + return m; +} + +class Down { + final double? rmssd, sdnn, pnn50, dc, ac, lf, hf, total; + Down(this.rmssd, this.sdnn, this.pnn50, this.dc, this.ac, this.lf, this.hf, + this.total); + List get all => [rmssd, sdnn, pnn50, dc, ac, lf, hf, total]; + static const names = ['RMSSD', 'SDNN', 'pNN50', 'DC', 'AC', 'LF', 'HF', 'TotP']; +} + +Down downstream(List nn, List times, double clean) { + final art = 1 - clean; + final h = hrvTime(nn, nnTimesMs: times, artifactFraction: art).value; + final dc = decelerationCapacity(nn).value; + final ac = accelerationCapacity(nn).value; + final f = hrvFreq(nn, times, artifactFraction: art).value; + return Down(h?.rmssd, h?.sdnn, h?.pnn50, dc?.capacity, ac?.capacity, f?.lf, + f?.hf, f?.total); +} + +String ms(int us) => (us / 1000).toStringAsFixed(2); +String f(double? x, [int p = 3]) => x == null ? 'null' : x.toStringAsFixed(p); + +bool sameList(List a, List b) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (a[i] != b[i]) return false; + } + return true; +} + +void run(int seed) { + final day = synth(seed); + final n = day.rr.length; + final i0 = lowerBound(day.ts, _sleepStartMs); + final i1All = lowerBound(day.ts, _sleepEndMs); + print('\n==== seed $seed: day $n beats, sleep window beats [$i0,$i1All) ' + '= ${i1All - i0} ===='); + + // Whole day, as edge already does: fold (timed) then derive the day result. + final swW = Stopwatch()..start(); + final dayCorr = RrCorrector(); + dayCorr.fold(day.rr, tsMs: day.ts); + dayCorr.snapshot(); + swW.stop(); + final swWd = Stopwatch()..start(); + final whole = correctRr(day.rr, rrTsMs: day.ts); + swWd.stop(); + final tWhole = beatTimes(day.rr, day.ts); + final wholeMap = byBeat(whole, tWhole, 0); + // normal-beat prefix sums for (b)'s cleanFraction + final normPre = List.filled(n + 1, 0); + for (var i = 0; i < n; i++) { + normPre[i + 1] = normPre[i] + (whole.classes[i] == BeatClass.normal ? 1 : 0); + } + + // (a) state + final corr = RrCorrector(); + final aNn = [], aT = []; + var folded = i0; + + final tToday = [], tA = [], tB = []; + final exact = []; + final diffCnt = [], diffMaxMs = [], dClean = []; + final nearStart = [], nearEnd = []; // diffs within 60 s of an edge + final farDiff = []; // diffs > 300 s from both edges + final worstEdgeS = []; // max over diffs of min(dist start, dist end) + final rel = List.generate(8, (_) => []); + final abs = List.generate(8, (_) => []); + Down? finalToday, finalB; + var finalLine = ''; + + for (var p = 1; p <= _passes; p++) { + final cut = _sleepStartMs + p * _passMs; + final i1 = lowerBound(day.ts, cut); + final rr = day.rr.sublist(i0, i1), ts = day.ts.sublist(i0, i1); + + final sw = Stopwatch()..start(); + final today = correctRr(rr, rrTsMs: ts); + sw.stop(); + tToday.add(sw.elapsedMicroseconds); + + sw + ..reset() + ..start(); + final settled = corr.fold(day.rr.sublist(folded, i1), + tsMs: day.ts.sublist(folded, i1)); + folded = i1; + aNn.addAll(settled.nn); + aT.addAll(settled.nnTimes); + final snap = corr.snapshot(); + final aOutNn = [...aNn, ...snap.tailNn]; + final aOutT = [...aT, ...snap.tailNnTimes]; + final aClean = snap.cleanFraction; + sw.stop(); + tA.add(sw.elapsedMicroseconds); + exact.add(sameList(aOutNn, today.nn) && + sameList(aOutT, today.nnTimesMs) && + aClean == today.cleanFraction); + + sw + ..reset() + ..start(); + final bNn = [], bT = []; + final base = tWhole[i0]; + final lo = lowerBound(whole.nnTimesMs, tWhole[i0]); + final hi = lowerBound(whole.nnTimesMs, tWhole[i1 - 1], lo); + final hiIncl = (hi < whole.nn.length && whole.nnTimesMs[hi] == tWhole[i1 - 1]) + ? hi + 1 + : hi; + for (var k = lo; k < hiIncl; k++) { + bNn.add(whole.nn[k]); + bT.add(whole.nnTimesMs[k] - base + rr[0]); // rebase to the slice's frame + } + final bClean = (normPre[i1] - normPre[i0]) / (i1 - i0); + sw.stop(); + tB.add(sw.elapsedMicroseconds); + + // Diff by input beat index. + final tAllToday = beatTimes(rr, ts); + final tm = byBeat(today, tAllToday, i0); + var cnt = 0, near0 = 0, near1 = 0, far = 0; + var mx = 0.0, worst = 0.0; + final keys = {...tm.keys, ...wholeMap.keys.where((k) => k >= i0 && k < i1)}; + for (final k in keys) { + final x = tm[k], y = wholeMap[k]; + if (x != null && y != null && x == y) continue; + cnt++; + if (x != null && y != null) mx = math.max(mx, (x - y).abs()); + final ds = (day.ts[k] - _sleepStartMs) / 1000; + final de = (cut - day.ts[k]) / 1000; + final dm = math.min(ds, de); + if (ds <= 60) near0++; + if (de <= 60) near1++; + if (dm > 300) far++; + worst = math.max(worst, dm); + } + diffCnt.add(cnt); + diffMaxMs.add(mx); + dClean.add(bClean - today.cleanFraction); + nearStart.add(near0); + nearEnd.add(near1); + farDiff.add(far); + worstEdgeS.add(worst); + + final dt = downstream(today.nn, today.nnTimesMs, today.cleanFraction); + final db = downstream(bNn, bT, bClean); + for (var m = 0; m < 8; m++) { + final x = dt.all[m], y = db.all[m]; + if (x == null || y == null) continue; + abs[m].add((y - x).abs()); + rel[m].add(x == 0 ? 0 : (y - x).abs() / x.abs() * 100); + } + if (p == _passes) { + finalToday = dt; + finalB = db; + finalLine = 'final pass: today nn=${today.nn.length} clean=' + '${today.cleanFraction.toStringAsFixed(5)} | (b) nn=${bNn.length} ' + 'clean=${bClean.toStringAsFixed(5)}'; + } + } + + double sum(Iterable x) => x.fold(0.0, (a, b) => a + b); + double mean(List x) => x.isEmpty ? 0 : sum(x) / x.length; + double lastHalf(List x) => mean(x.sublist(x.length ~/ 2)); + + print('-- 1. time (ms), AOT, single run per pass --'); + print('pass: ${[1, 8, 16, 24, 32].map((p) => 'p$p').join(' ')}'); + String row(String name, List t) => + '${name.padRight(8)}${[1, 8, 16, 24, 32].map((p) => ms(t[p - 1]).padLeft(8)).join()}' + ' total ${ms(t.fold(0, (a, b) => a + b)).padLeft(9)}' + ' mean/pass ${ms((sum(t) / t.length).round())}' + ' mean last16 ${(lastHalf(t) / 1000).toStringAsFixed(2)}'; + print(row('today', tToday)); + print(row('(a)', tA)); + print(row('(b)', tB)); + print('whole-day RrCorrector fold+snapshot (paid by edge): ${ms(swW.elapsedMicroseconds)} ms;' + ' whole-day correctRr: ${ms(swWd.elapsedMicroseconds)} ms'); + + print('-- 2. exactness of (a) vs today: all $_passes passes bit-identical = ' + '${exact.every((e) => e)} (${exact.where((e) => e).length}/$_passes) --'); + + print('-- 3. (b) vs today --'); + print(finalLine); + print('differing NN beats (changed value or kept in one only):'); + print(' final pass : ${diffCnt.last} of ' + '${i1All - i0} beats; within 60 s of window start ${nearStart.last}, ' + 'of window end ${nearEnd.last}, >300 s from both ${farDiff.last}; ' + 'farthest from an edge ${worstEdgeS.last.toStringAsFixed(1)} s; ' + 'max |dNN| ${diffMaxMs.last.toStringAsFixed(1)} ms; ' + 'dClean ${dClean.last.toStringAsExponential(2)}'); + print(' mean/pass : ${mean(diffCnt).toStringAsFixed(1)} beats; ' + 'within 60 s of start ${mean(nearStart).toStringAsFixed(1)}, of end ' + '${mean(nearEnd).toStringAsFixed(1)}, >300 s from both ' + '${mean(farDiff).toStringAsFixed(1)}; farthest max over passes ' + '${worstEdgeS.reduce(math.max).toStringAsFixed(1)} s; ' + 'max |dNN| over passes ${diffMaxMs.reduce(math.max).toStringAsFixed(1)} ms; ' + 'mean |dClean| ${mean([for (final x in dClean) x.abs()]).toStringAsExponential(2)}'); + print('downstream: metric | today(final) | (b)(final) | abs d final | rel% final' + ' | mean abs d | mean rel%'); + for (var m = 0; m < 8; m++) { + final x = finalToday!.all[m], y = finalB!.all[m]; + final ad = (x == null || y == null) ? null : (y - x).abs(); + final rd = (ad == null || x == 0) ? null : ad / x!.abs() * 100; + print(' ${Down.names[m].padRight(6)}| ${f(x).padLeft(11)} | ${f(y).padLeft(11)}' + ' | ${f(ad, 4).padLeft(9)} | ${f(rd, 3).padLeft(8)} | ' + '${f(mean(abs[m]), 4).padLeft(9)} | ${f(mean(rel[m]), 3).padLeft(8)}'); + } +} + +void main(List args) { + final seeds = args.isEmpty ? [1, 2, 3] : args.map(int.parse).toList(); + for (final s in seeds) { + run(s); + } +} diff --git a/tool/bench_phase0.dart b/tool/bench_phase0.dart new file mode 100644 index 00000000..c1edee88 --- /dev/null +++ b/tool/bench_phase0.dart @@ -0,0 +1,296 @@ +// Round 15, phase 0 benchmark. The frozen before implementations live in the +// parity characterization test and are deliberately imported rather than +// copied again: that file is the versioned source of truth for this comparison. + +import 'package:openstrap_analytics/onehz.dart'; +import '../test/onehz/phase0_parity_test.dart' as frozen; + +int _medianUs(void Function() f, {int runs = 3}) { + if (runs == 1) { + final sw = Stopwatch()..start(); + f(); + sw.stop(); + return sw.elapsedMicroseconds; + } + final samples = []; + f(); // warm up JIT and lazy static initialization. + for (var i = 0; i < runs; i++) { + final sw = Stopwatch()..start(); + f(); + sw.stop(); + samples.add(sw.elapsedMicroseconds); + } + samples.sort(); + return samples[samples.length ~/ 2]; +} + +int _medianPreparedUs(void Function() prepare, void Function() f, + {int runs = 3}) { + if (runs == 1) { + prepare(); + final sw = Stopwatch()..start(); + f(); + sw.stop(); + return sw.elapsedMicroseconds; + } + prepare(); + f(); // JIT warmup; each measured pass gets the same prepared cache state. + final samples = []; + for (var i = 0; i < runs; i++) { + prepare(); + final sw = Stopwatch()..start(); + f(); + sw.stop(); + samples.add(sw.elapsedMicroseconds); + } + samples.sort(); + return samples[samples.length ~/ 2]; +} + +bool _same(double? a, double? b) => + a == null || b == null ? a == b : (a.isNaN && b.isNaN) || a == b; + +bool _sameList(List? a, List? b) { + if (a == null || b == null) return a == null && b == null; + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!_same(a[i], b[i])) return false; + } + return true; +} + +bool _sameCvhr(Metric a, Metric b) { + final x = a.value, y = b.value; + return _same(a.confidence, b.confidence) && + ((x == null || y == null) + ? x == null && y == null + : x.cycleCount == y.cycleCount && + _same(x.cvhrPerHour, y.cvhrPerHour) && + _same(x.analyzedHours, y.analyzedHours) && + _same(x.meanDepthMs, y.meanDepthMs) && + _same(x.meanWidthSec, y.meanWidthSec) && + _sameList(x.depthQuartilesMs, y.depthQuartilesMs) && + _sameList(x.widthQuartilesSec, y.widthQuartilesSec)); +} + +void _cvhrStreamBench() { + // 32 × 15 min, about eight hours at 56 bpm. Each pass makes the newly + // settled prefix permanent and compares its snapshot with the batch scorer. + final night = frozen.genBeats(9019, 33000, period: 55, amp: 60); + final stream = CvhrStream(); + var from = 0, streamTotal = 0, batchTotal = 0; + var parity = true; + final start = night.tMs.first; + print('CVHR stream: pass stream us batch us parity'); + for (var pass = 1; pass <= 32; pass++) { + final until = start + pass * 15 * 60 * 1000.0; + var end = from; + while (end < night.nn.length && night.tMs[end] <= until) { + end++; + } + if (end == from) continue; + stream.fold(night.nn.sublist(from, end), night.tMs.sublist(from, end)); + from = end; + final streamWatch = Stopwatch()..start(); + final streamed = stream.snapshot(artifactFraction: 0.02); + streamWatch.stop(); + final batchWatch = Stopwatch()..start(); + final batch = cvhrApneaScreen(night.nn.sublist(0, end), + night.tMs.sublist(0, end), artifactFraction: 0.02); + batchWatch.stop(); + final streamUs = streamWatch.elapsedMicroseconds; + final batchUs = batchWatch.elapsedMicroseconds; + streamTotal += streamUs; + batchTotal += batchUs; + final same = _sameCvhr(streamed, batch); + parity = parity && same; + print('cvhr pass ${pass.toString().padLeft(2)} ' + '${streamUs.toString().padLeft(9)} ${batchUs.toString().padLeft(9)} ' + 'parity=$same'); + } + print('cvhr total ${streamTotal.toString().padLeft(9)} ' + '${batchTotal.toString().padLeft(9)}'); + print('cvhr stream parity: $parity'); + if (!parity) throw StateError('CVHR stream parity failed'); +} + +bool _sameHrv(Metric a, Metric b) { + final x = a.value, y = b.value; + return _same(a.confidence, b.confidence) && + ((x == null || y == null) + ? x == null && y == null + : _same(x.ulf, y.ulf) && + _same(x.vlf, y.vlf) && + _same(x.lf, y.lf) && + _same(x.hf, y.hf) && + _same(x.total, y.total) && + _same(x.lfhf, y.lfhf) && + _same(x.nuLf, y.nuLf) && + _same(x.nuHf, y.nuHf) && + x.hfGated == y.hfGated && + x.totalBands.toString() == y.totalBands.toString()); +} + +bool _sameStager(CardioStagerResult a, CardioStagerResult b) => + a.base.stages.toString() == b.base.stages.toString() && + a.base.epochSec == b.base.epochSec && + _same(a.base.wakePct, b.base.wakePct) && + _same(a.base.nremPct, b.base.nremPct) && + _same(a.base.remPct, b.base.remPct) && + a.deepFlag.toString() == b.deepFlag.toString() && + _same(a.confidence, b.confidence); + +bool _sameRsa(Metric a, Metric b) { + final x = a.value, y = b.value; + return _same(a.confidence, b.confidence) && + a.tier == b.tier && + a.inputs_used.toString() == b.inputs_used.toString() && + a.note == b.note && + ((x == null || y == null) + ? x == null && y == null + : _same(x.brpm, y.brpm) && + _same(x.peakHz, y.peakHz) && + _same(x.power, y.power) && + x.source == y.source); +} + +void _welchStreamBench() { + // 32 × 15 min, about eight hours at 56 bpm. The tail is provisional: it is + // included in each snapshot and batch call, then folded in a later pass. + final night = frozen.genBeats(9018, 33000, period: 53, amp: 50); + final hrv = HrvFreqStream(); + final rsa = RsaRespStream(); + var from = 0; + var hrvStreamTotal = 0, hrvBatchTotal = 0; + var rsaStreamTotal = 0, rsaBatchTotal = 0; + var parity = true; + final start = night.tMs.first; + print('Welch streams: pass stream hrv/rsa us batch hrv/rsa us parity'); + for (var pass = 1; pass <= 32; pass++) { + final until = start + pass * 15 * 60 * 1000.0; + var end = from; + while (end < night.nn.length && night.tMs[end] <= until) { + end++; + } + if (end == from) continue; + hrv.fold(night.nn.sublist(from, end), night.tMs.sublist(from, end)); + rsa.fold(night.nn.sublist(from, end), night.tMs.sublist(from, end)); + from = end; + final tailEnd = (end + 100).clamp(0, night.nn.length); + final tailNn = night.nn.sublist(end, tailEnd); + final tailTimes = night.tMs.sublist(end, tailEnd); + final hrvStreamWatch = Stopwatch()..start(); + final streamHrv = hrv.snapshot( + tailNnMs: tailNn, tailNnTimesMs: tailTimes, artifactFraction: 0.02); + hrvStreamWatch.stop(); + final rsaStreamWatch = Stopwatch()..start(); + final streamRsa = rsa.snapshot( + tailNnMs: tailNn, tailNnTimesMs: tailTimes, artifactFraction: 0.02); + rsaStreamWatch.stop(); + final hrvBatchWatch = Stopwatch()..start(); + final batchHrv = hrvFreq(night.nn.sublist(0, tailEnd), + night.tMs.sublist(0, tailEnd), artifactFraction: 0.02); + hrvBatchWatch.stop(); + final rsaBatchWatch = Stopwatch()..start(); + final batchRsa = rsaRespRate(night.nn.sublist(0, tailEnd), + night.tMs.sublist(0, tailEnd), artifactFraction: 0.02); + rsaBatchWatch.stop(); + final hrvStreamUs = hrvStreamWatch.elapsedMicroseconds; + final rsaStreamUs = rsaStreamWatch.elapsedMicroseconds; + final hrvBatchUs = hrvBatchWatch.elapsedMicroseconds; + final rsaBatchUs = rsaBatchWatch.elapsedMicroseconds; + hrvStreamTotal += hrvStreamUs; + rsaStreamTotal += rsaStreamUs; + hrvBatchTotal += hrvBatchUs; + rsaBatchTotal += rsaBatchUs; + final same = _sameHrv(streamHrv, batchHrv) && _sameRsa(streamRsa, batchRsa); + parity = parity && same; + print('welch pass ${pass.toString().padLeft(2)} ' + '${hrvStreamUs.toString().padLeft(8)}/${rsaStreamUs.toString().padLeft(8)} ' + '${hrvBatchUs.toString().padLeft(8)}/${rsaBatchUs.toString().padLeft(8)} ' + 'parity=$same'); + } + print('welch total ' + '${hrvStreamTotal.toString().padLeft(8)}/${rsaStreamTotal.toString().padLeft(8)} ' + '${hrvBatchTotal.toString().padLeft(8)}/${rsaBatchTotal.toString().padLeft(8)}'); + print('welch stream parity: $parity'); + if (!parity) throw StateError('Welch stream parity failed'); +} + +void main(List args) { + // 8 h / 30k beats / one 8 h night, respectively. Fixed seeds make both the + // timings and the parity gate reproducible. + final cvhr = frozen.genBeats(9001, 32000, period: 55, amp: 60); + final hrv = frozen.genBeats(9002, 30000, period: 53, amp: 50); + final night = frozen.genNight(9003, 8 * 60 * 60, rrOutliers: true); + + final cvhrBefore = () => frozen.refCvhrApneaScreen(cvhr.nn, cvhr.tMs, + artifactFraction: 0.02); + final cvhrAfter = () => + cvhrApneaScreen(cvhr.nn, cvhr.tMs, artifactFraction: 0.02); + final hrvBefore = () => + frozen.refHrvFreq(hrv.nn, hrv.tMs, artifactFraction: 0.02); + final hrvAfter = () => hrvFreq(hrv.nn, hrv.tMs, artifactFraction: 0.02); + final stagerBefore = () => frozen.refCardioStager(night.hr, night.accel, + rrMs: night.rr, rrTsMs: night.rrTs); + final stagerAfter = () => cardioStager(night.hr, night.accel, + rrMs: night.rr, rrTsMs: night.rrTs); + + final selected = args.isEmpty ? {'cvhr', 'hrv', 'cardio'} : args.toSet(); + print('case before us after us'); + var parity = true; + void row(String key, String name, void Function() before, void Function() after, + bool same) { + if (!selected.contains(key)) return; + parity = parity && same; + final beforeUs = _medianUs(before); + final afterUs = _medianUs(after); + print('${name.padRight(28)} ${beforeUs.toString().padLeft(9)} ' + '${afterUs.toString().padLeft(9)} parity=$same'); + } + row('cvhr', 'cvhrApneaScreen (8 h)', cvhrBefore, cvhrAfter, + _sameCvhr(cvhrBefore(), cvhrAfter())); + row('hrv', 'hrvFreq (30k beats)', hrvBefore, hrvAfter, + _sameHrv(hrvBefore(), hrvAfter())); + row('cardio', 'cardioStager (one night)', stagerBefore, stagerAfter, + _sameStager(stagerBefore(), stagerAfter())); + if (args.isEmpty || selected.contains('cvhr-stream')) _cvhrStreamBench(); + if (selected.contains('cardio') || selected.contains('cache')) { + final extra = frozen.genNight(9004, 15 * 60, rrOutliers: true); + const nightMs = 8 * 60 * 60 * 1000.0; + final appendedAccel = [ + ...night.accel, + for (final a in extra.accel) + AccelSample(a.tsMs + nightMs, a.x, a.y, a.z, valid: a.valid), + ]; + final appendedHr = [...night.hr, ...extra.hr]; + final appendedRr = [...night.rr, ...extra.rr]; + final appendedRrTs = [ + ...night.rrTs, + for (final ts in extra.rrTs) ts + nightMs, + ]; + CardioRowCache? cache; + void warmNight() { + cache = CardioRowCache(); + cardioStager(night.hr, night.accel, + rrMs: night.rr, rrTsMs: night.rrTs, rowCache: cache); + } + final coldUs = _medianUs(() => cardioStager(night.hr, night.accel, + rrMs: night.rr, rrTsMs: night.rrTs, rowCache: CardioRowCache()), + runs: 1); + final warmUs = _medianPreparedUs(warmNight, () => cardioStager( + night.hr, night.accel, + rrMs: night.rr, rrTsMs: night.rrTs, rowCache: cache), runs: 1); + final appendedUs = _medianPreparedUs(warmNight, () => cardioStager( + appendedHr, appendedAccel, + rrMs: appendedRr, rrTsMs: appendedRrTs, rowCache: cache), runs: 1); + print('cardio cache (8 h) cold us $coldUs'); + print('cardio cache (8 h) warm us $warmUs'); + print('cardio cache warm +15 min us $appendedUs'); + } + if (args.isEmpty || selected.contains('welch')) _welchStreamBench(); + print('parity: frozen references == production: $parity'); + if (!parity) { + throw StateError('frozen-reference parity failed'); + } +} diff --git a/tool/combined_bump_probe.dart b/tool/combined_bump_probe.dart new file mode 100644 index 00000000..6a526e8f --- /dev/null +++ b/tool/combined_bump_probe.dart @@ -0,0 +1,189 @@ +import 'dart:io'; +import 'dart:convert'; +import 'package:openstrap_analytics/onehz.dart'; + +void main(List args) { + const base = 1700000013; + final accel = [], hr = []; + final rr = [], ts = []; + for (final l in File('test/onehz/fixtures/real_night_2026_07_onehz.csv') + .readAsLinesSync() + .skip(1)) { + final p = l.split(','); + if (p.length < 5) continue; + final t = (base + int.parse(p[0])) * 1000.0; + hr.add(HrSample(t, double.parse(p[1]))); + accel.add(AccelSample( + t, double.parse(p[2]), double.parse(p[3]), double.parse(p[4]))); + } + for (final l in File('test/onehz/fixtures/real_night_2026_07_rr.csv') + .readAsLinesSync() + .skip(1)) { + final p = l.split(','); + if (p.length < 2) continue; + rr.add(double.parse(p[1])); + ts.add(base * 1000.0 + double.parse(p[0])); + } + if (args.contains('rr-bench')) { + for (var i = 0; i < 8; i++) { + correctRr(rr, rrTsMs: ts); + correctRr(rr, rrTsMs: ts, maxArtifactRun: null); + } + print('correctRr capped: ${medianUs(() => correctRr(rr, rrTsMs: ts))} us'); + print( + 'correctRr uncapped: ${medianUs(() => correctRr(rr, rrTsMs: ts, maxArtifactRun: null))} us'); + return; + } + if (args.contains('bench')) { + benchmark(hr, accel, rr, ts); + return; + } + final old = <(int, int, String)>[]; + for (final l + in File('test/onehz/fixtures/real_night_2026_07_hypnogram_v1.csv') + .readAsLinesSync() + .skip(1)) { + final p = l.split(','); + if (p.length < 3) continue; + old.add((base + int.parse(p[0]), base + int.parse(p[1]), p[2])); + } + void compare(String name, List segments) { + var same = 0, count = 0; + final confusion = {}; + for (var t = (base + 29) ~/ 30 * 30 + 15; + t + 15 <= base + hr.length; + t += 30) { + final a = old.where((s) => t >= s.$1 && t < s.$2).first.$3; + final b = segments.where((s) => t >= s.start && t < s.end).first.stage; + if (a == 'unobserved' || b == 'unobserved') continue; + count++; + if (a == b) same++; + if (a != b) confusion.update('$a->$b', (v) => v + 1, ifAbsent: () => 1); + } + print('$name agreement=$same/$count=${same / count} $confusion'); + } + + final er = EpochRows(); + final sw = Stopwatch()..start(); + er.fold(hr: hr, accel: accel, rrMs: rr, rrTsMs: ts); + final rows = er.allRows(); + sw.stop(); + print('EpochRows full ${sw.elapsedMicroseconds} us, ${rows.length} rows'); + compare('EpochRows', er.stageWindow(base, base + hr.length)); + for (final lowHz in [1 / 180, 0.04]) { + final lfhf = []; + for (final row in rows) { + final b = cleanRrBeatsBetween( + rr, ts, row.startMs - 75000.0, row.startMs + 105000.0); + final ls = b.beats.length < 16 + ? null + : lombScargle(b.tsSec, b.beats, freqGrid(lowHz, 0.4, 240)); + final hf = ls?.bandPower(0.15, 0.4) ?? 0; + lfhf.add(hf > 0 ? ls!.bandPower(0.04, 0.15) / hf : double.nan); + } + final result = classifyCardioEpochs(CardioEpochFeatures( + motion: [for (final r in rows) r.motion], + hr: [for (final r in rows) r.hr], + hrSd: [for (final r in rows) r.hrSd], + rmssd: [for (final r in rows) r.rmssd], + lfhf: lfhf, + rk: [for (final r in rows) r.rk], + sdnn: [for (final r in rows) r.sdnn])); + compare('fixed grid $lowHz', [ + for (var i = 0; i < result.base.stages.length; i++) + StageSegment( + i == 0 ? base : rows[i].startMs ~/ 1000, + i == result.base.stages.length - 1 + ? base + hr.length + : rows[i + 1].startMs ~/ 1000, + result.base.stages[i] == SleepStage.wake + ? 'wake' + : result.base.stages[i] == SleepStage.rem + ? 'rem' + : result.deepFlag[i] + ? 'deep' + : 'light') + ]); + } + for (final skip in [0, 27]) { + final s = Stopwatch()..start(); + final result = cardioStager( + [for (final h in hr.skip(skip)) h.hr], accel.sublist(skip), + rrMs: rr, rrTsMs: ts); + s.stop(); + print('cardioStager skip=$skip ${s.elapsedMicroseconds} us'); + compare('direct skip=$skip', [ + for (var i = 0; i < result.base.stages.length; i++) + StageSegment( + base + skip + i * 30, + i == result.base.stages.length - 1 + ? base + hr.length + : base + skip + (i + 1) * 30, + result.base.stages[i] == SleepStage.wake + ? 'wake' + : result.base.stages[i] == SleepStage.rem + ? 'rem' + : result.deepFlag[i] + ? 'deep' + : 'light') + ]); + } +} + +int medianUs(void Function() run) { + run(); + final samples = []; + for (var i = 0; i < 3; i++) { + final sw = Stopwatch()..start(); + run(); + samples.add(sw.elapsedMicroseconds); + } + samples.sort(); + return samples[1]; +} + +void benchmark(List hr, List accel, List rr, + List ts) { + final values = [for (final h in hr) h.hr]; + print('Real night: ${hr.length} seconds, ${rr.length} beats'); + print('EpochRows fold + allRows: ${medianUs(() { + final er = EpochRows(); + er.fold(hr: hr, accel: accel, rrMs: rr, rrTsMs: ts); + er.allRows(); + })} us'); + print( + 'cardioEpochFeatures: ${medianUs(() => cardioEpochFeatures(values, accel, rrMs: rr, rrTsMs: ts))} us'); + print('correctRr capped: ${medianUs(() => correctRr(rr, rrTsMs: ts))} us'); + print( + 'correctRr uncapped: ${medianUs(() => correctRr(rr, rrTsMs: ts, maxArtifactRun: null))} us'); + final er = EpochRows(); + var rb = 0, streamed = 0, batch = 0, checkpointChars = 0; + final t0 = hr.first.tsMs; + for (var begin = 0; begin < hr.length; begin += 900) { + final end = (begin + 900).clamp(0, hr.length); + final rt = t0 + end * 1000; + var re = rb; + while (re < ts.length && ts[re] < rt) { + re++; + } + final watch = Stopwatch()..start(); + er.fold( + hr: hr.sublist(begin, end), + accel: accel.sublist(begin, end), + rrMs: rr.sublist(rb, re), + rrTsMs: ts.sublist(rb, re)); + er.allRows(); + streamed += watch.elapsedMicroseconds; + checkpointChars = jsonEncode(er.toJson()).length; + watch.reset(); + cardioEpochFeatures(values.sublist(0, end), accel.sublist(0, end), + rrMs: rr.sublist(0, re), rrTsMs: ts.sublist(0, re)); + batch += watch.elapsedMicroseconds; + rb = re; + } + print( + '15-minute passes total: EpochRows $streamed us, batch features $batch us'); + print('EpochRows final checkpoint: $checkpointChars chars'); + print( + 'RrSandwich benchmark unavailable: generic clock translation is unresolved'); +} diff --git a/tool/incremental/bench_all.dart b/tool/incremental/bench_all.dart new file mode 100644 index 00000000..9c26413a --- /dev/null +++ b/tool/incremental/bench_all.dart @@ -0,0 +1,283 @@ +// Timing: whole-series oracle vs incremental (fold + evaluate) per 15-min pass. +// dart run tool/incremental/bench_all.dart +import 'dart:convert'; +import 'package:openstrap_analytics/onehz.dart' + hide RrCorrector, RrSettled, RrSnapshot, IrregularScreenState; +import 'driver.dart'; +import 'edge_oracles.dart'; +import 'hrv_incr.dart'; +import 'oracle_util.dart'; +import 'resp_incr.dart'; +import 'synth.dart'; + +double ms(Stopwatch s) => s.elapsedMicroseconds / 1000.0; +double sum(List x) => x.fold(0.0, (a, b) => a + b); +double mean(List x) => x.isEmpty ? 0 : sum(x) / x.length; +double mx(List x) => x.fold(0.0, (a, b) => a > b ? a : b); + +class Row { + final String name; + final double oracleFullMs; + final List passMs; // fold+evaluate per pass + final int stateBytes; + Row(this.name, this.oracleFullMs, this.passMs, this.stateBytes); + String line() => + '| $name | ${oracleFullMs.toStringAsFixed(0)} | ${mean(passMs).toStringAsFixed(1)} | ${mx(passMs).toStringAsFixed(1)} | ${sum(passMs).toStringAsFixed(0)} | $stateBytes |'; +} + +T timed(Stopwatch sw, T Function() f, List sink) { + sw..reset()..start(); + final r = f(); + sink.add(ms(sw)); + return r; +} + +void main() { + final night = realNightRr()!; + final cuts = timeCuts(night.ts, 900); + final steps = []; + driveRr(night, cuts, steps.add); + final o = correctRr(night.rr, rrTsMs: night.ts); + final af = (1.0 - o.cleanFraction).clamp(0.0, 1.0).toDouble(); + print('night: ${night.length} beats, ${steps.length} folds, ' + 'NN=${o.nn.length}, artifactFraction=${af.toStringAsFixed(4)}'); + final sw = Stopwatch(); + final rows = []; + + Row bench(String name, double Function() oracle, void Function(FoldStep, List) pass, int Function() bytes) { + oracle(); // warm + sw..reset()..start(); + oracle(); + final fullMs = ms(sw); + final ps = []; + for (final f in steps) { + pass(f, ps); + } + final r = Row(name, fullMs, ps, bytes()); + rows.add(r); + return r; + } + + // hrvTime + { + var acc = HrvTimeAcc(); + bench('hrvTime scalars (night)', () { + sw..reset()..start(); + hrvTime(o.nn, nnTimesMs: o.nnTimesMs, artifactFraction: af); + return ms(sw); + }, (f, ps) { + timed(sw, () { + acc.fold(f.settled.nn, f.settled.nnTimes); + acc.evaluate(f.snap.tailNn, f.snap.tailNnTimes, artifactFraction: f.artifactFraction); + }, ps); + }, () => jsonEncode(acc.toJson()).length); + } + // nocturnal rmssd + { + final st = NocturnalRmssdState(); + bench('nocturnalRmssd', () { + sw..reset()..start(); + nocturnalRmssd(o.nn, o.nnTimesMs); + return ms(sw); + }, (f, ps) { + timed(sw, () { + st.fold(f.settled.nn, f.settled.nnTimes); + st.evaluate(f.snap.tailNn, f.snap.tailNnTimes); + }, ps); + }, () => st.recs.length * 60); + } + // session rmssd (raw) + { + final start = (night.ts.first / 1000).floor() + 1234; + final end = (night.ts.last / 1000).floor() - 987; + final st = SessionRmssdState(start, end); + var from = 0; + var k = 0; + bench('sleepSessionWindowedRmssd (raw RR)', () { + sw..reset()..start(); + sleepSessionWindowedRmssd(night.rr, night.ts, startSec: start, endSec: end); + return ms(sw); + }, (f, ps) { + timed(sw, () { + st.fold(night.rr.sublist(from, cuts[k]), night.ts.sublist(from, cuts[k])); + from = cuts[k]; + k++; + st.evaluate(); + }, ps); + }, () => st.recs.length * 60); + } + // night shape + { + final st = NightShapeState(); + bench('nightHrvShape', () { + sw..reset()..start(); + nightHrvShape(o.nn, o.nnTimesMs); + return ms(sw); + }, (f, ps) { + timed(sw, () { + st.fold(f.settled.nn, f.settled.nnTimes); + st.evaluate(f.snap.tailNn, f.snap.tailNnTimes); + }, ps); + }, () => st.closed.length * 40); + } + // hrv timeline + { + final origin = night.ts.first - night.rr.first; + final st = HrvTimelineState(origin); + bench('_hrvTimeline (night)', () { + sw..reset()..start(); + oracleHrvTimeline(o.nn, o.nnTimesMs, origin); + return ms(sw); + }, (f, ps) { + timed(sw, () { + st.fold(f.settled.nn, f.settled.nnTimes); + st.curve(f.snap.tailNn, f.snap.tailNnTimes); + }, ps); + }, () => st.out.length * 20 + 900 * 16); + } + // rsa + { + var st = RsaWelchState(); + bench('rsaRespRate (night)', () { + sw..reset()..start(); + rsaRespRate(o.nn, o.nnTimesMs, artifactFraction: af); + return ms(sw); + }, (f, ps) { + timed(sw, () { + st.fold(f.settled.nn, f.settled.nnTimes); + st.evaluate(f.snap.tailNn, f.snap.tailNnTimes, artifactFraction: f.artifactFraction); + }, ps); + }, () => jsonEncode(st.toJson()).length); + } + // resp windows + { + final st = RespWindowsState(); + bench('_respPerWindow (30-min bins)', () { + sw..reset()..start(); + oracleRespPerWindow(o.nn, o.nnTimesMs); + return ms(sw); + }, (f, ps) { + timed(sw, () { + st.fold(f.settled.nn, f.settled.nnTimes); + st.evaluate(f.snap.tailNn, f.snap.tailNnTimes); + }, ps); + }, () => st.closed.length * 8 + 12000); + } + // hrvFreq + { + var st = HrvFreqState(); + bench('hrvFreq LF/HF (night)', () { + sw..reset()..start(); + hrvFreq(o.nn, o.nnTimesMs, artifactFraction: af); + return ms(sw); + }, (f, ps) { + timed(sw, () { + st.fold(f.settled.nn, f.settled.nnTimes); + st.evaluate(f.snap.tailNn, f.snap.tailNnTimes, artifactFraction: f.artifactFraction); + }, ps); + }, () => jsonEncode(st.toJson()).length); + } + print('\nNIGHT estimators (8.9 h, 32k beats, 15-min passes)'); + print('| estimator | oracle whole-night ms | incr mean ms/pass | max | sum over passes | state bytes (json) |'); + print('|---|---|---|---|---|---|'); + rows.forEach((r) => print(r.line())); + + // ---------------- day-long ---------------- + final day = realShapedDay(); + final dcuts = timeCuts(day.ts, 900); + final dsteps = []; + driveRr(day, dcuts, dsteps.add); + final from = (day.ts.first / 1000).floor() - 2, to = (day.ts.last / 1000).ceil() + 3; + final acc = synthAccel(5, from, to); + final dsub = DaySub(day.rr, day.ts, acc.tsSec, acc.ax, acc.ay, acc.az); + final drows = []; + print('\nday: ${day.length} beats, ${dsteps.length} folds'); + // irregular + { + final o2 = correctRr(day.rr, rrTsMs: day.ts); + final af2 = (1.0 - o2.cleanFraction).clamp(0.0, 1.0).toDouble(); + irregularBeatScreen(o2.nn, nnTimesMs: o2.nnTimesMs, artifactFraction: af2); + sw..reset()..start(); + irregularBeatScreen(o2.nn, nnTimesMs: o2.nnTimesMs, artifactFraction: af2); + final full = ms(sw); + var st = IrregularScreenState(); + final ps = []; + for (final f in dsteps) { + timed(sw, () { + st.fold(f.settled.nn, f.settled.nnTimes); + st.evaluate(f.snap.tailNn, f.snap.tailNnTimes, artifactFraction: f.artifactFraction); + }, ps); + } + drows.add(Row('irregularBeatScreen (day NN, excl. correctRr)', full, ps, jsonEncode(st.toJson()).length)); + } + // day hrv curve + { + oracleDayHrvCurve(dsub); + sw..reset()..start(); + oracleDayHrvCurve(dsub); + final full = ms(sw); + final st = DayHrvCurveState(); + final ps = []; + var a = 0; + for (final cut in dcuts) { + timed(sw, () { + st.fold(day.rr.sublist(a, cut), day.ts.sublist(a, cut)); + }, ps); + a = cut; + } + drows.add(Row('dayHrvCurve', full, ps, jsonEncode(st.toJson()).length)); + } + // daytime hrv + { + oracleDaytimeHrv(dsub, 1700000000, 1700032040); + sw..reset()..start(); + oracleDaytimeHrv(dsub, 1700000000, 1700032040); + final full = ms(sw); + final st = DaytimeHrvState(1700000000, 1700032040); + final ps = []; + var ra = 0, aa = 0; + for (final cut in dcuts) { + final T = (day.ts[cut - 1] / 1000).floor() + 1; + var at = aa; + while (at < acc.length && acc.tsSec[at] < T) { + at++; + } + timed(sw, () { + st.fold(day.rr.sublist(ra, cut), day.ts.sublist(ra, cut), acc.tsSec.sublist(aa, at), + acc.ax.sublist(aa, at), acc.ay.sublist(aa, at), acc.az.sublist(aa, at), T); + st.result(); + }, ps); + ra = cut; + aa = at; + } + drows.add(Row('_daytimeHrv', full, ps, jsonEncode(st.toJson()).length)); + } + // day resp curve + { + oracleDayRespCurve(dsub); + sw..reset()..start(); + oracleDayRespCurve(dsub); + final full = ms(sw); + final st = DayRespCurveState(); + final ps = []; + var ra = 0, aa = 0; + for (final cut in dcuts) { + final T = (day.ts[cut - 1] / 1000).floor() + 1; + var at = aa; + while (at < acc.length && acc.tsSec[at] < T) { + at++; + } + timed(sw, () { + st.fold(day.rr.sublist(ra, cut), day.ts.sublist(ra, cut), acc.tsSec.sublist(aa, at), + acc.ax.sublist(aa, at), acc.ay.sublist(aa, at), acc.az.sublist(aa, at), T); + }, ps); + ra = cut; + aa = at; + } + drows.add(Row('dayRespCurve', full, ps, jsonEncode(st.toJson()).length)); + } + print('\nDAY estimators (23 h, 97k beats, 15-min passes)'); + print('| estimator | oracle whole-day ms | incr mean ms/pass | max | sum over passes | state bytes (json) |'); + print('|---|---|---|---|---|---|'); + drows.forEach((r) => print(r.line())); +} diff --git a/tool/incremental/size_probe.dart b/tool/incremental/size_probe.dart new file mode 100644 index 00000000..bfb88ecf --- /dev/null +++ b/tool/incremental/size_probe.dart @@ -0,0 +1,27 @@ +// JSON size of each carried state at the end of the real night (no timing). +import 'dart:convert'; +import 'driver.dart'; +import 'hrv_incr.dart'; +import 'oracle_util.dart'; +import 'resp_incr.dart'; +import 'synth.dart'; + +void main() { + final night = realNightRr()!; + final noc = NocturnalRmssdState(); + final shape = NightShapeState(); + final tl = HrvTimelineState(night.ts.first - night.rr.first); + final wins = RespWindowsState(); + final cuts = timeCuts(night.ts, 900); + driveRr(night, cuts, (f) { + noc.fold(f.settled.nn, f.settled.nnTimes); + shape.fold(f.settled.nn, f.settled.nnTimes); + tl.fold(f.settled.nn, f.settled.nnTimes); + wins.fold(f.settled.nn, f.settled.nnTimes); + }); + int sz(Object o) => jsonEncode(o).length; + print('nocturnal recs=${noc.recs.length} bytes=${sz([for (final r in noc.recs) (r as dynamic).toJson()])}'); + print('night shape closed bins=${shape.closed.length} bytes=${sz(shape.closed)}'); + print('timeline points=${tl.out.length} bytes=${sz(tl.out)} (+ ring <= 900 beats x 2 doubles ~ 30 KB)'); + print('resp windows closed=${wins.closed.length} bytes=${sz(wins.closed)}'); +} diff --git a/tool/incremental_benchmark.dart b/tool/incremental_benchmark.dart new file mode 100644 index 00000000..c814058f --- /dev/null +++ b/tool/incremental_benchmark.dart @@ -0,0 +1,142 @@ +// Run with: nix develop -c dart run tool/incremental_benchmark.dart +// Same deterministic growing inputs on both paths. Timings describe this +// process and machine; processed counts measure the incremental math work. +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; + +void main() { + final nn = [], times = []; + var time = 0.0; + for (var i = 0; i < 7200; i++) { + final value = 800 + 60 * math.sin(i * .13) + 20 * math.sin(i * .031); + time += value; + if (i % 731 == 730) time += 17000; + nn.add(value); + times.add(time); + } + final frequencies = List.generate(32, (i) => .01 + i * .012); + final seconds = [for (final t in times) t / 1000]; + final accel = List.generate( + 7200, + (i) => AccelSample(i * 1000.0, .1 * math.sin(i * .19), + .05 * math.cos(i * .07), 1 + .04 * math.sin(i * .31))); + final hr = List.generate(600, (i) => 75.0 + 65 * (1 + math.sin(i * .07)) / 2); + final keys = List.generate(hr.length, (i) => 28000000 + i); + final cadence = + List.generate(hr.length, (i) => i % 5 == 0 ? 110 : null); + const profile = + WorkoutUserProfile(weightKg: 80, heightCm: 180, age: 35, sex: 'male'); + + final spectrum = IncrementalLombScargle(frequencies); + compare( + 'Lomb, 32 frequencies', + nn.length, + 60, + (n) => lombScargle(seconds.sublist(0, n), nn.sublist(0, n), frequencies), + (n) => spectrum.sync(seconds.sublist(0, n), nn.sublist(0, n)), + () => spectrum.processedPoints); + final hrv = IncrementalHrvTime(); + compare( + 'Time HRV', + nn.length, + 60, + (n) => hrvTime(nn.sublist(0, n), nnTimesMs: times.sublist(0, n)), + (n) => hrv.sync(nn.sublist(0, n), nnTimesMs: times.sublist(0, n)), + () => hrv.processedPoints); + final motion = IncrementalEnmoSeries(); + compare( + 'Motion, explicit gravity reference', + accel.length, + 60, + (n) => enmoSeries(accel.sublist(0, n), gRef: 1), + (n) => motion.sync(accel.sublist(0, n), gRef: 1), + () => motion.processedPoints); + final minutes = IncrementalMinuteMetrics(); + compare('Minute load and energy', hr.length, 1, (n) { + final input = hr.sublist(0, n), walking = cadence.sublist(0, n); + final load = banisterTrimp(input, restingHr: 55, maxHr: 185, sex: Sex.male); + final strain = strainScoreMetric(load.value, + wakeMinutes: n.toDouble(), quietHrr: .12, quietSettled: false, female: false); + final energy = Calories.dailyEnergy(input, + profile: profile, hrmax: 185, restingHr: 55, cadenceSpmPerMin: walking); + final series = Calories.minuteEnergy(input, + profile: profile, + hrmax: 185, + restingHr: 55, + cadenceSpmPerMin: walking, + epochMinutes: keys.sublist(0, n)); + return MinuteMetrics( + trimp: load, strain: strain, energy: energy, minutes: series); + }, + (n) => minutes.sync(keys.sublist(0, n), hr.sublist(0, n), + cadenceSpm: cadence.sublist(0, n), + restingHr: 55, + maxHr: 185, + profile: profile, + quietHrr: .12), + () => minutes.processedMinutes); + final summary = IncrementalMinuteMetrics(); + compare('Minute summaries (no minute series)', hr.length, 1, (n) { + final input = hr.sublist(0, n), walking = cadence.sublist(0, n); + final load = banisterTrimp(input, restingHr: 55, maxHr: 185, sex: Sex.male); + final strain = strainScoreMetric(load.value, + wakeMinutes: n.toDouble(), quietHrr: .12, quietSettled: false, female: false); + final energy = Calories.dailyEnergy(input, + profile: profile, hrmax: 185, restingHr: 55, cadenceSpmPerMin: walking); + return MinuteMetrics(trimp: load, strain: strain, energy: energy); + }, + (n) => summary.sync(keys.sublist(0, n), hr.sublist(0, n), + cadenceSpm: cadence.sublist(0, n), + restingHr: 55, + maxHr: 185, + profile: profile, + quietHrr: .12, + includeMinuteSeries: false), + () => summary.processedMinutes); +} + +Object? _sink; + +void compare(String name, int length, int chunk, Object? Function(int) batch, + Object? Function(int) incremental, int Function() work) { + // Warm the same growing-prefix workloads that will be measured, including + // the append checks and artifact rendering on the incremental path. + for (var warm = 0; warm < 3; warm++) { + for (var n = chunk; n <= length; n += chunk) { + _sink = batch(n); + } + incremental(0); + for (var n = chunk; n <= length; n += chunk) { + _sink = incremental(n); + } + } + final fullTimes = [], deltaTimes = []; + var batchPoints = 0, processed = 0; + for (var repeat = 0; repeat < 5; repeat++) { + batchPoints = 0; + final full = Stopwatch()..start(); + for (var n = chunk; n <= length; n += chunk) { + _sink = batch(n); + batchPoints += n; + } + full.stop(); + fullTimes.add(full.elapsedMicroseconds); + incremental(0); + final workBefore = work(); + final delta = Stopwatch()..start(); + for (var n = chunk; n <= length; n += chunk) { + _sink = incremental(n); + } + delta.stop(); + deltaTimes.add(delta.elapsedMicroseconds); + processed = work() - workBefore; + } + fullTimes.sort(); + deltaTimes.sort(); + print('$name: batch ${fullTimes[2] / 1000} ms, ' + 'incremental ${deltaTimes[2] / 1000} ms (median of 5 warmed runs); ' + '$batchPoints batch input visits, $processed incremental points'); + // Keep results observable so the benchmark represents published output. + if (_sink == null) throw StateError('Benchmark produced no output'); +} From 9b23985e4748c845c24dbd9e5d05df58896e2b5d Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Fri, 13 Nov 2026 05:22:36 -0700 Subject: [PATCH 02/14] Refuse invalid clocks in incremental HRV Apply the batch clock refusal before touching retained state for regressions and supplied length mismatches. Keep the replacement and force fixture monotonic by shifting its suffix, and cover invalid clocks separately. --- CHANGELOG.md | 2 + lib/src/onehz/incremental.dart | 1 + lib/src/onehz/incremental_hrv.dart | 7 +++ test/onehz/incremental/hrv_time_test.dart | 59 +++++++++++++++++++++-- 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fca54ca..1185ae2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,8 @@ Use origin-shifted moments for NN levels and five-minute bins, and centered lag Method: Task Force 1996; Welford 1962 centered sums and products. Reproduce with `dart run tool/reproduce_outputs.dart`. ### Changed +- `IncrementalHrvTime` applies the batch refusal for regressing or mismatched + supplied clocks before updating its retained summary. - CTL/ATL aggregator checkpoints retain total and finite day counts. Older checkpoints without validity evidence are refused so unknown load cannot be silently restored as observed history. diff --git a/lib/src/onehz/incremental.dart b/lib/src/onehz/incremental.dart index a4a7c4a2..1eebd404 100644 --- a/lib/src/onehz/incremental.dart +++ b/lib/src/onehz/incremental.dart @@ -3,6 +3,7 @@ library; import 'dart:math' as math; import 'types.dart'; +import 'util.dart' show hasRegressingBeatTimes; import 'clinical/hrv_time.dart'; import 'clinical/load_trimp.dart'; import 'motion/enmo.dart'; diff --git a/lib/src/onehz/incremental_hrv.dart b/lib/src/onehz/incremental_hrv.dart index 3c576a30..21830992 100644 --- a/lib/src/onehz/incremental_hrv.dart +++ b/lib/src/onehz/incremental_hrv.dart @@ -104,6 +104,13 @@ class IncrementalHrvTime { {List? nnTimesMs, double artifactFraction = 0, bool force = false}) { + // CLAIM(incremental-hrv-clock-refusal): a supplied invalid clock must + // never become NN-only input or update the retained valid summary. + if (nnTimesMs != null && + (nnTimesMs.length != nnMs.length || hasRegressingBeatTimes(nnTimesMs))) { + return hrvTime(nnMs, + nnTimesMs: nnTimesMs, artifactFraction: artifactFraction); + } final timed = nnTimesMs != null && nnTimesMs.length == nnMs.length; final oldTimed = _times != null && _times!.length == _nn.length; final finite = nnMs.every((v) => v.isFinite) && diff --git a/test/onehz/incremental/hrv_time_test.dart b/test/onehz/incremental/hrv_time_test.dart index 9570da38..0d6c8000 100644 --- a/test/onehz/incremental/hrv_time_test.dart +++ b/test/onehz/incremental/hrv_time_test.dart @@ -69,7 +69,12 @@ void main() { state.sync(nn, nnTimesMs: times); nn[119] += 60; hrvClose(state.sync(nn, nnTimesMs: times), hrvTime(nn, nnTimesMs: times)); - times[113] += 18000; + // Move the suffix so this replacement remains a valid clock with a gap. + // A single moved timestamp regresses at the following beat; that refusal + // is covered separately below. + for (var i = 113; i < times.length; i++) { + times[i] += 18000; + } hrvClose(state.sync(nn, nnTimesMs: times), hrvTime(nn, nnTimesMs: times)); nn = nn.sublist(27, 700); times = times.sublist(27, 700); @@ -81,6 +86,46 @@ void main() { hrvClose(state.sync(nn), hrvTime(nn)); hrvClose(state.sync(nn, nnTimesMs: [1]), hrvTime(nn, nnTimesMs: [1])); }); + // why: review 9eea67c/d059e6e made invalid clocks absent in the batch; + // incremental reuse must apply that refusal before updating its cache. + // proves: CLAIM(incremental-hrv-clock-refusal) + // catches: accepting a regressing clock or treating a supplied short clock + // as an NN-only request. + test('HRV regressing clocks refuse without changing retained state', () { + final f = incrementalFixture(); + var state = IncrementalHrvTime(); + state.sync(f.nn, nnTimesMs: f.times); + final before = checkpoint(state.toJson()); + final regressing = [...f.times]..[113] += 18000; + for (final force in [false, true]) { + final result = state.sync(f.nn, nnTimesMs: regressing, force: force); + expect(result.value, isNull); + expect(result.note, 'regressing beat time'); + expect(state.toJson(), before); + } + state = IncrementalHrvTime.fromJson(before); + hrvClose(state.sync(f.nn, nnTimesMs: f.times), + hrvTime(f.nn, nnTimesMs: f.times)); + }); + + test('HRV supplied clock length mismatch never becomes NN-only input', () { + final f = incrementalFixture(); + final state = IncrementalHrvTime(); + state.sync(f.nn, nnTimesMs: f.times); + final before = checkpoint(state.toJson()); + for (final times in [ + [], + [1.0], + [...f.times, f.times.last + 900] + ]) { + final result = state.sync(f.nn, nnTimesMs: times); + expect(result.value, isNull); + expect(result.note, 'beat time length mismatch'); + expect(state.toJson(), before); + } + hrvClose(state.sync(f.nn), hrvTime(f.nn)); + }); + test('HRV constant series, singleton, empty and all separated seams', () { final state = IncrementalHrvTime(); for (final n in [0, 1, 2, 29, 30, 31, 32, 80]) { @@ -120,14 +165,22 @@ void main() { ( 'a respiratory line exempts the night', breathing( - hours: 1, baseMs: 1350, driftMs: 80, breathMs: 40, breathHz: .3, + hours: 1, + baseMs: 1350, + driftMs: 80, + breathMs: 40, + breathHz: .3, jitterMs: 3), 'PRV not ECG-HRV; RMSSD/pNN50', ), ( 'a refused night publishes from a breathing line steady in Hz', breathing( - hours: 2.5, baseMs: 900, driftMs: 400, breathMs: 10, breathHz: .25, + hours: 2.5, + baseMs: 900, + driftMs: 400, + breathMs: 10, + breathHz: .25, jitterMs: 40), 'breathing line steady in Hz', ), From d3fd56d1d6121eed04e01d50ac39cfb703b8ee7c Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sat, 14 Nov 2026 14:26:06 +0530 Subject: [PATCH 03/14] a slice with a non-finite value is refused as that, whether or not it is also out of order --- .../onehz/clinical/hrv_time_aggregator.dart | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/src/onehz/clinical/hrv_time_aggregator.dart b/lib/src/onehz/clinical/hrv_time_aggregator.dart index 58da4597..b20b4f50 100644 --- a/lib/src/onehz/clinical/hrv_time_aggregator.dart +++ b/lib/src/onehz/clinical/hrv_time_aggregator.dart @@ -167,18 +167,26 @@ class _CheckedBeats extends Aggregator<_CheckedBeatState, Beats, R> { @override _CheckedBeatState addSlice( _CheckedBeatState state, Beats batch, int start, int end) { - var previous = state.last; + // Finiteness of every row first, then order: a slice that is both out of + // order and holds a non-finite value is refused as the non-finite one, the + // same answer however the rows are arranged. Nothing changes either way. for (var i = start; i < end; i++) { if (!batch.nnMs[i].isFinite) { throw ArgumentError('NN intervals must be finite'); } - if (!gapAware) continue; - final time = batch.timeMs[i]; - if (!time.isFinite) throw ArgumentError('beat times must be finite'); - if (previous != null && time < previous) { - throw StateError('beat times must be non-decreasing'); + if (gapAware && !batch.timeMs[i].isFinite) { + throw ArgumentError('beat times must be finite'); + } + } + var previous = state.last; + if (gapAware) { + for (var i = start; i < end; i++) { + final time = batch.timeMs[i]; + if (previous != null && time < previous) { + throw StateError('beat times must be non-decreasing'); + } + previous = time; } - previous = time; } state.inner = inner.addSlice(state.inner, batch, start, end); if (gapAware && start < end) { From c1fc06fd48ec59fd7d04e393077d2eb47ee50e4a Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 06:15:12 -0700 Subject: [PATCH 04/14] Preserve restored basal pricing until parameters change --- lib/src/onehz/incremental_minutes.dart | 4 +++- .../incremental/minute_metrics_test.dart | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/src/onehz/incremental_minutes.dart b/lib/src/onehz/incremental_minutes.dart index 8affd79c..52fa1adc 100644 --- a/lib/src/onehz/incremental_minutes.dart +++ b/lib/src/onehz/incremental_minutes.dart @@ -213,7 +213,9 @@ class IncrementalMinuteMetrics { ? null : Calories.minuteEnergyPricer( profile: profile, hrmax: maxHr, restingHr: restingHr); - _basalPerMinute = _energyPricer?.basalKcalPerMin; + // Restoring the callable must not change an accepted checkpoint's rate. + // Only an explicit repricing replaces retained pricing state. + if (reprice) _basalPerMinute = _energyPricer?.basalKcalPerMin; _pricerReady = true; } if (reprice) { diff --git a/test/onehz/incremental/minute_metrics_test.dart b/test/onehz/incremental/minute_metrics_test.dart index bad8f0da..de1e5e0d 100644 --- a/test/onehz/incremental/minute_metrics_test.dart +++ b/test/onehz/incremental/minute_metrics_test.dart @@ -5,6 +5,28 @@ import '../support/incremental_compare.dart'; import '../support/incremental_fixtures.dart'; void main() { + // why: an accepted checkpoint must remain identical on an unchanged sync, + // even when its stored basal rate differs from the current calculator. + test('restored basal rate survives unchanged sync until repricing', () { + final profile = energyProfiles['male']!; + final original = IncrementalMinuteMetrics(); + original.sync([0], [110], restingHr: 54, maxHr: 186, profile: profile); + final saved = checkpoint(original.toJson()); + final basal = (saved['basalPerMinute'] as num).toDouble() + 1; + saved['basalPerMinute'] = basal; + final restored = IncrementalMinuteMetrics.fromJson(saved); + final accepted = checkpoint(restored.toJson()); + final result = restored.sync([0], [110], + restingHr: 54, maxHr: 186, profile: profile); + expect(restored.toJson(), accepted); + expect(result.minutes!.basalKcalPerMin, basal); + restored.sync([0, 1], [110, 120], + restingHr: 54, maxHr: 186, profile: profile); + expect(restored.toJson()['basalPerMinute'], basal); + final forced = restored.sync([0, 1], [110, 120], + restingHr: 54, maxHr: 186, profile: profile, force: true); + expect(forced.minutes!.basalKcalPerMin, basal - 1); + }); for (final entry in energyProfiles.entries) { for (final sex in Sex.values) { for (final chunk in [1, 19, 128]) { From f70602d3a744abdbf343618aede99a5535db31f1 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 06:15:12 -0700 Subject: [PATCH 05/14] Floor negative adjusted minutes into the correct hour --- lib/src/onehz/workout/calories.dart | 5 ++++- test/onehz/minute_energy_test.dart | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/src/onehz/workout/calories.dart b/lib/src/onehz/workout/calories.dart index a4926d07..d3033530 100644 --- a/lib/src/onehz/workout/calories.dart +++ b/lib/src/onehz/workout/calories.dart @@ -614,7 +614,10 @@ class Calories { }) { final byHour = >{}; for (final m in minutes) { - (byHour[(m.minute + minuteOffset) ~/ 60] ??= []).add(m); + final adjusted = m.minute + minuteOffset; + // Integer modulo keeps floor semantics for negative local-hour offsets. + final hour = (adjusted - adjusted % 60) ~/ 60; + (byHour[hour] ??= []).add(m); } final hours = byHour.keys.toList()..sort(); return [ diff --git a/test/onehz/minute_energy_test.dart b/test/onehz/minute_energy_test.dart index b8e1e50f..d18f2346 100644 --- a/test/onehz/minute_energy_test.dart +++ b/test/onehz/minute_energy_test.dart @@ -285,6 +285,18 @@ void main() { }); group('hourlyRollup', () { + // why: truncation toward zero moves negative adjusted minutes into hour 0. + test('negative adjusted minutes use floor at hour boundaries', () { + final series = Calories.minuteEnergy([70, 70, 70, 70], + profile: energyProfiles['male']!, hrmax: _hrmax, restingHr: _rhr, + epochMinutes: [0, 59, 60, 119])!; + final hours = Calories.hourlyRollup(series.minutes, minuteOffset: -60); + expect(hours.map((h) => h.hour), [-1, 0]); + expect(hours.map((h) => h.coveredMinutes), [2, 2]); + expect(hours.map((h) => h.coverage), [2 / 60, 2 / 60]); + expect(hours[0].total, series.minutes[0].total! + series.minutes[1].total!); + expect(hours[1].total, series.minutes[2].total! + series.minutes[3].total!); + }); final s = Calories.minuteEnergy(day.hr, profile: energyProfiles['male']!, hrmax: _hrmax, From c24ea0c368e8e7ad2b52a537b681f9a29bc28931 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 06:15:12 -0700 Subject: [PATCH 06/14] Reject unequal infinities in persistence comparisons --- test/aggregate/persistence_test.dart | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/aggregate/persistence_test.dart b/test/aggregate/persistence_test.dart index 8d2ca5f8..0f9bb3e9 100644 --- a/test/aggregate/persistence_test.dart +++ b/test/aggregate/persistence_test.dart @@ -61,7 +61,9 @@ Object? wire(Object? x) => jsonDecode(jsonEncode(x)); bool close(double a, double b) => a == b || (a.isNaN && b.isNaN) || - (a - b).abs() <= 1e-9 * [1.0, a.abs(), b.abs()].reduce((x, y) => x > y ? x : y); + (a.isFinite && b.isFinite && + (a - b).abs() <= + 1e-9 * [1.0, a.abs(), b.abs()].reduce((x, y) => x > y ? x : y)); bool closeOpt(double? a, double? b) => a == null || b == null ? a == b : close(a, b); @@ -266,6 +268,18 @@ class _SumCount extends Aggregator, _IB, int> { } void main() { + // why: Infinity <= Infinity must not let corrupt restored values pass. + test('checkpoint comparison distinguishes finite and non-finite values', () { + for (final pair in [(1.0, inf), (-inf, 1.0), (inf, -inf), (nan, inf)]) { + expect(close(pair.$1, pair.$2), isFalse, reason: '$pair'); + expect(close(pair.$2, pair.$1), isFalse, reason: 'reversed $pair'); + } + expect(close(inf, inf), isTrue); + expect(close(-inf, -inf), isTrue); + expect(close(nan, nan), isTrue); + expect(close(1.0, 1.0 + 1e-10), isTrue); + expect(close(1.0, 1.1), isFalse); + }); group('non-persistable defaults', () { test('an aggregator that overrides nothing is not persistable', () { const a = _Plain(); From dfd856edb3c2902eee277bfc2ef9f0ee313b78ba Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 06:15:12 -0700 Subject: [PATCH 07/14] Check aggregate exports and wrapped factories in registry closure --- lib/src/onehz/registry/group_registry.dart | 10 ++++-- test/registry_closure_test.dart | 40 ++++++++++++++++++---- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/lib/src/onehz/registry/group_registry.dart b/lib/src/onehz/registry/group_registry.dart index 97c89073..52887ea7 100644 --- a/lib/src/onehz/registry/group_registry.dart +++ b/lib/src/onehz/registry/group_registry.dart @@ -77,7 +77,12 @@ const GroupRegistry kAnalyticsGroupRegistry = GroupRegistry(1, [ counter: 2, members: ['median'], api: ['needBaselineNote'], - files: ['lib/src/onehz/types.dart', 'lib/src/onehz/util.dart'], + files: [ + 'lib/src/aggregate/checkpoint.dart', + 'lib/src/aggregate/laws.dart', + 'lib/src/onehz/types.dart', + 'lib/src/onehz/util.dart', + ], ), GroupDecl( id: 'found.rr_correction', @@ -113,7 +118,7 @@ const GroupRegistry kAnalyticsGroupRegistry = GroupRegistry(1, [ GroupDecl( id: 'found.energy', kind: GroupKind.foundation, - counter: 1, + counter: 2, members: ['Calories.dailyEnergy'], api: ['Calories.minuteEnergy', 'Calories.estimateBoutCalories'], files: ['lib/src/onehz/workout/calories.dart'], @@ -317,6 +322,7 @@ const GroupRegistry kAnalyticsGroupRegistry = GroupRegistry(1, [ 'lib/src/onehz/clinical/load_trimp_aggregator.dart', 'lib/src/onehz/clinical/nocturnal.dart', 'lib/src/onehz/clinical/readiness_lnrmssd.dart', + 'lib/src/onehz/clinical/readiness_lnrmssd_aggregator.dart', 'lib/src/onehz/clinical/stress_si.dart', 'lib/src/onehz/device.dart', 'lib/src/onehz/human/overreaching_conjunction.dart', diff --git a/test/registry_closure_test.dart b/test/registry_closure_test.dart index 857d4beb..d1614369 100644 --- a/test/registry_closure_test.dart +++ b/test/registry_closure_test.dart @@ -13,9 +13,9 @@ // // HOW (b) KNOWS THE PUBLIC API. The package has no runtime or analyzer // dependency (pure Dart, zero deps, by design), so the test does not link the -// analyzer. It follows `export` directives from lib/onehz.dart (honouring +// analyzer. It follows `export` directives from both public libraries (honouring // `show`/`hide`, which this package uses) and reads top-level declarations at -// column 0 after stripping comments. That reads the same files the compiler +// column 0, including wrapped return types, after stripping comments. That reads the same files the compiler // does, needs no hand-maintained map to rot, and a rename or a removed export // fails here. Its one weakness is an unusual top-level layout the regex does // not see; the sanity test below pins known names so a scanner regression is @@ -104,12 +104,16 @@ final _typeDecl = RegExp( r'^(?:(?:abstract|final|base|sealed|interface|mixin)\s+)*(?:class|mixin|enum|typedef|extension type)\s+(\w+)', multiLine: true); final _fnDecl = RegExp( - r'^(?!(?:import|export|part|library|class|enum|typedef|const|final|var|late|abstract|mixin|extension|static|return|if|for|while|switch|else|sealed|base|interface)\b)[A-Za-z_][\w<>,?\[\]. ]*?\s(\w+)\s*(?:<[^>(]*>)?\(', + r'^(?!(?:import|export|part|library|class|enum|typedef|const|final|var|late|abstract|mixin|extension|static|return|if|for|while|switch|else|sealed|base|interface)\b)[A-Za-z_][\w<>,?\[\]. \t\r\n]*?\s(\w+)\s*(?:<[^>(]*>)?\(', multiLine: true); /// Public top-level names declared in [file] itself, split by kind. ({Set fns, Set types}) _declared(String file) { - final src = _stripComments(_read(file)); + return _declaredSource(_read(file)); +} + +({Set fns, Set types}) _declaredSource(String source) { + final src = _stripComments(source); final types = {for (final m in _typeDecl.allMatches(src)) m.group(1)!}; final fns = {for (final m in _fnDecl.allMatches(src)) m.group(1)!} ..removeWhere((n) => n.startsWith('_') || types.contains(n)); @@ -117,10 +121,10 @@ final _fnDecl = RegExp( return (fns: fns, types: types); } -/// name -> declaring file, for everything lib/onehz.dart exports (public API). +/// name -> declaring file, for everything the public libraries export. /// [fnsOnly] keeps top-level functions only (constructors/types excluded). Map _publicApi( - {bool fnsOnly = false, List libs = const ['lib/onehz.dart']}) { + {bool fnsOnly = false, List libs = _publicLibs}) { final api = {}; final seen = {}; // Returns the names [file] exposes to an importer. @@ -249,6 +253,30 @@ void main() { }); }); + // why: aggregate-only multiline factories must participate in ownership + // checks just like functions exported through onehz.dart. + test('declaration scanner handles wrapped returns without scanning methods', () { + const source = """ +Aggregator> + factoryWithWrappedType({int window = 7}) => build(); +double oneLine() => 1; +class Example { + double method() => 1; +} +"""; + expect(_declaredSource(source).fns, {'factoryWithWrappedType', 'oneLine'}); + expect(_declaredSource(source).types, {'Example'}); + }); + test('public API scan includes the multiline aggregate-only readiness factory', () { + expect(_publicApi(libs: _publicLibs)['readinessLnRmssdAggregator'], + 'lib/src/onehz/clinical/readiness_lnrmssd_aggregator.dart'); + }); + test('orphan scan includes aggregate-only functions by default', () { + expect(_publicApi(fnsOnly: true)['readinessLnRmssdAggregator'], + 'lib/src/onehz/clinical/readiness_lnrmssd_aggregator.dart'); + }); + group('directive parser (guards the guard)', () { const f = 'lib/src/onehz/a.dart'; List paths(String src) => From 4e9ef20bccea0db057882b3d0f9ce7ce3638c903 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 06:15:12 -0700 Subject: [PATCH 08/14] Correct integration status and headline strain documentation --- docs/COMBINED_BUMP_STATUS.md | 11 ++++++----- docs/INCREMENTAL_INTEGRATION_PLAN.md | 25 ++++++++++--------------- docs/INCREMENTAL_MATH.md | 17 ++++++++++++----- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/docs/COMBINED_BUMP_STATUS.md b/docs/COMBINED_BUMP_STATUS.md index b17cdf65..e598f2c9 100644 --- a/docs/COMBINED_BUMP_STATUS.md +++ b/docs/COMBINED_BUMP_STATUS.md @@ -22,7 +22,8 @@ Implemented APIs: coverage is retained so sparse cardiac evidence cannot appear complete merely because each epoch has a mean. Carry-only epochs also remain in the classifier's coverage denominator. -- The default advanced stager delegates to these rows. Feature +- The default advanced stager delegates to `cardioStager` with a + `CardioRowCache`, not to `EpochRows`. Feature context extends beyond the requested session. The existing bounded accel coverage and unobserved-run semantics remain in use. - Registry counters `found.core`, `found.rr_correction`, `sleep.stage`, @@ -169,10 +170,10 @@ Tests changed: The existing advanced-stager efficiency assertion and the 0.97 agreement assertion have not been weakened. With option 2 the former passes again. -The capped latency is above target, the still-night efficiency guard fails -(see Grid), and the sandwich reach and the adversarial soft-run influence -bounds still need a proof beyond the passing prefix/oracle cases. These are -open limitations, not completed acceptance criteria. +The still-night efficiency guard passes with option 2 (see Grid). Capped +latency remains above target, and the sandwich reach and adversarial soft-run +influence bounds still need a proof beyond the passing prefix/oracle cases. +These remain open limitations. Benchmarks below used the supplied Dart 3.11.4 executable in JIT mode. They are local measurements, not device timings. The new probe is reproducible via diff --git a/docs/INCREMENTAL_INTEGRATION_PLAN.md b/docs/INCREMENTAL_INTEGRATION_PLAN.md index 434bbdac..750fc6f6 100644 --- a/docs/INCREMENTAL_INTEGRATION_PLAN.md +++ b/docs/INCREMENTAL_INTEGRATION_PLAN.md @@ -8,7 +8,7 @@ Hand-off for getting the incremental work into the app (`edge`). Read | Repo | Branch | Location | State | |---|---|---|---| -| analytics | `feat/minute-energy` | this checkout | `4bd76f7` committed; later changes below uncommitted | +| analytics | `split/area-6-integration` | [PR #108](https://github.com/OpenStrap/analytics/pull/108) | committed; integration review in progress | | edge | `feat/incremental-analytics` | `../edge.worktrees/incremental-analytics` (git worktree of `../edge`) | all incremental work uncommitted on top of `687b5bd0` | `687b5bd0` is a snapshot of the scheduler work that was uncommitted in @@ -79,20 +79,15 @@ before the work: about 350 ms compute per derive in ## Steps -1. **Land this repo's changes.** Uncommitted: - - `hrDipFromDayTotals`, which `hrDip` now shares its scoring with - - `IncrementalMinuteMetrics.sync` taking `int dayMinutes` (as - `Calories.dailyEnergy` does) - - day-duration tests - - `drift_test.dart` - - `IntHistogram` - - the first-sample time shift in `lombScargle` and its precision test - - this plan - - Commit them, open a PR, and note the new commit SHA. The incremental work - changes no algorithm output, but the `lombScargle` shift does (about 1e-6 - relative, worst case), so edge must bump `kAlgoVersion` when it pins a commit - that contains it. +1. **Land the analytics integration.** The analytics changes described here + are committed in [PR #108](https://github.com/OpenStrap/analytics/pull/108), + including the preceding split areas. Start app integration from that PR's + reviewed head, or its merged commit once available, and record the full SHA + used for the edge dependency pin. + + The `lombScargle` first-timestamp shift changes output (about 1e-6 relative, + worst case), so edge must bump `kAlgoVersion` when it pins a commit that + contains it. 2. **Move the edge work onto the current edge branch.** Commit the worktree's changes on `feat/incremental-analytics`, then rebase onto diff --git a/docs/INCREMENTAL_MATH.md b/docs/INCREMENTAL_MATH.md index a72a94ef..3e7db499 100644 --- a/docs/INCREMENTAL_MATH.md +++ b/docs/INCREMENTAL_MATH.md @@ -237,15 +237,22 @@ With fixed resting HR `R`, maximum `H`, coefficients `c,b`, and time weight `dt` x = clamp((HR-R)/(H-R), 0, 1) f(x) = c*x*exp(b*x) TRIMP = sum(dt*f(x)) -baseline = wakeMinutes*f(q) -net = TRIMP-baseline +exercise = sum(f(x)-f(q), over valid minutes with x >= 0.40) +living = sum(f(x)-f(q), over valid minutes with x < 0.40) +net = exercise + max(0, living) u = clamp(net/400, 0, 1) strain = 21*log(1+14*u)/log(15) ``` -For net at or below zero, current strain is zero. Awake minute count is part -of the state because additional quiet time changes the subtraction too. -The separate legacy 0–100 map remains a separate API. +This is the headline `netTrimpAboveQuiet` rule for one-minute inputs. Each +valid minute subtracts the quiet rate `f(q)`, but the living block is floored +at zero so quiet living minutes cannot debit the exercise block. Off-skin +and non-finite HR minutes are excluded. + +The separate lump form is `max(0, TRIMP - wakeMinutes*f(q))`; it subtracts +quiet cost across all minutes together and can debit exercise with quiet +living minutes. Awake minute count is part of that state because additional +quiet time changes the subtraction. The legacy 0–100 map is also separate. For timestamp-weighted scoring, the last sample provisionally gets median cadence. Arrival of the next sample replaces that duration with the capped From 3c60d1cde783c24af648634c8ef041c144e91a93 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 06:15:12 -0700 Subject: [PATCH 09/14] Report complete stream timings and distinguish estimated state sizes --- tool/bench_phase0.dart | 21 +++++++++++---------- tool/incremental/bench_all.dart | 24 +++++++++++++----------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/tool/bench_phase0.dart b/tool/bench_phase0.dart index c1edee88..b6a0da4b 100644 --- a/tool/bench_phase0.dart +++ b/tool/bench_phase0.dart @@ -81,7 +81,7 @@ void _cvhrStreamBench() { var from = 0, streamTotal = 0, batchTotal = 0; var parity = true; final start = night.tMs.first; - print('CVHR stream: pass stream us batch us parity'); + print('CVHR stream: pass stream fold+snapshot us batch us parity'); for (var pass = 1; pass <= 32; pass++) { final until = start + pass * 15 * 60 * 1000.0; var end = from; @@ -89,9 +89,9 @@ void _cvhrStreamBench() { end++; } if (end == from) continue; + final streamWatch = Stopwatch()..start(); stream.fold(night.nn.sublist(from, end), night.tMs.sublist(from, end)); from = end; - final streamWatch = Stopwatch()..start(); final streamed = stream.snapshot(artifactFraction: 0.02); streamWatch.stop(); final batchWatch = Stopwatch()..start(); @@ -165,7 +165,7 @@ void _welchStreamBench() { var rsaStreamTotal = 0, rsaBatchTotal = 0; var parity = true; final start = night.tMs.first; - print('Welch streams: pass stream hrv/rsa us batch hrv/rsa us parity'); + print('Welch streams: pass stream fold+snapshot hrv/rsa us batch hrv/rsa us parity'); for (var pass = 1; pass <= 32; pass++) { final until = start + pass * 15 * 60 * 1000.0; var end = from; @@ -173,20 +173,20 @@ void _welchStreamBench() { end++; } if (end == from) continue; - hrv.fold(night.nn.sublist(from, end), night.tMs.sublist(from, end)); - rsa.fold(night.nn.sublist(from, end), night.tMs.sublist(from, end)); - from = end; final tailEnd = (end + 100).clamp(0, night.nn.length); final tailNn = night.nn.sublist(end, tailEnd); final tailTimes = night.tMs.sublist(end, tailEnd); final hrvStreamWatch = Stopwatch()..start(); + hrv.fold(night.nn.sublist(from, end), night.tMs.sublist(from, end)); final streamHrv = hrv.snapshot( tailNnMs: tailNn, tailNnTimesMs: tailTimes, artifactFraction: 0.02); hrvStreamWatch.stop(); final rsaStreamWatch = Stopwatch()..start(); + rsa.fold(night.nn.sublist(from, end), night.tMs.sublist(from, end)); final streamRsa = rsa.snapshot( tailNnMs: tailNn, tailNnTimesMs: tailTimes, artifactFraction: 0.02); rsaStreamWatch.stop(); + from = end; final hrvBatchWatch = Stopwatch()..start(); final batchHrv = hrvFreq(night.nn.sublist(0, tailEnd), night.tMs.sublist(0, tailEnd), artifactFraction: 0.02); @@ -240,8 +240,9 @@ void main(List args) { print('case before us after us'); var parity = true; void row(String key, String name, void Function() before, void Function() after, - bool same) { + bool Function() compare) { if (!selected.contains(key)) return; + final same = compare(); parity = parity && same; final beforeUs = _medianUs(before); final afterUs = _medianUs(after); @@ -249,11 +250,11 @@ void main(List args) { '${afterUs.toString().padLeft(9)} parity=$same'); } row('cvhr', 'cvhrApneaScreen (8 h)', cvhrBefore, cvhrAfter, - _sameCvhr(cvhrBefore(), cvhrAfter())); + () => _sameCvhr(cvhrBefore(), cvhrAfter())); row('hrv', 'hrvFreq (30k beats)', hrvBefore, hrvAfter, - _sameHrv(hrvBefore(), hrvAfter())); + () => _sameHrv(hrvBefore(), hrvAfter())); row('cardio', 'cardioStager (one night)', stagerBefore, stagerAfter, - _sameStager(stagerBefore(), stagerAfter())); + () => _sameStager(stagerBefore(), stagerAfter())); if (args.isEmpty || selected.contains('cvhr-stream')) _cvhrStreamBench(); if (selected.contains('cardio') || selected.contains('cache')) { final extra = frozen.genNight(9004, 15 * 60, rrOutliers: true); diff --git a/tool/incremental/bench_all.dart b/tool/incremental/bench_all.dart index 9c26413a..ff9420f0 100644 --- a/tool/incremental/bench_all.dart +++ b/tool/incremental/bench_all.dart @@ -20,9 +20,11 @@ class Row { final double oracleFullMs; final List passMs; // fold+evaluate per pass final int stateBytes; - Row(this.name, this.oracleFullMs, this.passMs, this.stateBytes); + final bool estimated; + Row(this.name, this.oracleFullMs, this.passMs, this.stateBytes, + {this.estimated = false}); String line() => - '| $name | ${oracleFullMs.toStringAsFixed(0)} | ${mean(passMs).toStringAsFixed(1)} | ${mx(passMs).toStringAsFixed(1)} | ${sum(passMs).toStringAsFixed(0)} | $stateBytes |'; + '| $name | ${oracleFullMs.toStringAsFixed(0)} | ${mean(passMs).toStringAsFixed(1)} | ${mx(passMs).toStringAsFixed(1)} | ${sum(passMs).toStringAsFixed(0)} | $stateBytes (${estimated ? "estimate" : "JSON"}) |'; } T timed(Stopwatch sw, T Function() f, List sink) { @@ -44,7 +46,7 @@ void main() { final sw = Stopwatch(); final rows = []; - Row bench(String name, double Function() oracle, void Function(FoldStep, List) pass, int Function() bytes) { + Row bench(String name, double Function() oracle, void Function(FoldStep, List) pass, int Function() bytes, {bool estimated = false}) { oracle(); // warm sw..reset()..start(); oracle(); @@ -53,7 +55,7 @@ void main() { for (final f in steps) { pass(f, ps); } - final r = Row(name, fullMs, ps, bytes()); + final r = Row(name, fullMs, ps, bytes(), estimated: estimated); rows.add(r); return r; } @@ -84,7 +86,7 @@ void main() { st.fold(f.settled.nn, f.settled.nnTimes); st.evaluate(f.snap.tailNn, f.snap.tailNnTimes); }, ps); - }, () => st.recs.length * 60); + }, () => st.recs.length * 60, estimated: true); } // session rmssd (raw) { @@ -104,7 +106,7 @@ void main() { k++; st.evaluate(); }, ps); - }, () => st.recs.length * 60); + }, () => st.recs.length * 60, estimated: true); } // night shape { @@ -118,7 +120,7 @@ void main() { st.fold(f.settled.nn, f.settled.nnTimes); st.evaluate(f.snap.tailNn, f.snap.tailNnTimes); }, ps); - }, () => st.closed.length * 40); + }, () => st.closed.length * 40, estimated: true); } // hrv timeline { @@ -133,7 +135,7 @@ void main() { st.fold(f.settled.nn, f.settled.nnTimes); st.curve(f.snap.tailNn, f.snap.tailNnTimes); }, ps); - }, () => st.out.length * 20 + 900 * 16); + }, () => st.out.length * 20 + 900 * 16, estimated: true); } // rsa { @@ -161,7 +163,7 @@ void main() { st.fold(f.settled.nn, f.settled.nnTimes); st.evaluate(f.snap.tailNn, f.snap.tailNnTimes); }, ps); - }, () => st.closed.length * 8 + 12000); + }, () => st.closed.length * 8 + 12000, estimated: true); } // hrvFreq { @@ -178,7 +180,7 @@ void main() { }, () => jsonEncode(st.toJson()).length); } print('\nNIGHT estimators (8.9 h, 32k beats, 15-min passes)'); - print('| estimator | oracle whole-night ms | incr mean ms/pass | max | sum over passes | state bytes (json) |'); + print('| estimator | oracle whole-night ms | incr mean ms/pass | max | sum over passes | state size (JSON bytes or estimate) |'); print('|---|---|---|---|---|---|'); rows.forEach((r) => print(r.line())); @@ -277,7 +279,7 @@ void main() { drows.add(Row('dayRespCurve', full, ps, jsonEncode(st.toJson()).length)); } print('\nDAY estimators (23 h, 97k beats, 15-min passes)'); - print('| estimator | oracle whole-day ms | incr mean ms/pass | max | sum over passes | state bytes (json) |'); + print('| estimator | oracle whole-day ms | incr mean ms/pass | max | sum over passes | state size (JSON bytes or estimate) |'); print('|---|---|---|---|---|---|'); drows.forEach((r) => print(r.line())); } From 048156ade65ffcafcafcaa283e987577678988f2 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 06:17:09 -0700 Subject: [PATCH 10/14] Keep untimed parity evaluation outside benchmark reporting fixes --- tool/bench_phase0.dart | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tool/bench_phase0.dart b/tool/bench_phase0.dart index b6a0da4b..0806bcef 100644 --- a/tool/bench_phase0.dart +++ b/tool/bench_phase0.dart @@ -240,9 +240,8 @@ void main(List args) { print('case before us after us'); var parity = true; void row(String key, String name, void Function() before, void Function() after, - bool Function() compare) { + bool same) { if (!selected.contains(key)) return; - final same = compare(); parity = parity && same; final beforeUs = _medianUs(before); final afterUs = _medianUs(after); @@ -250,11 +249,11 @@ void main(List args) { '${afterUs.toString().padLeft(9)} parity=$same'); } row('cvhr', 'cvhrApneaScreen (8 h)', cvhrBefore, cvhrAfter, - () => _sameCvhr(cvhrBefore(), cvhrAfter())); + _sameCvhr(cvhrBefore(), cvhrAfter())); row('hrv', 'hrvFreq (30k beats)', hrvBefore, hrvAfter, - () => _sameHrv(hrvBefore(), hrvAfter())); + _sameHrv(hrvBefore(), hrvAfter())); row('cardio', 'cardioStager (one night)', stagerBefore, stagerAfter, - () => _sameStager(stagerBefore(), stagerAfter())); + _sameStager(stagerBefore(), stagerAfter())); if (args.isEmpty || selected.contains('cvhr-stream')) _cvhrStreamBench(); if (selected.contains('cardio') || selected.contains('cache')) { final extra = frozen.genNight(9004, 15 * 60, rrOutliers: true); From 699a9bd5fa8539897ec2032eb029c2ebd9ef73b0 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 06:22:16 -0700 Subject: [PATCH 11/14] Record the hourly energy output version change Hourly rollup now changes results for negative adjusted minute keys, so found.energy must be in the exact set of output groups bumped to counter 2. --- docs/COMBINED_BUMP_STATUS.md | 3 ++- test/algo_version_test.dart | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/COMBINED_BUMP_STATUS.md b/docs/COMBINED_BUMP_STATUS.md index e598f2c9..19c9e738 100644 --- a/docs/COMBINED_BUMP_STATUS.md +++ b/docs/COMBINED_BUMP_STATUS.md @@ -26,12 +26,13 @@ Implemented APIs: `CardioRowCache`, not to `EpochRows`. Feature context extends beyond the requested session. The existing bounded accel coverage and unobserved-run semantics remain in use. -- Registry counters `found.core`, `found.rr_correction`, `sleep.stage`, +- Registry counters `found.core`, `found.rr_correction`, `found.energy`, `sleep.stage`, `hrv.freq`, `night.rhythm` and `activity.load` are 2. A group's own algorithm changes, including rounding and abstention, require a bump; dependency-only changes propagate through `deps`. One bump per group covers this PR: - `found.core`: Lomb-Scargle now subtracts the first timestamp. - `found.rr_correction`: the artifact-run cap, already bumped. + - `found.energy`: hourly buckets floor negative adjusted minute keys. - `sleep.stage`: retain the existing bump; its direct staging path also computes adopted HRV statistics. Session-onset/post-gap anchoring is retained. - `hrv.freq`: adopted HRV rounding and non-finite-time abstention. diff --git a/test/algo_version_test.dart b/test/algo_version_test.dart index b51b688c..1e7d53bb 100644 --- a/test/algo_version_test.dart +++ b/test/algo_version_test.dart @@ -8,6 +8,7 @@ void main() { const changed = { 'found.core', // Lomb-Scargle first-sample origin. 'found.rr_correction', // Artifact-run cap, already bumped. + 'found.energy', // Floor negative adjusted minutes in hourly energy. 'sleep.stage', // Already bumped; also owns adopted HRV staging paths. 'hrv.freq', // HRV aggregator rounding and invalid-time abstention. 'night.rhythm', // Cosinor/circadian and gap-aware irregular screening. From 793b19343fabf6b22370402ed8878e8269412423 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 07:51:17 -0700 Subject: [PATCH 12/14] Use restored basal for incremental minute pricing --- lib/src/onehz/incremental_minutes.dart | 9 ++- lib/src/onehz/workout/calories.dart | 13 ++-- .../incremental/minute_metrics_test.dart | 63 +++++++++++++++++++ 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/lib/src/onehz/incremental_minutes.dart b/lib/src/onehz/incremental_minutes.dart index 52fa1adc..f2b6ea3d 100644 --- a/lib/src/onehz/incremental_minutes.dart +++ b/lib/src/onehz/incremental_minutes.dart @@ -212,9 +212,14 @@ class IncrementalMinuteMetrics { _energyPricer = profile == null || restingHr == null || maxHr == null ? null : Calories.minuteEnergyPricer( - profile: profile, hrmax: maxHr, restingHr: restingHr); + profile: profile, + hrmax: maxHr, + restingHr: restingHr, + basalKcalPerMin: reprice ? null : _basalPerMinute); // Restoring the callable must not change an accepted checkpoint's rate. - // Only an explicit repricing replaces retained pricing state. + // CLAIM(restored-minute-pricing): retained and new bills use that rate + // [minute_metrics_test: restored basal prices appended surplus]. + // Parameter changes or force select the current profile's rate instead. if (reprice) _basalPerMinute = _energyPricer?.basalKcalPerMin; _pricerReady = true; } diff --git a/lib/src/onehz/workout/calories.dart b/lib/src/onehz/workout/calories.dart index d3033530..32bfa63c 100644 --- a/lib/src/onehz/workout/calories.dart +++ b/lib/src/onehz/workout/calories.dart @@ -372,7 +372,8 @@ class Calories { double hrmax, double flexHr, double basalPerMin, - })? _energyCtx(WorkoutUserProfile profile, double hrmax, double restingHr) { + })? _energyCtx(WorkoutUserProfile profile, double hrmax, double restingHr, + {double? basalKcalPerMin}) { final weightKg = profile.weightKg > 0 ? profile.weightKg : 70.0; final heightCm = profile.heightCm > 0 ? profile.heightCm : 170.0; final age = profile.age > 0 ? profile.age : 30.0; @@ -384,8 +385,8 @@ class Calories { age: age, hrmax: hrmax, flexHr: flexHr, - basalPerMin: mifflinBmrKcalDay(weightKg, heightCm, age, profile.sex) / - 1440.0, + basalPerMin: basalKcalPerMin ?? + mifflinBmrKcalDay(weightKg, heightCm, age, profile.sex) / 1440.0, ); } @@ -529,6 +530,8 @@ class Calories { /// Prepares fixed profile and anchor math once for independent minute bills. /// Returns null under the same anchor gate as [dailyEnergy]. + /// [basalKcalPerMin] retains a checkpoint's pricing basis for both HR and + /// walking surplus; otherwise the profile determines the basal rate. // Keytel et al. 2005 active energy, Mifflin-St Jeor 1990 basal energy and Tudor- // Locke et al. 2019 cadence METs, as in dailyEnergy(). Shared minute bills // preserve bill order and separate HR/walking accumulation; hourly output sums @@ -537,8 +540,10 @@ class Calories { required WorkoutUserProfile profile, required double hrmax, required double restingHr, + double? basalKcalPerMin, }) { - final ctx = _energyCtx(profile, hrmax, restingHr); + final ctx = + _energyCtx(profile, hrmax, restingHr, basalKcalPerMin: basalKcalPerMin); if (ctx == null) return null; return MinuteEnergyPricer._( ctx.basalPerMin, (hr, cadence) => _bill(ctx, hr, cadence)); diff --git a/test/onehz/incremental/minute_metrics_test.dart b/test/onehz/incremental/minute_metrics_test.dart index de1e5e0d..d15b14bc 100644 --- a/test/onehz/incremental/minute_metrics_test.dart +++ b/test/onehz/incremental/minute_metrics_test.dart @@ -27,6 +27,69 @@ void main() { restingHr: 54, maxHr: 186, profile: profile, force: true); expect(forced.minutes!.basalKcalPerMin, basal - 1); }); + // why: restored bills and appended HR/walking surplus must share the + // reported basal, even if the current calculator would choose another rate. + // proves: CLAIM(restored-minute-pricing) + // catches: rebuilding the pricer without its checkpoint's basal rate. + for (final includeMinuteSeries in [true, false]) { + test('restored basal prices appended surplus series=$includeMinuteSeries', + () { + const profile = + WorkoutUserProfile(weightKg: 80, heightCm: 180, age: 30, sex: 'male'); + const taller = + WorkoutUserProfile(weightKg: 80, heightCm: 200, age: 30, sex: 'male'); + const keys = [0, 1, 2, 3]; + const hr = [60.0, 150.0, 60.0, 150.0]; + const cadence = [110, null, 110, null]; + final original = IncrementalMinuteMetrics(); + original.sync(keys.sublist(0, 2), hr.sublist(0, 2), + cadenceSpm: cadence.sublist(0, 2), + restingHr: 54, + maxHr: 186, + profile: profile); + final saved = checkpoint(original.toJson()); + // Simulate a checkpoint whose accepted basal differs from today's + // profile calculation. Height changes basal only, so a fresh computation + // with the original profile is an oracle for every retained/new bill. + ((saved['parameters'] as List)[3] as List)[1] = taller.heightCm; + final restored = IncrementalMinuteMetrics.fromJson(saved); + final result = restored.sync(keys, hr, + cadenceSpm: cadence, + restingHr: 54, + maxHr: 186, + profile: taller, + includeMinuteSeries: includeMinuteSeries); + final freshState = IncrementalMinuteMetrics(); + final fresh = freshState.sync(keys, hr, + cadenceSpm: cadence, restingHr: 54, maxHr: 186, profile: profile); + // Mifflin: (10*80 + 6.25*180 - 5*30 + 5)/1440 = 89/72. + const basal = 89 / 72; + // 110 spm is 4 METs; Keytel male at 150 bpm is + // (-55.0969 + .6309*150 + .1988*80 + .2017*30)/4.184 kcal/min. + const walking = 3 * basal; + const active = 61.4931 / 4.184 - basal; + final bills = restored.toJson()['bills'] as List; + expect(bills[2]['source'], 'cadence'); + expect(bills[2]['walking'], closeTo(walking, 1e-12)); + expect(bills[2]['active'], 0); + expect(bills[3]['source'], 'hr'); + expect(bills[3]['active'], closeTo(active, 1e-12)); + expect(bills[3]['walking'], 0); + expect(restored.toJson()['basalPerMinute'], basal); + expect(result.energy!.basal, basal * 1440); + expect(result.energy, fresh.energy); + expect(bills, freshState.toJson()['bills']); + expect(restored.processedMinutes, 4, + reason: 'accepted prefix bills remain priced exactly once'); + if (includeMinuteSeries) { + expect(result.minutes!.basalKcalPerMin, basal); + expect(result.minutes!.minutes[2].walking, closeTo(walking, 1e-12)); + expect(result.minutes!.minutes[3].active, closeTo(active, 1e-12)); + } else { + expect(result.minutes, isNull); + } + }); + } for (final entry in energyProfiles.entries) { for (final sex in Sex.values) { for (final chunk in [1, 19, 128]) { From 8a44d4d99e8d89ba503928ba169fcbd61c094577 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 14 Nov 2026 16:01:23 -0700 Subject: [PATCH 13/14] Assert equal Epochs column lengths --- lib/src/onehz/sleep/sri_aggregator.dart | 3 ++- test/aggregate/sri_aggregator_test.dart | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/src/onehz/sleep/sri_aggregator.dart b/lib/src/onehz/sleep/sri_aggregator.dart index 107da07f..1fdd2014 100644 --- a/lib/src/onehz/sleep/sri_aggregator.dart +++ b/lib/src/onehz/sleep/sri_aggregator.dart @@ -11,7 +11,8 @@ import 'sri.dart'; /// [firstIndex] is the absolute index of row zero. It lets [Lookback] preserve /// the day a comparison belongs to when it joins copied chunks at a seam. class Epochs extends Batch { - Epochs(this.asleep, {this.valid, this.firstIndex = 0}); + Epochs(this.asleep, {this.valid, this.firstIndex = 0}) + : assert(valid == null || valid.length == asleep.length); factory Epochs.fromJson(Object? json) { final map = decodeMap(json); diff --git a/test/aggregate/sri_aggregator_test.dart b/test/aggregate/sri_aggregator_test.dart index bffc41d6..7090563e 100644 --- a/test/aggregate/sri_aggregator_test.dart +++ b/test/aggregate/sri_aggregator_test.dart @@ -230,6 +230,18 @@ void _checkRecord(Rng r, _Rec rec, int epd, int? floor, String tag, void main() { group('Epochs batch', () { + // why: reject mismatched columns at construction before aggregation can + // overrun a short mask or silently ignore a long one (analytics#107). + test('constructor asserts equal column lengths', () { + for (final maskLength in [1, 3]) { + expect( + () => Epochs(Uint8List(2), valid: Uint8List(maskLength)), + throwsA(isA()), + reason: 'valid length $maskLength must match asleep length 2', + ); + } + }); + Epochs mk(int n, {bool mask = false, int first = 0}) => Epochs( Uint8List.fromList([for (var i = 0; i < n; i++) i % 3 == 0 ? 1 : 0]), valid: mask From 128972201f7604ad384bc4f59367bba7bb289f84 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 15 Nov 2026 13:18:35 +0530 Subject: [PATCH 14/14] a negative basal rate is refused at restore and as an override, the rsa round-trip law gets a budget that fits a slow runner --- lib/src/onehz/incremental_minutes.dart | 6 ++ lib/src/onehz/workout/calories.dart | 7 ++ .../incremental/minute_metrics_test.dart | 77 +++++++++++++++++++ test/properties/welch_stream_laws_test.dart | 13 +++- 4 files changed, 101 insertions(+), 2 deletions(-) diff --git a/lib/src/onehz/incremental_minutes.dart b/lib/src/onehz/incremental_minutes.dart index f2b6ea3d..a54a0e4c 100644 --- a/lib/src/onehz/incremental_minutes.dart +++ b/lib/src/onehz/incremental_minutes.dart @@ -48,6 +48,12 @@ class IncrementalMinuteMetrics { state._basalPerMinute = json['basalPerMinute'] == null ? null : _number(json['basalPerMinute'], finite: true); + // A basal rate is energy per minute at rest: never negative. A negative one + // would pass through an unchanged-parameter sync and price a walking minute + // at a negative surplus, which the next restore then refuses. + if (state._basalPerMinute != null && state._basalPerMinute! < 0) { + throw const FormatException('Invalid basal rate'); + } if (json['parameters'] != null) { final parameters = json['parameters']; if (parameters is! List || diff --git a/lib/src/onehz/workout/calories.dart b/lib/src/onehz/workout/calories.dart index 32bfa63c..17860045 100644 --- a/lib/src/onehz/workout/calories.dart +++ b/lib/src/onehz/workout/calories.dart @@ -379,6 +379,13 @@ class Calories { final age = profile.age > 0 ? profile.age : 30.0; final flexHr = activeGateHr(hrmax, restingHr); if (flexHr == null) return null; + // An override stands in for the profile's basal rate, which is never + // negative or non-finite: a bad one is refused like a bad anchor (no + // pricing) instead of billing minutes at a negative surplus. + if (basalKcalPerMin != null && + !(basalKcalPerMin.isFinite && basalKcalPerMin >= 0)) { + return null; + } return ( coeffs: resolveCoeffs(profile.sex), weightKg: weightKg, diff --git a/test/onehz/incremental/minute_metrics_test.dart b/test/onehz/incremental/minute_metrics_test.dart index d15b14bc..70776072 100644 --- a/test/onehz/incremental/minute_metrics_test.dart +++ b/test/onehz/incremental/minute_metrics_test.dart @@ -333,4 +333,81 @@ void main() { profile: p); } }); + + // why: a basal rate is energy per minute at rest, never negative. A checkpoint + // carrying -1 decoded, passed through an unchanged-parameter sync, and then + // priced an appended walking minute (HR 60, cadence 110, resting HR 54, max HR + // 186) at a walking surplus of -3, a contribution the next restore refuses. + // catches: IncrementalMinuteMetrics.fromJson accepting a finite negative rate. + group('a negative basal rate is refused', () { + final profile = energyProfiles['male']!; + + Map saved() { + final original = IncrementalMinuteMetrics(); + original.sync([0], [110], restingHr: 54, maxHr: 186, profile: profile); + return checkpoint(original.toJson()); + } + + test('restoring a checkpoint whose basalPerMinute is negative', () { + expect(() => IncrementalMinuteMetrics.fromJson(saved()), returnsNormally); + for (final bad in [-1.0, -1e-9, -1e300]) { + final json = saved()..['basalPerMinute'] = bad; + expect(() => IncrementalMinuteMetrics.fromJson(json), + throwsFormatException, + reason: 'basalPerMinute $bad'); + } + }); + + test('zero and a larger rate are still accepted', () { + for (final ok in [0.0, 1.2, 5.0]) { + final json = saved()..['basalPerMinute'] = ok; + // A bill that is not a rest or cadence minute needs no particular + // rate, so these restore. + expect(() => IncrementalMinuteMetrics.fromJson(json), returnsNormally, + reason: 'basalPerMinute $ok'); + } + }); + + test('no restored state can price a walking minute at a negative surplus', + () { + // The rate that used to be accepted, then the appended minute from the + // thread: with the check, restoration stops before any minute is priced. + final json = saved()..['basalPerMinute'] = -1.0; + expect(() => IncrementalMinuteMetrics.fromJson(json), throwsFormatException); + // A valid rate prices that minute at a non-negative surplus. + final state = IncrementalMinuteMetrics.fromJson(saved()); + final result = state.sync([0, 1], [110, 60], + cadenceSpm: [null, 110], + restingHr: 54, maxHr: 186, profile: profile); + for (final m in result.minutes!.minutes) { + if (m.abstained != null) continue; + if (m.active != null) expect(m.active, greaterThanOrEqualTo(0)); + if (m.walking != null) expect(m.walking, greaterThanOrEqualTo(0)); + } + }); + + test('a negative or non-finite basal override prices nothing', () { + for (final bad in [-1.0, -1e-9, double.nan, double.infinity, + double.negativeInfinity]) { + expect( + Calories.minuteEnergyPricer( + profile: profile, + hrmax: 186, + restingHr: 54, + basalKcalPerMin: bad), + isNull, + reason: 'override $bad'); + } + for (final ok in [0.0, 1.0, 1.4]) { + final pricer = Calories.minuteEnergyPricer( + profile: profile, hrmax: 186, restingHr: 54, basalKcalPerMin: ok); + expect(pricer, isNotNull, reason: 'override $ok'); + expect(pricer!.basalKcalPerMin, ok); + // The thread's minute: surplus is never below zero. + final bill = pricer.price(60, 110); + expect(bill.active, greaterThanOrEqualTo(0)); + expect(bill.walking, greaterThanOrEqualTo(0)); + } + }); + }); } diff --git a/test/properties/welch_stream_laws_test.dart b/test/properties/welch_stream_laws_test.dart index da95aa56..f567d93f 100644 --- a/test/properties/welch_stream_laws_test.dart +++ b/test/properties/welch_stream_laws_test.dart @@ -470,7 +470,8 @@ void _observe(_Ops ops, _Case c, void Function(String) bump) { ops.observe(c, bump); } -void _register(_Ops ops, {int maxN = 1200, int cases = 60}) { +void _register(_Ops ops, + {int maxN = 1200, int cases = 60, Duration? roundTripBudget}) { const pool = [0, 1, 15, 16, 19, 20, 59, 60, 61, 300, 700, 1200]; final shares = {..._sharesOf(ops), ...ops.shares}; final reach = Reach<_Case>(shares, (c, bump) => _observe(ops, c, bump)); @@ -505,6 +506,7 @@ void _register(_Ops ops, {int maxN = 1200, int cases = 60}) { examples: ops.forced, cases: cases, reach: noEval, + budget: roundTripBudget, ); _laws.law<(_Case, Mut)>( 'L3 ${ops.name}: another version or type, a missing, mistyped or ' @@ -1084,7 +1086,14 @@ class _CvhrOps extends _Ops { } void main() { - _register(_RsaOps()); + // L2 folds each night, writes real JSON text, restores, and goes on, for 60 + // generated cases plus the forced examples on nights up to 1,200 beats; RSA is + // the costly one (each evaluation re-runs Lomb-Scargle over the sub-windows). + // It took 5,073 ms against the 5,000 ms guard on the slow Dart 3.5.0 runner. + // The guard exists to stop a runaway, not to time a healthy run, so this law + // gets a larger explicit one. Nothing it checks changes: same 60 cases, same + // night lengths, same forced examples. + _register(_RsaOps(), roundTripBudget: const Duration(seconds: 20)); _register(_HrvOps()); _register(_CvhrOps());