diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index d8af739ec..fed9e8f95 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -2680,6 +2680,7 @@ class _BaselineHistoryCache { 'rhr', 'resp_rate', 'skin_temp_adc', + 'enmo_median_night', 'readiness', // The readiness composite z — later days' scores are calibrated against // its spread (`calibratedReadinessScore`). @@ -6889,6 +6890,19 @@ class DerivationEngine { sleepRrTsMs: sleepSub.rrTsMs, sleepRrMs: sleepSub.rrMs, sleepSkinTemp: sleepSub.skinTemp, + // Per-second ENMO over the sleep window (gravity vector magnitude − 1 g), + // for the hybrid HMM input. Absent accel (0,0,0 sentinel) -> 0.0, which + // the classifier reads as "no motion evidence", never "still". + sleepEnmo: [ + for (var i = 0; i < sleepSub.tsSec.length; i++) + sleepSub.accelPresentAt(i) + ? math.sqrt( + sleepSub.ax[i] * sleepSub.ax[i] + + sleepSub.ay[i] * sleepSub.ay[i] + + sleepSub.az[i] * sleepSub.az[i], + ) - 1.0 + : -1.0, // absent sentinel (CodeRabbit #756): measured zero motion is >= 0 + ], sleepJson: day.sleepJson, hypnoStages: day.hypnoStages, sleepOnsetSec: day.sleepOnsetSec, @@ -7759,6 +7773,10 @@ class DerivationEngine { // EVERY day (even during the bootstrap window where skin_temp_z is null) // so the baseline fills and z begins computing from ~day 4. 'skin_temp_adc': sc('skin_temp_adc'), + // The night's stable-core ENMO median, only for HMM-restaged nights + // (null otherwise -> no series row, history fills from HMM nights). + if (((bundle['sleep'] as Map?)?['stager_enmo_median'] as num?)?.toDouble() case final v?) + if (v > 0) 'enmo_median_night': v, 'dip_pct': sc('dip_pct'), // Headline 0–21 strain (for trend/sparkline); raw TRIMP kept too. // Written even when NULL: an abstaining day must overwrite a value a @@ -8383,6 +8401,8 @@ class DerivationEngine { // mismatch that left z permanently null. The raw mean is stored every day so // this series fills and z starts computing once ≥3 days exist. m['skin_temp_adc_history'] = before('skin_temp_adc'); + // Per-night stable-core ENMO medians (written only for HMM-staged nights). + m['enmo_history'] = before('enmo_median_night'); // TS-03/TS-04 — the observed ceiling this day's ZONES are banded on. A max, // not a window (see [maxBefore]), and strictly before today so a day is // never banded on a ceiling its own session set. diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 71d8545b7..210ba2f47 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -337,6 +337,13 @@ class DayBundleInput { final List sleepRrTsMs; final List sleepRrMs; final List sleepSkinTemp; + /// Per-second ENMO (g) over the sleep window — HYBRID HMM input. -1 means + /// ABSENT (accel sentinel (0,0,0)), never a still wrist; the classifier + /// treats absent as "no motion evidence" rather than "perfectly still". + final List sleepEnmo; + /// Trailing per-night ENMO medians (stable core), oldest first — the + /// persisted motion history for multi-night self-calibration. + final List enmoHistory; // ── the SINGLE-SOURCE sleep segmentation (JSON of SleepSegmentation) ────── final Map sleepJson; // {window,tst_sec,…,confidence} @@ -436,6 +443,8 @@ class DayBundleInput { required this.sleepRrTsMs, required this.sleepRrMs, required this.sleepSkinTemp, + this.sleepEnmo = const [], + this.enmoHistory = const [], required this.sleepJson, required this.hypnoStages, required this.sleepOnsetSec, @@ -471,6 +480,8 @@ class DayBundleInput { 'sleep_rr_ts_ms': sleepRrTsMs, 'sleep_rr_ms': sleepRrMs, 'sleep_skin_temp': sleepSkinTemp, + 'sleep_enmo': sleepEnmo, + 'enmo_history': enmoHistory, 'sleep_json': sleepJson, 'hypno_stages': hypnoStages, 'sleep_onset_sec': sleepOnsetSec, @@ -523,6 +534,8 @@ class DayBundleInput { sleepRrTsMs: dbls('sleep_rr_ts_ms'), sleepRrMs: dbls('sleep_rr_ms'), sleepSkinTemp: ints('sleep_skin_temp'), + sleepEnmo: dbls('sleep_enmo'), + enmoHistory: dbls('enmo_history'), sleepJson: ((m['sleep_json'] as Map?) ?? const {}) .cast(), hypnoStages: strs('hypno_stages'), @@ -561,7 +574,7 @@ class DayBundleInput { /// efficiency. Figures `segmentSleep` derives that edge does not recompute /// here are dropped rather than left describing the old labels. Null when /// the labels hold no sleep at all: the caller keeps the original night. - DayBundleInput? withHypnoStages(List stages) { + DayBundleInput? withHypnoStages(List stages, {bool atypical = false, double? enmoMedian}) { var light = 0, deep = 0, rem = 0, wake = 0, unobserved = 0; var firstSleep = -1, lastSleep = -1; for (var i = 0; i < stages.length; i++) { @@ -606,7 +619,18 @@ class DayBundleInput { ..['rem_sec'] = rem ..['wake_sec'] = wake ..['efficiency_pct'] = observedSec > 0 ? 100.0 * tst / observedSec : null - ..['stager_source'] = 'hmm'; + ..['stager_source'] = 'hmm' + // Multi-night calibration flag: tonight's baseline deviated >2 robust z + // from the user's long-term baseline (illness, alcohol, travel...) — + // the staging is still published, but consumers can treat it as lower + // confidence. + ..['stager_atypical_night'] = atypical; + if (enmoMedian != null && enmoMedian > 0) { + // Persisted as the night's motion baseline -> metric_series + // 'enmo_median_night', feeding later nights' 7d/long-term motion + // calibration (enmo_history). + json['stager_enmo_median'] = enmoMedian; + } return DayBundleInput( date: date, dayTsSec: dayTsSec, @@ -618,6 +642,8 @@ class DayBundleInput { sleepRrTsMs: sleepRrTsMs, sleepRrMs: sleepRrMs, sleepSkinTemp: sleepSkinTemp, + sleepEnmo: sleepEnmo, + enmoHistory: enmoHistory, sleepJson: json, hypnoStages: stages, sleepOnsetSec: sleepOnsetSec, @@ -678,7 +704,10 @@ Map _deriveDayBundle(Map inputJson, // from the final labels and the night is tagged `stager_source: hmm`. if (d.useHmmStages && d.hypnoStages.isNotEmpty) { final restaged = _hmmRestage(d); - if (restaged != null) d = d.withHypnoStages(restaged) ?? d; + if (restaged != null) { + d = d.withHypnoStages(restaged.labels, + atypical: restaged.atypicalNight, enmoMedian: restaged.enmoMedian) ?? d; + } } // ── HR over the DAY (for the curve, strain zones, dip day-side) ─────────── @@ -1566,7 +1595,17 @@ Map _deriveDayBundle(Map inputJson, 'absence_reason': absenceReason, // Present only when the experimental re-staging labelled this night (PR // #628), so a night of the shipped stager is stored exactly as before. - if (d.sleepJson['stager_source'] == 'hmm') 'stager_source': 'hmm', + if (d.sleepJson['stager_source'] == 'hmm') ...{ + 'stager_source': 'hmm', + // Forward the atypical-night (outlier) flag alongside, so the persisted + // night carries the calibration confidence, not just the source tag. + if (d.sleepJson['stager_atypical_night'] == true) + 'stager_atypical_night': true, + // The night's stable-core ENMO median, persisted to metric_series as + // 'enmo_median_night' for multi-night motion calibration. + if ((d.sleepJson['stager_enmo_median'] as num?)?.toDouble() case final v?) + if (v > 0) 'stager_enmo_median': v, + }, // Seconds the band's own last SLEEP cut off the end of the auto window; null = rule did not apply. Provenance, never a metric. 'band_offset_trim_sec': bandOffsetTrimSec, 'stager': _envelope( @@ -2591,13 +2630,30 @@ List _rmssdPerSecond( /// later label. Within the night the decoder abstains per 30 s epoch: an /// epoch with no plausible heart rate or no RMSSD keeps its shipped label, /// and an unobserved second stays unobserved. -List? _hmmRestage(DayBundleInput d) { +/// Result of a restage: the merged labels plus whether the night's baseline +/// deviated strongly from the user's long-term baseline (atypical night). +class HmmRestageResult { + final List labels; + final bool atypicalNight; + final double? enmoMedian; + const HmmRestageResult(this.labels, this.atypicalNight, [this.enmoMedian]); +} + +HmmRestageResult? _hmmRestage(DayBundleInput d) { final n = d.hypnoStages.length; final t0 = d.sleepOnsetSec; + var atypical = false; + double? enmoMedian; final hrAt = List.filled(n, 0.0); + final enmoAt = List.filled(n, 0.0); + final tempAt = List.filled(n, 0.0); for (var i = 0; i < d.sleepTsSec.length && i < d.sleepHr.length; i++) { final k = d.sleepTsSec[i] - t0; - if (k >= 0 && k < n) hrAt[k] = d.sleepHr[i].toDouble(); + if (k >= 0 && k < n) { + hrAt[k] = d.sleepHr[i].toDouble(); + if (i < d.sleepEnmo.length) enmoAt[k] = d.sleepEnmo[i]; + if (i < d.sleepSkinTemp.length) tempAt[k] = d.sleepSkinTemp[i].toDouble(); + } } final corrected = correctRr(d.sleepRrMs, rrTsMs: d.sleepRrTsMs); // nnTimesMs is relative to the start of the first input beat; put it back @@ -2612,14 +2668,30 @@ List? _hmmRestage(DayBundleInput d) { [for (var i = 0; i < n; i++) t0 + i], windowSec: 60, ); - final labels = _classifySleepStagesHMM([ - for (var i = 0; i < n; i++) - { - 'ts': t0 + i, - 'hr': d.hypnoStages[i] == 'unobserved' ? 0.0 : hrAt[i], - 'rmssd': d.hypnoStages[i] == 'unobserved' ? 0.0 : rmssdAt[i], - }, - ]); + final labels = _classifySleepStagesHMM( + [ + for (var i = 0; i < n; i++) + { + 'ts': t0 + i, + 'hr': d.hypnoStages[i] == 'unobserved' ? 0.0 : hrAt[i], + 'rmssd': d.hypnoStages[i] == 'unobserved' ? 0.0 : rmssdAt[i], + // HYBRID: motion + skin temp per second (0 = absent sentinel). + 'enmo': enmoAt[i], + 'skin_temp': tempAt[i], + }, + ], + // Multi-night self-calibration: 7d rolling + long-term baselines from the + // persisted nightly histories (HR, RMSSD, skin temp) feed the blended + // emission baseline and the atypical-night (outlier) flag. + rhrHistory: d.rhrHistory, + rmssdHistory: d.rmssdHistory, + skinTempHistory: d.skinTempAdcHistory, + enmoHistory: d.enmoHistory, + onAnalyzer: (a) { + atypical = a.atypicalNight; + enmoMedian = a.enmoMedianNight; + }, + ); if (labels == null) return null; final merged = [ for (var i = 0; i < n; i++) @@ -2638,14 +2710,48 @@ List? _hmmRestage(DayBundleInput d) { } prev = merged[i]; } - return merged; + // AASM-aligned wake post-processing on the merged per-second labels: + // - a wake run shorter than 15 s is an AROUSAL, not wake (AASM: arousal + // >= 3 s, wake epoch needs > 50% of the 30 s epoch awake) -> the seconds + // revert to the surrounding sleep stage; + // - after a REAL (>= 15 s) wake phase the following sleep does NOT restart + // from scratch — staging continues where it left off (AASM scores + // sequentially, no reset), which the Viterbi path already respects; long + // wake runs simply stay wake and count towards WASO. + const minWakeRunSec = 15; + var i = 0; + while (i < merged.length) { + if (merged[i] != 'wake' || merged[i] == 'unobserved') { + i++; + continue; + } + var j = i; + while (j < merged.length && merged[j] == 'wake') { + j++; + } + if (j - i < minWakeRunSec) { + final surrounding = i > 0 ? merged[i - 1] : 'light'; + for (var k = i; k < j; k++) { + merged[k] = surrounding == 'unobserved' ? 'light' : surrounding; + } + } + i = j; + } + return HmmRestageResult(merged, atypical, enmoMedian); } /// The Viterbi decode over 30 s epochs of [epochData] (per-second records /// with `ts`, `hr`, `rmssd`). Null when the whole night abstains (see /// [_hmmRestage]); otherwise one label per record, null where its epoch had /// no usable heart rate or RMSSD. -List? _classifySleepStagesHMM(List> epochData) { +List? _classifySleepStagesHMM( + List> epochData, { + List rhrHistory = const [], + List rmssdHistory = const [], + List skinTempHistory = const [], + List enmoHistory = const [], + void Function(SleepStageAnalyzer)? onAnalyzer, +}) { if (epochData.isEmpty) return null; final analyzer = SleepStageAnalyzer(); // Baseline from the first 120 minutes BY ELAPSED TIME, not the first 240 @@ -2655,6 +2761,19 @@ List? _classifySleepStagesHMM(List> epochData) { .where((e) => ((e['ts'] as num?)?.toDouble() ?? 0.0) - t0 <= 120 * 60) .toList(); analyzer.computeBaseline(first120m); + // Motion baseline from the stable core of the night (middle 50 %), not the + // opening window — settling-in movement would inflate the person's normal + // sleep-motion level and push the wake threshold too high. + analyzer.computeMotionBaseline(epochData); + // THREE-LEVEL calibration: blend in the 7-day rolling and long-term + // baselines from the persisted nightly histories. + analyzer.computeMultiNightBaselines( + rhrHistory: rhrHistory, + rmssdHistory: rmssdHistory, + skinTempHistory: skinTempHistory, + enmoHistory: enmoHistory, + ); + onAnalyzer?.call(analyzer); // No baseline (RR that only starts after the first 120 minutes, say) makes // every emission neutral, and wake's self-transition then labels a measured // night as wake. Keep the shipped labels instead. @@ -2680,11 +2799,15 @@ List? _classifySleepStagesHMM(List> epochData) { final idx = {}; for (var i = 0; i < epochData.length; i++) { final k = idx.putIfAbsent(epochOf[i], () { - epochs.add({'hr': 0.0, 'rmssd': 0.0, 'nh': 0, 'nr': 0}); + // HYBRID: epochs also average ENMO and skin temp (0 sentinel = absent, + // excluded from the mean, never counted as "still"/"cold"). + epochs.add({'hr': 0.0, 'rmssd': 0.0, 'enmo': 0.0, 'skin_temp': 0.0, 'nh': 0, 'nr': 0, 'ne': 0, 'nt': 0}); return epochs.length - 1; }); final hr = (epochData[i]['hr'] as num?)?.toDouble() ?? 0.0; final rmssd = (epochData[i]['rmssd'] as num?)?.toDouble() ?? 0.0; + final enmo = (epochData[i]['enmo'] as num?)?.toDouble() ?? 0.0; + final temp = (epochData[i]['skin_temp'] as num?)?.toDouble() ?? 0.0; final ep = epochs[k]; if (hr > 30.0 && hr < 200.0) { ep['hr'] = (ep['hr'] as double) + hr; @@ -2694,11 +2817,22 @@ List? _classifySleepStagesHMM(List> epochData) { ep['rmssd'] = (ep['rmssd'] as double) + rmssd; ep['nr'] = (ep['nr'] as int) + 1; } + if (enmo >= 0.0) { // measured (incl. real zero motion); -1 = absent + ep['enmo'] = (ep['enmo'] as double) + enmo; + ep['ne'] = (ep['ne'] as int) + 1; + } + if (temp > 0.0) { + ep['skin_temp'] = (ep['skin_temp'] as double) + temp; + ep['nt'] = (ep['nt'] as int) + 1; + } } for (final ep in epochs) { final nh = ep['nh'] as int, nr = ep['nr'] as int; + final ne = ep['ne'] as int, nt = ep['nt'] as int; ep['hr'] = nh == 0 ? 0.0 : (ep['hr'] as double) / nh; ep['rmssd'] = nr == 0 ? 0.0 : (ep['rmssd'] as double) / nr; + ep['enmo'] = ne == 0 ? 0.0 : (ep['enmo'] as double) / ne; + ep['skin_temp'] = nt == 0 ? 0.0 : (ep['skin_temp'] as double) / nt; } final states = analyzer.viterbiDecode(epochs); if (states.length != epochs.length) return null; diff --git a/lib/compute/sleep_pipeline_hmm.dart b/lib/compute/sleep_pipeline_hmm.dart index c09826f8d..b7d1e1799 100644 --- a/lib/compute/sleep_pipeline_hmm.dart +++ b/lib/compute/sleep_pipeline_hmm.dart @@ -34,26 +34,26 @@ class HMMConfig { SleepState.rem: log(0.005) }, SleepState.nremLight: { - SleepState.wake: log(0.05), - SleepState.nremLight: log(0.80), + SleepState.wake: log(0.12), + SleepState.nremLight: log(0.73), SleepState.nremDeep: log(0.10), SleepState.rem: log(0.05) }, SleepState.nremDeep: { - SleepState.wake: log(0.02), - SleepState.nremLight: log(0.15), + SleepState.wake: log(0.06), + SleepState.nremLight: log(0.11), // rows must sum to 1.0 (CodeRabbit #756) SleepState.nremDeep: log(0.80), SleepState.rem: log(0.03) }, SleepState.rem: { - SleepState.wake: log(0.05), + SleepState.wake: log(0.10), SleepState.nremLight: log(0.10), // INTENTIONAL HARD FORBID: REM never goes straight to deep (it passes // through light). log(0) is -infinity; the decoder only ever ADDS it to // finite emissions and compares, so it rules the move out without ever // producing NaN. SleepState.nremDeep: double.negativeInfinity, - SleepState.rem: log(0.85) + SleepState.rem: log(0.80) // rows must sum to 1.0 (CodeRabbit #756) }, }; } @@ -61,6 +61,100 @@ class HMMConfig { class SleepStageAnalyzer { double? hrMinNight; double? rmssdMeanNight; + // HYBRID baselines: motion (ENMO, g) and skin temp (centi-°C) medians from + // the opening window. 0 = absent sentinel and is EXCLUDED — a missing sensor + // must never set the baseline to "perfectly still" / "cold". + double? enmoMedianNight; + double? skinTempMedianNight; + + // THREE-LEVEL self-calibration (per user, across nights): + // - nightly baseline (computeBaseline, first 120 min) — primary, always there + // - 7-day rolling baseline — smooths single restless nights out + // - long-term baseline — the user's personal reference; a nightly baseline + // that deviates strongly from it marks the night as ATYPICAL (illness, + // alcohol, travel) instead of silently bending the thresholds. + double? hrMin7d; + double? rmssdMean7d; + double? hrMinLong; + double? rmssdMeanLong; + /// True when tonight's baseline deviates from the long-term baseline by more + /// than [atypicalZThreshold] robust z — an outlier night, lower confidence. + bool atypicalNight = false; + static const double atypicalZThreshold = 2.0; + + /// Feed the multi-night histories (trailing nightly series, oldest first). + /// [rhrHistory]/[rmssdHistory] are the persisted per-night resting-HR / + /// nocturnal-RMSSD series; the last 7 form the rolling baseline, all of them + /// the long-term one. Missing/empty history simply leaves the level unset. + void computeMultiNightBaselines({ + List rhrHistory = const [], + List rmssdHistory = const [], + List skinTempHistory = const [], + List enmoHistory = const [], + }) { + double? medianOf(List xs) { + final v = xs.where((x) => x.isFinite && x > 0).toList()..sort(); + if (v.isEmpty) return null; + return v[v.length ~/ 2]; + } + double? madOf(List xs, double? med) { + if (med == null) return null; + final dev = xs.where((x) => x.isFinite).map((x) => (x - med).abs()).toList()..sort(); + if (dev.isEmpty) return null; + return dev[dev.length ~/ 2]; + } + List last7(List xs) => xs.length > 7 ? xs.sublist(xs.length - 7) : xs; + hrMin7d = medianOf(last7(rhrHistory)); + rmssdMean7d = medianOf(last7(rmssdHistory)); + hrMinLong = medianOf(rhrHistory); + rmssdMeanLong = medianOf(rmssdHistory); + // Motion + thermal channels self-calibrate too when a nightly history + // exists (skinTempAdcHistory); without one they stay nightly-only. + skinTempMedian7d = medianOf(last7(skinTempHistory)); + skinTempMedianLong = medianOf(skinTempHistory); + // Motion history: persisted per-night ENMO medians (stable-core), so the + // motion channel self-calibrates across nights like the others. Absent + // history (pre-feature nights) simply leaves the levels unset. + enmoMedian7d = medianOfEnmo(last7(enmoHistory)); + enmoMedianLong = medianOfEnmo(enmoHistory); + // Outlier check: nightly vs long-term, robust z via MAD (1.4826*MAD ~ sd). + if (hrMinNight != null && hrMinLong != null) { + final mad = madOf(rhrHistory, hrMinLong); + final sd = mad != null && mad > 0 ? 1.4826 * mad : 3.0; // floor: 3 bpm + final z = (hrMinNight! - hrMinLong!) / sd; + if (z.abs() > atypicalZThreshold) atypicalNight = true; + } + } + + double? skinTempMedian7d; + double? skinTempMedianLong; + double? enmoMedian7d; + double? enmoMedianLong; + // ENMO medians use >= 0 as valid (a measured still night is a real 0); + // the -1 absent sentinel is excluded. + double? medianOfEnmo(List xs) { + final v = xs.where((x) => x.isFinite && x >= 0).toList()..sort(); + if (v.isEmpty) return null; + return v[v.length ~/ 2]; + } + /// Effective thermal baseline: blend nightly with 7d rolling when both exist. + double? get effectiveSkinTempMedian => + skinTempMedian7d != null && skinTempMedianNight != null + ? (skinTempMedianNight! + skinTempMedian7d!) / 2 + : (skinTempMedianNight ?? skinTempMedian7d); + /// Effective motion baseline: blend nightly (stable core) with 7d rolling. + double? get effectiveEnmoMedian => + enmoMedian7d != null && enmoMedianNight != null + ? (enmoMedianNight! + enmoMedian7d!) / 2 + : (enmoMedianNight ?? enmoMedian7d); + + /// Effective HR baseline for the emission: blend nightly with the 7-day + /// rolling level when both exist (half/half) — the night's own physiology + /// anchors it, the rolling window keeps single odd nights from dominating. + double? get effectiveHrMin => + hrMin7d != null && hrMinNight != null ? (hrMinNight! + hrMin7d!) / 2 : (hrMinNight ?? hrMin7d); + double? get effectiveRmssdMean => + rmssdMean7d != null && rmssdMeanNight != null ? (rmssdMeanNight! + rmssdMean7d!) / 2 : (rmssdMeanNight ?? rmssdMean7d); // The same plausibility window decides which samples form the baseline and // which observations get scored. A value outside it (a 250 bpm spike, a NaN @@ -68,6 +162,8 @@ class SleepStageAnalyzer { // would push the decoder towards wake. static bool _plausibleHr(double? h) => h != null && h.isFinite && h > 30.0 && h < 200.0; static bool _plausibleRmssd(double? r) => r != null && r.isFinite && r > 0.0 && r < 500.0; + static double _sigmoid(double x, double center, double scale) => + 1.0 / (1.0 + exp(-(x - center) / scale)); void computeBaseline(List> first120minData) { if (first120minData.isEmpty) return; @@ -81,41 +177,141 @@ class SleepStageAnalyzer { .where(_plausibleRmssd) .cast() .toList(); - + final enmos = first120minData + .map((d) => (d['enmo'] as num?)?.toDouble() ?? -1.0) + .where((e) => e.isFinite && e >= 0.0) + .toList(); + final temps = first120minData + .map((d) => (d['skin_temp'] as num?)?.toDouble() ?? 0.0) + .where((t) => t.isFinite && t > 0.0) + .toList(); + if (hrs.isNotEmpty) { hrs.sort(); hrMinNight = hrs[(hrs.length * 0.1).floor()]; } if (rmssds.isNotEmpty) rmssdMeanNight = rmssds.reduce((a, b) => a + b) / rmssds.length; + if (enmos.isNotEmpty) { + enmos.sort(); + enmoMedianNight = enmos[enmos.length ~/ 2]; + } + if (temps.isNotEmpty) { + temps.sort(); + skinTempMedianNight = temps[temps.length ~/ 2]; + } + } + + /// Motion (and thermal) baseline from the STABLE CORE of the night — the + /// middle 50 % by elapsed time — NOT the first 120 min. The opening window + /// contains settling-in / turning over and is systematically more active, so + /// a median taken from it would overstate the person's normal sleep motion + /// and the wake-vs-arousal threshold would sit too high. HR/RMSSD keep the + /// opening-window baseline (autonomic settling is exactly what we want there). + /// Missing ENMO (0 sentinel) is excluded — never "still". + void computeMotionBaseline(List> nightData) { + if (nightData.length < 10) return; + final t0 = (nightData.first['ts'] as num?)?.toDouble() ?? 0.0; + final t1 = (nightData.last['ts'] as num?)?.toDouble() ?? t0; + final span = t1 - t0; + if (span <= 0) return; + final core = nightData + .where((e) => ((e['ts'] as num?)?.toDouble() ?? 0.0) - t0 > span * 0.25 && + ((e['ts'] as num?)?.toDouble() ?? 0.0) - t0 < span * 0.75) + .toList(); + if (core.isEmpty) return; + final enmos = core + .map((d) => (d['enmo'] as num?)?.toDouble() ?? -1.0) + .where((e) => e.isFinite && e >= 0.0) + .toList(); + if (enmos.isNotEmpty) { + enmos.sort(); + enmoMedianNight = enmos[enmos.length ~/ 2]; + } } - double _getEmissionLogProb(SleepState state, double hr, double rmssd, {bool isContactLost = false}) { + /// HYBRID emission over ALL available channels: HR, RMSSD, ENMO (motion), + /// skin temp. Absent channels (0 sentinel) contribute a NEUTRAL factor — + /// never evidence. Wake is boosted by ANY of: elevated HR + low RMSSD + /// (autonomic arousal), high movement, warm skin (peripheral vasodilation + /// on arousal); sleep states are penalised by movement. + double _getEmissionLogProb(SleepState state, double hr, double rmssd, { + double enmo = 0.0, + double skinTemp = 0.0, + bool isContactLost = false, + }) { if (isContactLost || - !_plausibleHr(hr) || - !_plausibleRmssd(rmssd) || - hrMinNight == null || - rmssdMeanNight == null) { + effectiveHrMin == null || + effectiveRmssdMean == null) { return log(0.25); } + if (!_plausibleHr(hr) || !_plausibleRmssd(rmssd)) { + // HR/RR unusable: motion-only evidence. A clearly moving wrist cannot + // be deep sleep. If motion is ALSO absent (0 sentinel), the epoch is + // fully unmeasured — return the uniform neutral for every state so the + // gap cannot bias adjacent measured epochs towards wake (CodeRabbit #756). + if (enmo < 0) return log(0.25); // absent sentinel: fully unmeasured epoch + final motionWake = _sigmoid(enmo, 0.08, 0.03); + switch (state) { + case SleepState.wake: + return log(max(0.01, 0.2 + 0.8 * motionWake)); + case SleepState.nremDeep: + return log(max(0.01, 0.1 * (1.0 - motionWake))); + case SleepState.rem: + return log(max(0.01, 0.15 * (1.0 - motionWake))); + case SleepState.nremLight: + return log(max(0.01, 0.3)); + } + } - final hrNorm = hr / hrMinNight!; - final rmssdNorm = rmssd / rmssdMeanNight!; - - double sigmoid(double x, double center, double scale) => 1.0 / (1.0 + exp(-(x - center) / scale)); + final hrNorm = hr / effectiveHrMin!; + final rmssdNorm = rmssd / effectiveRmssdMean!; + // Motion relative to the night's own median (robust for restless + // sleepers). Absent ENMO (0 sentinel) = NO evidence either way: wake + // factor 0 (does not boost wake), sleep factor 1.0 (does not penalise). + final motionMeasured = enmo >= 0 && effectiveEnmoMedian != null && effectiveEnmoMedian! > 0; + final motion = motionMeasured ? enmo / effectiveEnmoMedian! : 1.0; + final motionWake = motionMeasured ? _sigmoid(motion, 3.0, 1.5) : 0.0; + final motionSleep = motionMeasured ? 1.0 - _sigmoid(motion, 2.0, 1.0) : 1.0; + // Skin temp vs the night median, normalised so ~100 centi-°C = 1 °C. + // Absent temp = no thermal evidence (excluded from the noisy-OR, i.e. + // factor 0, never the 0.5 that would silently boost wake) — CodeRabbit #756. + final tempMeasured = skinTemp > 0 && effectiveSkinTempMedian != null; + final tempWake = tempMeasured + ? _sigmoid((skinTemp - effectiveSkinTempMedian!) / 100.0, 1.5, 1.0) + : 0.0; double prob; switch (state) { case SleepState.wake: - prob = sigmoid(hrNorm, 1.2, 0.05) * (1.0 - sigmoid(rmssdNorm, 0.8, 0.05)); + // Noisy-OR over autonomic, motion and thermal arousal cues — any one + // strong cue can carry wake, so brief awakenings with movement but + // barely-elevated HR are still caught. + final autonomic = + _sigmoid(hrNorm, 1.15, 0.05) * (1.0 - _sigmoid(rmssdNorm, 0.8, 0.05)); + // Weak HR-only bonus for "quiet wakefulness" (issue #593 pattern: + // lying still, HR clearly above the personal sleep baseline, RMSSD + // barely moved). Capped at 0.3 and centred high (1.22×) so REM — + // whose HR/HRV is wake-like (Zemaityte et al.) — is not flipped to + // wake; REM's own emission + self-transition guard it instead. + final hrOnlyWake = _sigmoid(hrNorm, 1.22, 0.07) * 0.3; + prob = 1.0 - + (1.0 - autonomic) * + (1.0 - motionWake) * + (1.0 - tempWake) * + (1.0 - hrOnlyWake); break; case SleepState.nremDeep: - prob = (1.0 - sigmoid(hrNorm, 1.0, 0.05)) * sigmoid(rmssdNorm, 1.1, 0.05); + prob = (1.0 - _sigmoid(hrNorm, 1.0, 0.05)) * + _sigmoid(rmssdNorm, 1.1, 0.05) * + motionSleep; break; case SleepState.rem: - prob = sigmoid(hrNorm, 1.05, 0.05) * sigmoid(rmssdNorm, 1.0, 0.05); + prob = _sigmoid(hrNorm, 1.05, 0.05) * + _sigmoid(rmssdNorm, 1.0, 0.05) * + (0.5 + 0.5 * motionSleep); break; case SleepState.nremLight: - prob = 0.4; + prob = 0.4 * (0.5 + 0.5 * motionSleep); break; } return log(max(0.01, prob)); @@ -138,6 +334,8 @@ class SleepStageAnalyzer { final initProb = (state == SleepState.wake || state == SleepState.nremLight) ? log(0.5) : double.negativeInfinity; viterbi[0][i] = initProb + _getEmissionLogProb(state, hr, rmssd, + enmo: (nightData[0]['enmo'] as num?)?.toDouble() ?? 0.0, + skinTemp: (nightData[0]['skin_temp'] as num?)?.toDouble() ?? 0.0, isContactLost: nightData[0]['isContactLost'] == true); } @@ -145,10 +343,13 @@ class SleepStageAnalyzer { final hr = (nightData[t]['hr'] as num?)?.toDouble() ?? 0.0; final rmssd = (nightData[t]['rmssd'] as num?)?.toDouble() ?? 0.0; final contactLost = nightData[t]['isContactLost'] == true; + final enmo = (nightData[t]['enmo'] as num?)?.toDouble() ?? 0.0; + final skinTemp = (nightData[t]['skin_temp'] as num?)?.toDouble() ?? 0.0; for (var j = 0; j < N; j++) { final nextState = states[j]; final emission = - _getEmissionLogProb(nextState, hr, rmssd, isContactLost: contactLost); + _getEmissionLogProb(nextState, hr, rmssd, + enmo: enmo, skinTemp: skinTemp, isContactLost: contactLost); var maxLogProb = double.negativeInfinity; var bestPrevState = 0; for (var i = 0; i < N; i++) { diff --git a/test/sleep_hmm_aasm_wake_test.dart b/test/sleep_hmm_aasm_wake_test.dart new file mode 100644 index 000000000..02e762126 --- /dev/null +++ b/test/sleep_hmm_aasm_wake_test.dart @@ -0,0 +1,90 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/sleep_pipeline_hmm.dart'; + +Map epoch(double hr, double rmssd, {double enmo = 0.0, double skinTemp = 0.0}) => + {'hr': hr, 'rmssd': rmssd, 'enmo': enmo, 'skin_temp': skinTemp}; + +SleepStageAnalyzer baselined() { + final a = SleepStageAnalyzer(); + a.computeBaseline(List.generate(7200, (_) => epoch(55, 45, enmo: 0.01, skinTemp: 3350))); + return a; +} + +/// Mirror of the AASM 15 s minimum-wake-run rule applied in _hmmRestage. +List applyAasmWakeRule(List labels, {int minWakeRunSec = 15}) { + final merged = [...labels]; + var i = 0; + while (i < merged.length) { + if (merged[i] != 'wake') { + i++; + continue; + } + var j = i; + while (j < merged.length && merged[j] == 'wake') { + j++; + } + if (j - i < minWakeRunSec) { + final surrounding = i > 0 ? merged[i - 1] : 'light'; + for (var k = i; k < j; k++) { + merged[k] = surrounding == 'unobserved' ? 'light' : surrounding; + } + } + i = j; + } + return merged; +} + +void main() { + test('AASM: short wake run (<15 s) is an arousal, reverts to sleep stage', () { + // 10 s of wake between deep phases must NOT stay wake (AASM: arousal + // >=3 s but < half a 30 s epoch is not stage W). + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), // deep + ...List.generate(10, (_) => epoch(70, 25, enmo: 0.3, skinTemp: 3450)), // brief arousal + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), // deep + ]; + final states = a.viterbiDecode(night); + final labels = states.map((s) => switch (s) { + SleepState.wake => 'wake', + SleepState.nremLight => 'light', + SleepState.nremDeep => 'deep', + SleepState.rem => 'rem', + }).toList(); + final cleaned = applyAasmWakeRule(labels); + expect(cleaned.sublist(30, 40).where((l) => l == 'wake').length, lessThan(10), + reason: 'a 10 s wake burst is an arousal, not wake'); + }); + + test('AASM: long wake run (>=15 s) stays wake', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), + ...List.generate(60, (_) => epoch(85, 15, enmo: 0.4, skinTemp: 3500)), // real awakening + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + final labels = states.map((s) => switch (s) { + SleepState.wake => 'wake', + SleepState.nremLight => 'light', + SleepState.nremDeep => 'deep', + SleepState.rem => 'rem', + }).toList(); + final cleaned = applyAasmWakeRule(labels); + expect(cleaned.sublist(30, 90).where((l) => l == 'wake').length, greaterThan(30), + reason: 'a sustained awakening must remain wake (counts to WASO)'); + }); + + test('AASM: staging continues after wake — no restart (deep resumes as deep)', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), + ...List.generate(30, (_) => epoch(85, 15, enmo: 0.4, skinTemp: 3500)), + ...List.generate(30, (_) => epoch(53, 50, enmo: 0.005, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + // After the wake block, the deep-sleep signature must decode as deep again + // (Viterbi continuity), not be forced back to light/wake. + expect(states[75], SleepState.nremDeep); + }); +} diff --git a/test/sleep_hmm_hybrid_test.dart b/test/sleep_hmm_hybrid_test.dart new file mode 100644 index 000000000..99f7f9bbb --- /dev/null +++ b/test/sleep_hmm_hybrid_test.dart @@ -0,0 +1,117 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/sleep_pipeline_hmm.dart'; + +Map epoch(double hr, double rmssd, {double enmo = 0.0, double skinTemp = 0.0}) => + {'hr': hr, 'rmssd': rmssd, 'enmo': enmo, 'skin_temp': skinTemp}; + +SleepStageAnalyzer baselined() { + final a = SleepStageAnalyzer(); + // 120 min opening window: calm sleep baseline, HR ~55, RMSSD ~45, + // low motion (enmo ~0.01), skin temp ~3350 (33.5 °C). + a.computeBaseline(List.generate(7200, (_) => epoch(55, 45, enmo: 0.01, skinTemp: 3350))); + return a; +} + +void main() { + test('hybrid: movement-only arousal (HR barely up) is detected as wake', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + // 60 s of tossing/turning: HR only +3, but ENMO clearly elevated. + ...List.generate(60, (_) => epoch(58, 40, enmo: 0.25, skinTemp: 3400)), + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + final wakeDuringArousal = states.sublist(30, 90).where((s) => s == SleepState.wake).length; + expect(wakeDuringArousal, greaterThan(20), + reason: 'motion-only arousal should yield wake, not light'); + }); + + test('hybrid: quiet sleep stays sleep even with normal HR/RMSSD', () { + final a = baselined(); + final night = List.generate(120, (_) => epoch(56, 44, enmo: 0.008, skinTemp: 3345)); + final states = a.viterbiDecode(night); + expect(states.where((s) => s == SleepState.wake).length, lessThan(10)); + }); + + test('hybrid: high HR arousal is still wake', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ...List.generate(60, (_) => epoch(85, 18, enmo: 0.02, skinTemp: 3350)), + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + expect(states.sublist(30, 90).where((s) => s == SleepState.wake).length, greaterThan(20)); + }); + + test('hybrid: absent ENMO (0 sentinel) is neutral, never "still"', () { + final a = baselined(); + // enmo 0 everywhere: baseline has no motion median -> motion factor neutral. + final night = List.generate(60, (_) => epoch(54, 46, enmo: -1.0, skinTemp: 3350)); // absent sentinel + final states = a.viterbiDecode(night); + expect(states.where((s) => s == SleepState.wake).length, lessThan(10), + reason: 'absent motion must not be read as evidence either way'); + }); + + test('hybrid: unusable HR falls back to motion-only evidence', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(0, 0, enmo: -1.0)), + ...List.generate(60, (_) => epoch(0, 0, enmo: 0.3)), // moving, no HR + ...List.generate(30, (_) => epoch(0, 0, enmo: -1.0)), + ]; + final states = a.viterbiDecode(night); + expect(states.sublist(30, 90).where((s) => s == SleepState.wake).length, greaterThan(20)); + }); + + test('neutral: fully missing epoch (no HR/RR, absent ENMO) is uniform, not wake-biased', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ...List.generate(60, (_) => epoch(0, 0, enmo: -1.0)), // gap: nothing measured + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.01, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + // The gap must not drag the surrounding sleep into wake. + expect(states.where((s) => s == SleepState.wake).length, lessThan(10), + reason: 'a fully unmeasured stretch must stay neutral, never wake'); + }); + + test('neutral: absent temp/motion do not add wake evidence (noisy-OR excludes them)', () { + final a = baselined(); + // same HR/RMSSD, but optional sensors absent -> wake prob must not exceed + // the case where they are present-and-calm + final withSensors = a.viterbiDecode(List.generate(60, (_) => epoch(60, 50, enmo: 0.01, skinTemp: 3350))); + final withoutSensors = a.viterbiDecode(List.generate(60, (_) => epoch(60, 50, enmo: -1.0, skinTemp: 0))); + final wakeWith = withSensors.where((s) => s == SleepState.wake).length; + final wakeWithout = withoutSensors.where((s) => s == SleepState.wake).length; + expect(wakeWithout, lessThanOrEqualTo(wakeWith), + reason: 'absent optional sensors must be neutral, never extra wake evidence'); + }); + + test('hr-only bonus: quiet wakefulness (still, HR +14 bpm over baseline) -> wake', () { + final a = baselined(); // baseline HR 55 + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.005, skinTemp: 3350)), + // lying still and awake: HR 69 (~1.25x), RMSSD near normal, no motion + ...List.generate(60, (_) => epoch(69, 44, enmo: 0.002, skinTemp: 3350)), + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.005, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + expect(states.sublist(30, 90).where((s) => s == SleepState.wake).length, greaterThan(20), + reason: 'still but clearly elevated HR must read as wake (quiet wakefulness)'); + }); + + test('hr-only bonus guarded: REM-like (HR 1.22x, still, RMSSD normal) stays non-wake', () { + final a = baselined(); + final night = >[ + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.005, skinTemp: 3350)), + ...List.generate(60, (_) => epoch(67, 45, enmo: 0.004, skinTemp: 3350)), // REM-like HR spike, still + ...List.generate(30, (_) => epoch(54, 46, enmo: 0.005, skinTemp: 3350)), + ]; + final states = a.viterbiDecode(night); + expect(states.sublist(30, 90).where((s) => s == SleepState.wake).length, lessThan(30), + reason: 'REM-like HR with atonia must not flip to wake (REM guard)'); + }); +} diff --git a/test/sleep_hmm_motion_baseline_test.dart b/test/sleep_hmm_motion_baseline_test.dart new file mode 100644 index 000000000..e634b1ee7 --- /dev/null +++ b/test/sleep_hmm_motion_baseline_test.dart @@ -0,0 +1,30 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/sleep_pipeline_hmm.dart'; + +Map epoch(double hr, double rmssd, double ts, {double enmo = 0.0, double skinTemp = 0.0}) => + {'hr': hr, 'rmssd': rmssd, 'ts': ts, 'enmo': enmo, 'skin_temp': skinTemp}; + +void main() { + test('motion baseline uses the stable core, not the restless opening', () { + final a = SleepStageAnalyzer(); + // 8 h night: first 2 h restless (enmo 0.2 — settling in), core calm (0.01). + final night = >[ + for (var t = 0; t < 7200; t++) epoch(55, 45, t.toDouble(), enmo: 0.2, skinTemp: 3350), + for (var t = 7200; t < 21600; t++) epoch(55, 45, t.toDouble(), enmo: 0.01, skinTemp: 3350), + for (var t = 21600; t < 28800; t++) epoch(55, 45, t.toDouble(), enmo: 0.2, skinTemp: 3350), + ]; + // opening-window baseline (old behavior) would see enmo 0.2: + a.computeBaseline(night.take(7200).toList()); + expect(a.enmoMedianNight, 0.2); + // core-based motion baseline sees the calm middle 50 %: + a.computeMotionBaseline(night); + expect(a.enmoMedianNight, 0.01); + }); + + test('motion baseline: absent ENMO (0 sentinel) never becomes the baseline', () { + final a = SleepStageAnalyzer(); + final night = [for (var t = 0; t < 1000; t++) epoch(55, 45, t.toDouble(), enmo: -1.0)]; + a.computeMotionBaseline(night); + expect(a.enmoMedianNight, isNull); + }); +} diff --git a/test/sleep_hmm_multinight_test.dart b/test/sleep_hmm_multinight_test.dart new file mode 100644 index 000000000..f490f93d6 --- /dev/null +++ b/test/sleep_hmm_multinight_test.dart @@ -0,0 +1,69 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/sleep_pipeline_hmm.dart'; + +Map epoch(double hr, double rmssd, {double enmo = 0.0, double skinTemp = 0.0}) => + {'hr': hr, 'rmssd': rmssd, 'enmo': enmo, 'skin_temp': skinTemp}; + +SleepStageAnalyzer baselined() { + final a = SleepStageAnalyzer(); + a.computeBaseline(List.generate(7200, (_) => epoch(55, 45, enmo: 0.01, skinTemp: 3350))); + return a; +} + +void main() { + test('multi-night: 7d and long-term baselines are computed from histories', () { + final a = baselined(); + final rhr = List.generate(30, (_) => 55.0); + final rmssd = List.generate(30, (_) => 45.0); + a.computeMultiNightBaselines(rhrHistory: rhr, rmssdHistory: rmssd); + expect(a.hrMin7d, 55.0); + expect(a.hrMinLong, 55.0); + expect(a.rmssdMean7d, 45.0); + expect(a.atypicalNight, isFalse); + // effective baseline blends nightly + 7d + expect(a.effectiveHrMin, 55.0); + }); + + test('multi-night: a deviating night is flagged atypical (outlier detection)', () { + final a = SleepStageAnalyzer(); + // tonight's baseline is 12 bpm higher than the personal norm (illness/alcohol) + a.computeBaseline(List.generate(7200, (_) => epoch(67, 45, enmo: 0.01, skinTemp: 3350))); + final rhr = List.generate(30, (_) => 55.0); + a.computeMultiNightBaselines(rhrHistory: rhr, rmssdHistory: List.filled(30, 45.0)); + expect(a.atypicalNight, isTrue, + reason: 'HR baseline +12 bpm vs long-term with tiny MAD must be an outlier night'); + }); + + test('multi-night: no history -> no 7d/long-term, nightly baseline still works', () { + final a = baselined(); + a.computeMultiNightBaselines(); + expect(a.hrMin7d, isNull); + expect(a.hrMinLong, isNull); + expect(a.effectiveHrMin, 55.0); // falls back to nightly + expect(a.atypicalNight, isFalse); + }); + + test('multi-night: normal variation within long-term spread is NOT atypical', () { + final a = baselined(); + // long-term history with realistic spread (±3 bpm), tonight at +2 + final rhr = [52.0, 53.0, 54.0, 55.0, 56.0, 57.0, 58.0, ...List.filled(23, 55.0)]; + a.computeMultiNightBaselines(rhrHistory: rhr, rmssdHistory: List.filled(30, 45.0)); + expect(a.atypicalNight, isFalse); + }); + + test('multi-night: ENMO history feeds 7d/long-term motion calibration', () { + final a = baselined(); + final enmo = List.generate(30, (_) => 0.02); + a.computeMultiNightBaselines(enmoHistory: enmo); + expect(a.enmoMedian7d, 0.02); + expect(a.enmoMedianLong, 0.02); + // blended with the nightly core median (0.01 from baselined()) + expect(a.effectiveEnmoMedian, closeTo(0.015, 1e-9)); + }); + + test('multi-night: ENMO absent sentinel (-1) is excluded from the history median', () { + final a = baselined(); + a.computeMultiNightBaselines(enmoHistory: [-1.0, -1.0, 0.02, 0.02]); + expect(a.enmoMedianLong, 0.02); + }); +}