From efe658f602ac1a3548751dcb11be9fc77f82a9a8 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 11:22:29 -0600 Subject: [PATCH 01/11] deps(analytics): adopt analytics #99; bump algorithm version to 122 Pin analytics to f8e7d3c for its incremental calculations, streams and checkpoint APIs. Main's 3a7ca34 pin does not contain them. This existing #99 head also omits #98's workout-rest bridge; that dependency must be reconciled before merge. Bump 121 to 122 for the capped RR corrector, toleranced batch aggregators, spectral and absence changes, inclusive ownership, and target-night profile folding. Preserve main's step-counter and nap-cap changes. Regenerate WHOOP freeze fixtures through GENERATE_GOLDEN=1. Changed metric values match the original PR exactly; the remaining changes are version stamps and payload hashes. The two-device fixture retains the PR's SDNN rounding change. Protocol stays at main's fbadb5bd pin. --- lib/compute/derivation_engine.dart | 73 ++++++++++++++++++- lib/compute/derive_prepare.dart | 8 ++ lib/compute/ed2b_checkpoint_diagnostics.dart | 24 ++++++ lib/compute/ed2b_stream_diagnostics.dart | 43 +++++++++++ lib/compute/onehz_pipeline.dart | 20 ++++- lib/compute/substrate.dart | 2 + pubspec.lock | 4 +- pubspec.yaml | 4 +- test/ed2a_substrate_transfer_test.dart | 2 +- test/ed2b_algo_version_114_test.dart | 21 ++++++ test/fixtures/two_device_day_expected.json | 2 +- .../fixtures/whoop_freeze/gen4_minus0400.json | 60 +++++++-------- test/fixtures/whoop_freeze/gen4_plus0530.json | 60 +++++++-------- test/fixtures/whoop_freeze/gen4_utc.json | 60 +++++++-------- .../fixtures/whoop_freeze/gen5_minus0400.json | 52 ++++++------- test/fixtures/whoop_freeze/gen5_plus0530.json | 52 ++++++------- test/fixtures/whoop_freeze/gen5_utc.json | 52 ++++++------- 17 files changed, 359 insertions(+), 180 deletions(-) create mode 100644 lib/compute/ed2b_checkpoint_diagnostics.dart create mode 100644 lib/compute/ed2b_stream_diagnostics.dart create mode 100644 test/ed2b_algo_version_114_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 885b4d920..7170be6a6 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -72,7 +72,9 @@ import 'hr_max.dart' show estimatedMaxHr, kHrFloorBpm, smoothedMaxHr, smoothedMinHr; import 'movement_floor_policy.dart' as mfp; import 'sleep_profile_policy.dart'; +import 'ed2b_checkpoint_diagnostics.dart'; import 'staged_sleep_cache.dart'; +import 'ed2b_stream_diagnostics.dart'; import 'substrate_retention.dart'; import 'derive_prepare.dart'; import 'onehz_pipeline.dart'; @@ -1861,7 +1863,16 @@ import 'vendor_sleep.dart'; // applyNapEdits — so a pre-cap manual nap longer than 3 h no longer counts // into naps/nap_min. Days derived at v120 keep their stale naps without this // bump. -const int kAlgoVersion = 121; +// 121 → 122 (analytics #99 @ f8e7d3c): RR artifact-run cap 32; +// toleranced batch aggregators hrvTime, banisterTrimp, ctlAtlTsb, cosinor, +// timeInZone and circadianNonparametric move last digits; Lomb–Scargle origin, +// singular-cosinor abstention, non-finite guards and IrregularScreenState's +// irregular-screen gaps change outputs. Retains the cosinor significance gate, +// nullable phase/amplitudeP, RSA floor guard and checkpoint v3, and SRI rules. +// Pooled-run batch staging labels unchanged; cardio caching respects run bounds. +// This single bump also covers D35 inclusive ownership and D36 target-night +// profile folding, forward-only. No profile reset. +const int kAlgoVersion = 122; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. /// @@ -2048,7 +2059,9 @@ const int kAlgoVersion = 121; // merge + spread calibration + capped re-centring + robust lnRMSSD // centre), for v112. // REPIN @ 3a7ca34: analytics main, #101 + #100 + #97 + #98, for v119. -const String kAnalyticsPin = '3a7ca34d30233ff11e6c2877083f319a907ccb31'; +// REPIN @ f8e7d3c: rebased analytics #99, for v122 above. +// This head forks at 9db83ce and omits #98's workout rest bridge at 3a7ca34. +const String kAnalyticsPin = 'f8e7d3c478cc59707fde41f3edff7736026573fe'; // Repinned to analytics main's tip, which carries BOTH PR #72 (hrv_freq // Welch gap guard) and PR #73 (overreachingConjunction rhr quantum guard) — // the two independent kAlgoVersion bumps above (93 and 94). Verified both @@ -3111,6 +3124,52 @@ class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; + // ED-2b RED observations. Controls stay inert until their GREEN phase. + final debugCheckpoints = Ed2bCheckpointDiagnostics(); + @visibleForTesting + void Function(String day)? debugOnProfileFold; + @visibleForTesting + Future debugFoldObservation(String day, String observation) => + _foldObservationIntoProfile(day, observation); + @visibleForTesting + // ED-2b RED: optional observations follow the real day-bundle worker, + // carrying counters only. No night stream is retained or enabled here. + Ed2bStreamDiagnostics? debugNightStreams; + @visibleForTesting + void Function(Map input, Map output)? + debugOnNightBundle; + @visibleForTesting + Uint8List? debugCardioRowBytes; + @visibleForTesting + bool debugDisableCardioRows = false; + @visibleForTesting + int debugCardioRowHits = 0; + @visibleForTesting + int debugCardioRowMisses = 0; + @visibleForTesting + List get debugCardioRowWorkerOperations => const []; + @visibleForTesting + void debugRekeyCardioRows({required int inputRevision, + required int generation, required int algorithm}) {} + @visibleForTesting + bool debugDisableUnionLoads = false; + @visibleForTesting + int get debugUnionWorkerCuts => 0; + @visibleForTesting + void debugRekeyUnionAlgorithm(int algorithm) {} + @visibleForTesting + void Function(int from, int to, String load)? debugOnSubstrateLoad; + @visibleForTesting + Future<(Map>, Map>)> + debugResolveOwnership(int from, int to) => _resolveOwnership(from, to); + @visibleForTesting + Future debugLoadOwnedSubstrate(int from, int to, + {required String day}) async { + final (ownership, _) = await _resolveOwnership(from, to); + return _loadSubstrateRange(from, to, dayId: day, ownership: ownership); + } + + // ED-2a RED seams. Empty cache views and no-op controls describe the current // uncached engine. Counters below observe real worker work, never results. // E12/E13 back staging and retention; E18 backs finished-session reuse. @@ -4876,6 +4935,7 @@ class DerivationEngine { // profile, fold, and clobber the other, losing both the fold and its // day_id from folded_days. SQLite's write lock is cross-connection and // therefore cross-isolate. + debugOnProfileFold?.call(dayId); await LocalDb.updateBaseline('sleep_user_profile', (current) { // Re-derive freshness INSIDE the transaction: the value this day read // before staging is stale by definition, another lane may have folded @@ -4983,6 +5043,7 @@ class DerivationEngine { Map> ownership = const {}, String load = 'direct', }) async { + debugOnSubstrateLoad?.call(fromRecTs, toRecTs, load); if (toRecTs < fromRecTs) { _dropRetainedDay(dayId); return Substrate.empty; @@ -6368,11 +6429,15 @@ class DerivationEngine { // Cancellable: on timeout the isolate is KILLED, not merely abandoned to // keep burning a core behind the worker pool's back. - final bundle = await _runIsolateCancellable( - () => deriveDayBundle(withHistory), + final diagnostics = debugNightStreams; + final workerResult = await _runIsolateCancellable( + () => (deriveDayBundle(withHistory, streamDiagnostics: diagnostics), diagnostics), _perDayTimeout, label: 'day-bundle ${day.date}', ); + final bundle = workerResult.$1; + if (diagnostics != null) debugNightStreams = workerResult.$2; + debugOnNightBundle?.call(withHistory, bundle); // Readiness came back absent for TODAY specifically (not a historical // backfill day, which would just be noise) — log why. This ran inside // Isolate.run so it couldn't call Firebase itself; it just returned the diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index b008c9473..8c3a7fdf5 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -415,6 +415,8 @@ PreparedDerivationPayload prepareDerivationPayload( List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], bool skipUnusedDays = false, + ana.CardioRowCache? rowCache, // ED-2b RED: unused until E6 adoption. + bool recordTargetOnly = false, // ED-2b RED: unused until D36 adoption. }) { if (sub.isEmpty || sub.lastTs == null) { return const PreparedDerivationPayload(dataNowSec: 0, days: []); @@ -422,6 +424,8 @@ PreparedDerivationPayload prepareDerivationPayload( final days = []; for (final day in calendarDays( sub, + rowCache: rowCache, + recordTargetOnly: recordTargetOnly, override: override, targetDay: targetDay, skipUnusedDays: skipUnusedDays, @@ -480,6 +484,8 @@ SleepSessionCandidate prepareSleepSessionCandidate( List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], bool skipUnusedDays = false, + ana.CardioRowCache? rowCache, // ED-2b RED: unused until E6 adoption. + bool recordTargetOnly = false, // ED-2b RED: unused until D36 adoption. }) { // An UNCLAIMED device night (our rows saw under half of it) ending today IS // the night: above our detection, below the user's word. It never reaches @@ -523,6 +529,8 @@ SleepSessionCandidate prepareSleepSessionCandidate( claimed.add(n); } final payload = prepareDerivationPayload(sub, + rowCache: rowCache, + recordTargetOnly: recordTargetOnly, targetDay: targetDay, override: override, skipUnusedDays: skipUnusedDays, diff --git a/lib/compute/ed2b_checkpoint_diagnostics.dart b/lib/compute/ed2b_checkpoint_diagnostics.dart new file mode 100644 index 000000000..a39c0ce58 --- /dev/null +++ b/lib/compute/ed2b_checkpoint_diagnostics.dart @@ -0,0 +1,24 @@ +import 'dart:typed_data'; + +/// Observations of the real checkpoint path. No storage, codec or resume policy +/// lives here. Counters remain zero until the GREEN implementation reports work. +class Ed2bCheckpointDiagnostics { + bool disabled = false; + int reads = 0; + int writes = 0; + int hits = 0; + int misses = 0; + final List rejections = []; + Uint8List? lastEncodedPayload; + Map? lastDecodedPayload; + + void reset() { + reads = 0; + writes = 0; + hits = 0; + misses = 0; + rejections.clear(); + lastEncodedPayload = null; + lastDecodedPayload = null; + } +} diff --git a/lib/compute/ed2b_stream_diagnostics.dart b/lib/compute/ed2b_stream_diagnostics.dart new file mode 100644 index 000000000..69b2a14a9 --- /dev/null +++ b/lib/compute/ed2b_stream_diagnostics.dart @@ -0,0 +1,43 @@ +// ED-2b RED observations only. This object retains no analytics state and never +// changes a result. Work is counted beside the existing batch calls. GREEN can +// replace those calls with streams while preserving these measurement meanings. +class Ed2bStreamDiagnostics { + int passes = 0; + int processedNightBeats = 0; + int nightRebuilds = 0; + final Map _batchBeats = {}; + final Map _batchCalls = {}; + bool disabled = false; + void Function(List nn, List times, double cleanFraction)? + onCorrected; + void Function()? onBeforePassReturn; + void Function(List)? onRespWindows; + + void beginPass() { passes++; } + void endPass() { onBeforePassReturn?.call(); } + void context({required int inputRevision, required int generation, + required int algorithm}) {} // RED: no retained state to invalidate. + // Total input work: GREEN counts only newly appended/settled beats here, + // regardless of whether the operation uses a batch call or an addSlice. + int get retainedNightBeats => 0; + int get settledNightBeats => 0; + int get identityMemoEntries => 0; + int get fullListDigests => 0; + int get maximumListDigestsPerIdentity => 0; + int retainedStreamBeats(String stream) => 0; + int processedBeats(String stream) => _batchBeats[stream] ?? 0; + int slicesProcessed(String stream) => _batchCalls[stream] ?? 0; + int streamRestarts(String stream) => 0; + + void observeNightCorrection(int rawBeats, List nn, + List times, double cleanFraction) { + processedNightBeats += rawBeats; + if (rawBeats > 0) nightRebuilds++; + onCorrected?.call(nn, times, cleanFraction); + } + + void observeBatch(String stream, int beats) { + _batchCalls[stream] = slicesProcessed(stream) + 1; + _batchBeats[stream] = processedBeats(stream) + beats; + } +} diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 9f188d117..96233c53e 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -32,6 +32,7 @@ import 'package:openstrap_analytics/onehz.dart'; // instead of two that can drift. import 'inputs/garmin_inputs.dart' show isWearableFamily, kHrCadenceSec, kOneHzRunGapSec; +import 'ed2b_stream_diagnostics.dart'; import 'hr_max.dart' show estimatedMaxHr, @@ -545,7 +546,10 @@ class DayBundleInput { /// Pure: takes the serialized [DayBundleInput] map, returns a plain JSON map (the /// full derived bundle). Call directly + synchronously in tests, or via /// `Isolate.run(() => deriveDayBundle(input))` in production. -Map deriveDayBundle(Map inputJson) { +Map deriveDayBundle(Map inputJson, { + Ed2bStreamDiagnostics? streamDiagnostics, +}) { + streamDiagnostics?.beginPass(); final d = DayBundleInput.fromJson(inputJson); // ── HR over the DAY (for the curve, strain zones, dip day-side) ─────────── @@ -592,6 +596,8 @@ Map deriveDayBundle(Map inputJson) { // the SLEEP RR (not the whole day) is what brings RMSSD back to physiological // tens-of-ms instead of the whole-day ~166 ms inflated value. final corrected = correctRr(d.sleepRrMs, rrTsMs: d.sleepRrTsMs); + streamDiagnostics?.observeNightCorrection(d.sleepRrMs.length, corrected.nn, + corrected.nnTimesMs, corrected.cleanFraction); final nn = corrected.nn; final nnTimes = corrected.nnTimesMs; final artifactFraction = (1.0 - corrected.cleanFraction).clamp(0.0, 1.0); @@ -659,6 +665,7 @@ Map deriveDayBundle(Map inputJson) { inputs_used: const ['rr_cleaned'], note: needInputNote('nn_beats', have: nn.length, need: 20), ); + if (nn.length >= 20) streamDiagnostics?.observeBatch('frequency', nn.length); // Nocturnal RHR over the SLEEP HR. NO SLEEP ⇒ NO RESTING HR. // // This used to fall back to the whole-day HR when no sleep was scored, and @@ -708,6 +715,7 @@ Map deriveDayBundle(Map inputJson) { final dip = hrDip(dayOnly, sleepHr); final dc = decelerationCapacity(nn); final ac = accelerationCapacity(nn); + streamDiagnostics?.observeBatch('prsa', nn.length); // Baevsky Stress Index over the sleep NN — resting autonomic tension (a // transparent RR-histogram metric; no ML). Daily resting-stress indicator. // nnTimesMs lets it segment at a charging/off-wrist gap instead of letting @@ -730,6 +738,8 @@ Map deriveDayBundle(Map inputJson) { inputs_used: const ['rr_cleaned'], note: needInputNote('nn_beats', have: nn.length, need: 60), ); + if (nn.length >= 30) streamDiagnostics?.observeBatch('rsa', nn.length); + if (nn.length >= 60) streamDiagnostics?.observeBatch('cvhr', nn.length); // ── 24/7 IRREGULAR-RHYTHM SCREEN (day-span RR; not a diagnosis) ──────────── // Runs over the WHOLE-DAY cleaned RR (not just sleep) so an arrhythmia screen // isn't limited to the sleep window. Hard-gated on beat count + artifact inside @@ -749,8 +759,9 @@ Map deriveDayBundle(Map inputJson) { // ── BREATHING-RATE VARIABILITY (per-window RSA over the sleep NN) ────────── // Window the cleaned sleep NN into ~30-min bins, take each bin's RSA resp rate, // then BRV = dispersion + Theil-Sen trend of those per-window rates. - final respWindows = - respPerWindow(nn, nnTimes, artifactFraction: artifactFraction); + final respWindows = respPerWindow(nn, nnTimes, + artifactFraction: artifactFraction, diagnostics: streamDiagnostics); + streamDiagnostics?.onRespWindows?.call(respWindows); final brv = respWindows.length >= 3 ? breathingRateVariability(respWindows) : Metric.absent( @@ -1697,6 +1708,7 @@ Map deriveDayBundle(Map inputJson) { ), }; + streamDiagnostics?.endPass(); return { 'date': d.date, 'day_confidence': _round(d.dayConfidence, 4), @@ -1958,6 +1970,7 @@ List respPerWindow( List nn, List nnTimes, { required double artifactFraction, + Ed2bStreamDiagnostics? diagnostics, double windowMs = 1800000.0, int minBeats = 60, }) { @@ -1977,6 +1990,7 @@ List respPerWindow( if (segNn.length < minBeats) continue; final r = rsaRespRate(segNn, binsTs[idx]!, artifactFraction: artifactFraction); + diagnostics?.observeBatch('rsa_bins', segNn.length); final b = r.present ? r.value!.brpm : null; if (b != null) out.add(b); } diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index 08b534fe9..f201dcdad 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -1267,6 +1267,8 @@ List calendarDays( List vendorNights = const [], String? targetDay, bool skipUnusedDays = false, + ana.CardioRowCache? rowCache, // ED-2b RED: unused until E6 adoption. + bool recordTargetOnly = false, // ED-2b RED: unused until D36 adoption. }) { final tzOffset = tzOffsetAt ?? tzOffsetSecondsAt; if (sub.isEmpty) return const []; diff --git a/pubspec.lock b/pubspec.lock index 6e95eb2e8..33ecd3c50 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -937,8 +937,8 @@ packages: dependency: "direct main" description: path: "." - ref: "3a7ca34d30233ff11e6c2877083f319a907ccb31" - resolved-ref: "3a7ca34d30233ff11e6c2877083f319a907ccb31" + ref: f8e7d3c478cc59707fde41f3edff7736026573fe + resolved-ref: f8e7d3c478cc59707fde41f3edff7736026573fe url: "https://github.com/OpenStrap/analytics.git" source: git version: "1.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index e922d408f..34fbc145a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -522,7 +522,9 @@ dependencies: # analytics main @ 3a7ca34: adds #101 (HRV/readiness/wellness fixes), #100 # (sleep staging and naps), #97 (stride-periodicity step counter) and #98 # (a mid-session rest stays inside one workout). - ref: 3a7ca34d30233ff11e6c2877083f319a907ccb31 + # Rebased analytics #99: incremental kernels, RSA v3 and cosinor gate (v122). + # This head forks at 9db83ce and omits #98's workout rest bridge at 3a7ca34. + ref: f8e7d3c478cc59707fde41f3edff7736026573fe # BLE — flutter_blue_plus is the maintained cross-platform GATT client. flutter_blue_plus: ^1.36.8 diff --git a/test/ed2a_substrate_transfer_test.dart b/test/ed2a_substrate_transfer_test.dart index de120beab..0511227bc 100644 --- a/test/ed2a_substrate_transfer_test.dart +++ b/test/ed2a_substrate_transfer_test.dart @@ -241,7 +241,7 @@ void main() { markTestSkipped('committed golden is UTC; run this test with TZ=UTC'); return; } - expect(kAlgoVersion, 121, reason: "ED-2a keeps main's algorithm version"); + expect(kAlgoVersion, 122, reason: 'ED-2b repins analytics; transfer remains exact'); final fixture = _fixture(); final baseline = substrateFromDecodedPage(fixture.frames, fixture.rr); final transferred = Substrate.fromTransfer(baseline.toJson()); diff --git a/test/ed2b_algo_version_114_test.dart b/test/ed2b_algo_version_114_test.dart new file mode 100644 index 000000000..30555cd37 --- /dev/null +++ b/test/ed2b_algo_version_114_test.dart @@ -0,0 +1,21 @@ +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; + +void main() { + test('ED-2b takes one version bump and names the output changes', () { + expect(kAlgoVersion, 122); + expect(kAnalyticsPin, 'f8e7d3c478cc59707fde41f3edff7736026573fe'); + final src = File('lib/compute/derivation_engine.dart').readAsStringSync(); + final at = src.indexOf('// 121 → 122'); + expect(at, isNonNegative); + final entry = src.substring(at, src.indexOf('const int kAlgoVersion', at)); + for (final change in ['f8e7d3c', '32', 'hrvTime', 'banisterTrimp', + 'ctlAtlTsb', 'cosinor', 'timeInZone', 'circadianNonparametric', + 'Lomb–Scargle', 'non-finite', 'irregular-screen gaps', 'D35', 'D36', + 'forward-only', 'staging labels unchanged']) { + expect(entry, contains(change)); + } + }); +} diff --git a/test/fixtures/two_device_day_expected.json b/test/fixtures/two_device_day_expected.json index db883f8d2..a682b3071 100644 --- a/test/fixtures/two_device_day_expected.json +++ b/test/fixtures/two_device_day_expected.json @@ -3895,7 +3895,7 @@ "skin_temp_adc": 30005.0, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 14.14213555551533, + "sdnn": 14.142135555515845, "sdnn_window": 14.164940954653554, "prsa_dc": 15.0, "prsa_dc_anchors": 2879.0, diff --git a/test/fixtures/whoop_freeze/gen4_minus0400.json b/test/fixtures/whoop_freeze/gen4_minus0400.json index 0cbf310b9..f850fa4a6 100644 --- a/test/fixtures/whoop_freeze/gen4_minus0400.json +++ b/test/fixtures/whoop_freeze/gen4_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 227.5841832798056, + "value": 227.58418327980579, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23019,12 +23019,12 @@ "skin_temp_adc": 30405.99974110511, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -24898,11 +24898,11 @@ "readiness": 27.033995461954163, "skipped": 0, "partial": 0, - "payload_sha256": "b57127eaba2e220875324219ed002d3801c330506d1303b87a413572f89ec560" + "payload_sha256": "62a201619a8a80c7a58a4f8feefe51b9b9b17b71fb395a35456b5f06c2705908" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25132,7 +25132,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 221.15132594116318, + "value": 221.15132594116335, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47739,12 +47739,12 @@ "skin_temp_adc": 30405.999593165176, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49591,11 +49591,11 @@ "readiness": 27.03409387728019, "skipped": 0, "partial": 0, - "payload_sha256": "c665785d82540466a55ea1cbce98b9c46288e0fec0fb90c64878f908412e0e2e" + "payload_sha256": "52604af7568a6615643226ce4d7f339e594e6e9ef256b6e9d9720ae3ae91d98f" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -49831,7 +49831,7 @@ "note": "need_baseline:have=2,need=3" }, "trimp": { - "value": 218.15549360346512, + "value": 218.15549360346523, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -71054,12 +71054,12 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 29.23441483806156, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -72717,7 +72717,7 @@ "readiness": 66.11544177079463, "skipped": 0, "partial": 0, - "payload_sha256": "274e77c395da40e489da25b343bbe71e207dcd4fbe51571a521b6ef3d9a42c90" + "payload_sha256": "339b824c3825df771b722599c2b712a19fb697c719ccb46cb77deef5ec91fb2b" } ], "metric_series": [ @@ -72864,7 +72864,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -72909,7 +72909,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 227.5841832798056 + "value": 227.58418327980579 }, { "date": "2026-07-13", @@ -73074,7 +73074,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73119,7 +73119,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 221.15132594116318 + "value": 221.15132594116335 }, { "date": "2026-07-14", @@ -73284,7 +73284,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73329,7 +73329,7 @@ { "date": "2026-07-15", "key": "trimp", - "value": 218.15549360346512 + "value": 218.15549360346523 }, { "date": "2026-07-15", @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -76995,7 +76995,7 @@ "skin_temp_adc": 30405.99974110511, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "strain": null, "steps": null, "nap_min": 0.0, @@ -77208,7 +77208,7 @@ "skin_temp_adc": 30405.999593165176, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "strain": null, "steps": null, "nap_min": 45.0, @@ -77421,7 +77421,7 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "strain": null, "steps": null, "nap_min": 0.0, diff --git a/test/fixtures/whoop_freeze/gen4_plus0530.json b/test/fixtures/whoop_freeze/gen4_plus0530.json index 192d21684..2fe1d50a5 100644 --- a/test/fixtures/whoop_freeze/gen4_plus0530.json +++ b/test/fixtures/whoop_freeze/gen4_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 227.5841832798056, + "value": 227.58418327980579, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23019,12 +23019,12 @@ "skin_temp_adc": 30406.0, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -24898,11 +24898,11 @@ "readiness": 27.03382323567632, "skipped": 0, "partial": 0, - "payload_sha256": "cda75458f39702ac4580c5d03d4bf6a6b5c5120aee516e8c32d1c6fe33383def" + "payload_sha256": "9df2d6b463c1d4668fe083d815dde535441090b667f7923c7601a80b2cd98cc0" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25132,7 +25132,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 221.15132594116318, + "value": 221.15132594116335, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47739,12 +47739,12 @@ "skin_temp_adc": 30406.000332864856, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49591,11 +49591,11 @@ "readiness": 27.033601802905416, "skipped": 0, "partial": 0, - "payload_sha256": "3228493c5545d3a4ee5f2da9410317da44cf8e9947d17b049572ca698c63112d" + "payload_sha256": "2cece0333903e0338a57f753ae55610073e575f4bd1931f74f6555b982237c15" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -49831,7 +49831,7 @@ "note": "need_baseline:have=2,need=3" }, "trimp": { - "value": 218.15549360346512, + "value": 218.15549360346523, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -71054,12 +71054,12 @@ "skin_temp_adc": 30406.00018492492, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 29.23441483806156, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -72717,7 +72717,7 @@ "readiness": 66.1155535435143, "skipped": 0, "partial": 0, - "payload_sha256": "bf1c2270ea868effc012ed7f83c834eac0f08e642bbde8aa6894e8aa2b394d18" + "payload_sha256": "a29b3dd5f3759041399c9ac0681eb643c449ba95e59b96b1a1a993d6fcee53c7" } ], "metric_series": [ @@ -72864,7 +72864,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -72909,7 +72909,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 227.5841832798056 + "value": 227.58418327980579 }, { "date": "2026-07-13", @@ -73074,7 +73074,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73119,7 +73119,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 221.15132594116318 + "value": 221.15132594116335 }, { "date": "2026-07-14", @@ -73284,7 +73284,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73329,7 +73329,7 @@ { "date": "2026-07-15", "key": "trimp", - "value": 218.15549360346512 + "value": 218.15549360346523 }, { "date": "2026-07-15", @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -76995,7 +76995,7 @@ "skin_temp_adc": 30406.0, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "strain": null, "steps": null, "nap_min": 0.0, @@ -77208,7 +77208,7 @@ "skin_temp_adc": 30406.000332864856, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "strain": null, "steps": null, "nap_min": 45.0, @@ -77421,7 +77421,7 @@ "skin_temp_adc": 30406.00018492492, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "strain": null, "steps": null, "nap_min": 0.0, diff --git a/test/fixtures/whoop_freeze/gen4_utc.json b/test/fixtures/whoop_freeze/gen4_utc.json index 9688b6b32..4dd0e0b8a 100644 --- a/test/fixtures/whoop_freeze/gen4_utc.json +++ b/test/fixtures/whoop_freeze/gen4_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 227.5841832798056, + "value": 227.58418327980579, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23019,12 +23019,12 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -24898,11 +24898,11 @@ "readiness": 27.03355259577805, "skipped": 0, "partial": 0, - "payload_sha256": "f13bcc5af9185bb1b8384a9bc439a551a03bc4367ea2fba85c5742ef910df9d8" + "payload_sha256": "a9027226526086f68120d3401a25663926b07ff7515415e7e4f25ccd592e12ac" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25132,7 +25132,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 221.15132594116318, + "value": 221.15132594116335, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47739,12 +47739,12 @@ "skin_temp_adc": 30406.00025889489, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49591,11 +49591,11 @@ "readiness": 27.033651010089166, "skipped": 0, "partial": 0, - "payload_sha256": "411649f61ac1083a68de60dd1ff589276d011a075e7d22d81f88d4a692c5346a" + "payload_sha256": "f171df77c61ba6ac83f14a9b68516fe1bf2fc433463febe2aaff4eed6531c827" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -49831,7 +49831,7 @@ "note": "need_baseline:have=2,need=3" }, "trimp": { - "value": 218.15549360346512, + "value": 218.15549360346523, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -71054,12 +71054,12 @@ "skin_temp_adc": 30406.000110954952, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 29.23441483806156, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -72717,7 +72717,7 @@ "readiness": 66.11559080104756, "skipped": 0, "partial": 0, - "payload_sha256": "5e74cc212eca9dfc9b06827c32ad9b70b18ea48b899b34eaef557741382b7143" + "payload_sha256": "7ce4b9c113c012b9e87932da751c13e2c985386e077c114c17caa6f7a158fdc8" } ], "metric_series": [ @@ -72864,7 +72864,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -72909,7 +72909,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 227.5841832798056 + "value": 227.58418327980579 }, { "date": "2026-07-13", @@ -73074,7 +73074,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73119,7 +73119,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 221.15132594116318 + "value": 221.15132594116335 }, { "date": "2026-07-14", @@ -73284,7 +73284,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73329,7 +73329,7 @@ { "date": "2026-07-15", "key": "trimp", - "value": 218.15549360346512 + "value": 218.15549360346523 }, { "date": "2026-07-15", @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -76995,7 +76995,7 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "strain": null, "steps": null, "nap_min": 0.0, @@ -77208,7 +77208,7 @@ "skin_temp_adc": 30406.00025889489, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "strain": null, "steps": null, "nap_min": 45.0, @@ -77421,7 +77421,7 @@ "skin_temp_adc": 30406.000110954952, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "strain": null, "steps": null, "nap_min": 0.0, diff --git a/test/fixtures/whoop_freeze/gen5_minus0400.json b/test/fixtures/whoop_freeze/gen5_minus0400.json index 9d36e9dea..71bdb89f5 100644 --- a/test/fixtures/whoop_freeze/gen5_minus0400.json +++ b/test/fixtures/whoop_freeze/gen5_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 267.9104825050325, + "value": 267.9104825050327, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23020,12 +23020,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -25076,11 +25076,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "a3f73d5823f32766fd6f8741f46e2a72ac064ea73b044321b9ebb50480b75058" + "payload_sha256": "7324cb5f4edff114920b2fa3d4ba97cb974cd712eadbbd27991ebd940299011c" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25310,7 +25310,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 260.0235516950477, + "value": 260.02355169504744, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47918,12 +47918,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49942,11 +49942,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "44312e737e4881671c0f43e00ddfbb7bc0c4b59a578ad331e9fcc9d850f34936" + "payload_sha256": "b0b0de9fba0fe2e9ba7e6cadc2eb602ea27798154e12c66488143b0a603c3382" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -71400,7 +71400,7 @@ "skin_temp_dev_c": 0.1, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, @@ -73230,7 +73230,7 @@ "readiness": 61.79377627786914, "skipped": 0, "partial": 0, - "payload_sha256": "52208554e4cb1c08d66b6b7467560e007151d583ea6155dc3327496f18b87e17" + "payload_sha256": "6c93cfdfb231c5b9c02369f29df360cbced3fda2c908355c6cd22314c53e6572" } ], "metric_series": [ @@ -73377,7 +73377,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -73422,7 +73422,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 267.9104825050325 + "value": 267.9104825050327 }, { "date": "2026-07-13", @@ -73587,7 +73587,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73632,7 +73632,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 260.0235516950477 + "value": 260.02355169504744 }, { "date": "2026-07-14", @@ -73797,7 +73797,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -77507,7 +77507,7 @@ "skin_temp_z": 3.808966711097082, "skin_temp_adc": 30405.99974110511, "skin_temp_family": "gen5", - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "strain": null, "steps": 18240.0, "nap_min": 0.0, @@ -77719,7 +77719,7 @@ "skin_temp_z": 2.581920140166515, "skin_temp_adc": 30405.999593165176, "skin_temp_family": "gen5", - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "strain": null, "steps": 17640.0, "nap_min": 45.0, diff --git a/test/fixtures/whoop_freeze/gen5_plus0530.json b/test/fixtures/whoop_freeze/gen5_plus0530.json index 69eb2e248..f42f13095 100644 --- a/test/fixtures/whoop_freeze/gen5_plus0530.json +++ b/test/fixtures/whoop_freeze/gen5_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 267.9104825050325, + "value": 267.9104825050327, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23020,12 +23020,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -25076,11 +25076,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "a6bb812440c40efd53dc915e78474d6581b354a787fd83f1d8d82274e94b052b" + "payload_sha256": "9f3dde3ed4f3d2bd7988104ed1312ffd3bb78cf2f9c0dc6c3bbd949763c917be" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25310,7 +25310,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 260.0235516950477, + "value": 260.02355169504744, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47918,12 +47918,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49942,11 +49942,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "e6ee25535499c542ebd13b8a1df85566c229944efba4dbdb8a44ca980c6442dc" + "payload_sha256": "785a07813a59e1e553b90599fbc43708ad5cf6063202796c1777f11aa886aa18" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -71400,7 +71400,7 @@ "skin_temp_dev_c": 0.1, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, @@ -73230,7 +73230,7 @@ "readiness": 61.79377627786914, "skipped": 0, "partial": 0, - "payload_sha256": "95786f495b2c7013ed03c36aaf66f0fd7018dbb42776f145314869085111d68e" + "payload_sha256": "be6cda6e9bd97c996db7f60f380433e10f15b05d35949248b39ad35d530a420b" } ], "metric_series": [ @@ -73377,7 +73377,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -73422,7 +73422,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 267.9104825050325 + "value": 267.9104825050327 }, { "date": "2026-07-13", @@ -73587,7 +73587,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73632,7 +73632,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 260.0235516950477 + "value": 260.02355169504744 }, { "date": "2026-07-14", @@ -73797,7 +73797,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -77507,7 +77507,7 @@ "skin_temp_z": 3.8090587513754603, "skin_temp_adc": 30406.0, "skin_temp_family": "gen5", - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "strain": null, "steps": 18240.0, "nap_min": 0.0, @@ -77719,7 +77719,7 @@ "skin_temp_z": 2.5820748428079483, "skin_temp_adc": 30406.000332864856, "skin_temp_family": "gen5", - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "strain": null, "steps": 17640.0, "nap_min": 45.0, diff --git a/test/fixtures/whoop_freeze/gen5_utc.json b/test/fixtures/whoop_freeze/gen5_utc.json index 2ed458474..215bdc3e0 100644 --- a/test/fixtures/whoop_freeze/gen5_utc.json +++ b/test/fixtures/whoop_freeze/gen5_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 267.9104825050325, + "value": 267.9104825050327, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23020,12 +23020,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -25076,11 +25076,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "64bd6e2d3cd9fb8a3762d32513c756dbfe5d33e204538d64f6188eb88b25afff" + "payload_sha256": "1bd3caf43bd58cb8ec23f7fbdc3ece0bf0c5c16d4e8ee0281da29dfb9f53d499" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25310,7 +25310,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 260.0235516950477, + "value": 260.02355169504744, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47918,12 +47918,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49942,11 +49942,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "42d874e0691a2e1f19f3ca870dbf497fd820f6daa556da8bec1cbc7f62b661ca" + "payload_sha256": "1a383a9942303b44cfa356dd2d7c2d50c0edfa97a1eaa2b02e53d56cbcd1b1bf" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -71400,7 +71400,7 @@ "skin_temp_dev_c": 0.1, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, @@ -73230,7 +73230,7 @@ "readiness": 61.79377627786914, "skipped": 0, "partial": 0, - "payload_sha256": "a7d36fabdb957577d3df84059f048e09a85ade8ecce048f891c00de4be9018fd" + "payload_sha256": "24ce9711e808adc5712473639cba81eb6c0fe1c84a7668b00c19ea8bcafcd362" } ], "metric_series": [ @@ -73377,7 +73377,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -73422,7 +73422,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 267.9104825050325 + "value": 267.9104825050327 }, { "date": "2026-07-13", @@ -73587,7 +73587,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73632,7 +73632,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 260.0235516950477 + "value": 260.02355169504744 }, { "date": "2026-07-14", @@ -73797,7 +73797,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 122, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -77507,7 +77507,7 @@ "skin_temp_z": 3.8092033860986256, "skin_temp_adc": 30406.000406834824, "skin_temp_family": "gen5", - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "strain": null, "steps": 18240.0, "nap_min": 0.0, @@ -77719,7 +77719,7 @@ "skin_temp_z": 2.581998719140345, "skin_temp_adc": 30406.00025889489, "skin_temp_family": "gen5", - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "strain": null, "steps": 17640.0, "nap_min": 45.0, From 7f71df78c3b13de4b8fe80a4b3affc6aee27923a Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 11:22:29 -0600 Subject: [PATCH 02/11] derive: resolve ownership through to+1; the sleep profile learns only from the target night - Coverage and ownership are queried and resolved through `to + 1`: the callers pass an inclusive `to` into half-open resolution, so a non-owner's rows in the last second were kept. - A folding pass records and folds only the target night's observation (none when the target has none) instead of the longest night it walked, and restores the recording flags. Forward-only: existing profiles and folded_days are not reset. The restricted walk may now skip unused days on folding passes too. Neither change moves a value in the committed fixtures. --- lib/compute/derivation_engine.dart | 19 +- lib/compute/derive_prepare.dart | 4 +- lib/compute/substrate.dart | 19 +- .../ed2a_e15_restricted_target_walk_test.dart | 20 +- test/ed2b_d35_ownership_endpoint_test.dart | 85 ++++++++ test/ed2b_d36_target_profile_test.dart | 195 ++++++++++++++++++ test/multidevice_coverage_derive_test.dart | 4 +- 7 files changed, 320 insertions(+), 26 deletions(-) create mode 100644 test/ed2b_d35_ownership_endpoint_test.dart create mode 100644 test/ed2b_d36_target_profile_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 7170be6a6..a90949e20 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -1870,8 +1870,10 @@ import 'vendor_sleep.dart'; // irregular-screen gaps change outputs. Retains the cosinor significance gate, // nullable phase/amplitudeP, RSA floor guard and checkpoint v3, and SRI rules. // Pooled-run batch staging labels unchanged; cardio caching respects run bounds. -// This single bump also covers D35 inclusive ownership and D36 target-night -// profile folding, forward-only. No profile reset. +// D35 queries coverage and resolves half-open ownership through inclusive to + 1, +// excluding non-owner rows at the last second. D36 folds only the target night's +// own observation, none when absent; forward-only, preserving existing profiles +// and folded_days. The restricted E15 walk now also applies to folding passes. const int kAlgoVersion = 122; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. @@ -4438,6 +4440,8 @@ class DerivationEngine { /// candidate — so the spans differ, but the rule producing them does not. Future<(Map>, Map>)> _resolveOwnership(int from, int to) async { + // Loads include `to`; coverage queries and ownership spans are half-open. + final end = to + 1; final ownership = >{}; // The order each signal ACTUALLY resolved under, carried on the prepared // day so `priority_hash` names it by construction instead of re-reading @@ -4466,7 +4470,7 @@ class DerivationEngine { // gives it a rank — the conservative default, and the one that keeps // today's single-device installs byte-identical. final rawPriority = await LocalDb.signalPriority(sig); - final coverage = await LocalDb.coverageIntervals(sig, from, to); + final coverage = await LocalDb.coverageIntervals(sig, from, end); var resolved = rawPriority.isEmpty ? const [LocalDb.kPrimaryDeviceId] // A device with a coverage row here is definitionally declaring @@ -4497,7 +4501,7 @@ class DerivationEngine { coverage: coverage, priority: resolved, from: from, - to: to, + to: end, signal: sig, ); } @@ -4706,7 +4710,7 @@ class DerivationEngine { // with no timeout at all, so a hung staging pass never completed its future // — `_running` stayed true and `DeriveScheduler._drain` never returned, i.e. // all derivation was dead until app restart. - final skipUnusedDays = !mayFold && !debugDisableTargetDaySkip; + final skipUnusedDays = !debugDisableTargetDaySkip; // Read here: a `_perDayTimeout` inside `stage` would capture `this` into // the context the worker closure ships, and the engine's callbacks (the // app's own fields) are not sendable. @@ -4737,11 +4741,12 @@ class DerivationEngine { searchSub, targetDay: dayId, skipUnusedDays: skipUnusedDays, + recordTargetOnly: true, override: override, priorSleep: priorSleep, vendorNights: vendorNights, ); - // Fold the MAIN sleep (most epochs) of a freshly-staged night into the + // Fold the TARGET sleep (most epochs of its own observations) into the // rolling profile — done here in the worker because the observations live // in THIS isolate's globals. Skipped for overrides. EWMA self-seeds. String? observationJson; @@ -4754,7 +4759,7 @@ class DerivationEngine { // onto whichever day was re-derived last. Measured effect of that // corrupt profile on the same nights: wake 4.3% -> 36.4%, deep 1.9% -> // 0.0%. One fold per day_id, ever. - if (mayFold) { + if (mayFold && candidate.present) { final obs = ana.takeCardioObservations(); if (obs.isNotEmpty) { obs.sort((a, b) => b.epochs.compareTo(a.epochs)); diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index 8c3a7fdf5..912f9b853 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -416,7 +416,7 @@ PreparedDerivationPayload prepareDerivationPayload( List vendorNights = const [], bool skipUnusedDays = false, ana.CardioRowCache? rowCache, // ED-2b RED: unused until E6 adoption. - bool recordTargetOnly = false, // ED-2b RED: unused until D36 adoption. + bool recordTargetOnly = false, }) { if (sub.isEmpty || sub.lastTs == null) { return const PreparedDerivationPayload(dataNowSec: 0, days: []); @@ -485,7 +485,7 @@ SleepSessionCandidate prepareSleepSessionCandidate( List vendorNights = const [], bool skipUnusedDays = false, ana.CardioRowCache? rowCache, // ED-2b RED: unused until E6 adoption. - bool recordTargetOnly = false, // ED-2b RED: unused until D36 adoption. + bool recordTargetOnly = false, }) { // An UNCLAIMED device night (our rows saw under half of it) ending today IS // the night: above our detection, below the user's word. It never reaches diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index f201dcdad..8aefbb876 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -1220,8 +1220,8 @@ class SleepWindowOverride { String localDateLabel(int epochSec) => dayLabelOf(DateTime.fromMillisecondsSinceEpoch(epochSec * 1000)); -// RED controls: the walk still segments every day. The counter observes its -// actual body; GREEN must preserve the full walk for folding callers. +// Walk controls: the counter observes the actual segmentation body; the full +// walk remains available to check the restricted walk's target output. int debugSegmentedDayCount = 0; bool debugStageEveryWalkedDay = false; @@ -1257,8 +1257,8 @@ bool debugStageEveryWalkedDay = false; /// /// With [skipUnusedDays], only [targetDay] is needed. Earlier days can be /// skipped when even one new history key per day cannot reach the prior's -/// 14-night threshold. Callers needing profile observations must keep the -/// full walk: its longest observation can belong to an earlier day. +/// 14-night threshold. Callers needing profile observations can use +/// [recordTargetOnly] to record the target night even during a full walk. List calendarDays( Substrate sub, { SleepWindowOverride? override, @@ -1268,7 +1268,9 @@ List calendarDays( String? targetDay, bool skipUnusedDays = false, ana.CardioRowCache? rowCache, // ED-2b RED: unused until E6 adoption. - bool recordTargetOnly = false, // ED-2b RED: unused until D36 adoption. + // D36: other walked nights must not train under the target's ID. + // Restore the caller's recording flag on every exit. + bool recordTargetOnly = false, }) { final tzOffset = tzOffsetAt ?? tzOffsetSecondsAt; if (sub.isEmpty) return const []; @@ -1296,6 +1298,9 @@ List calendarDays( } skipBefore = {for (final h in sleepHistory) h.dayKey}.length + earlier < 14; } + final recording = ana.cardioRecordObservations; + final onlyTarget = recordTargetOnly && targetDay != null; + try { var guard = 0; while (dayStart < dataEnd && guard++ < 400) { final dayEnd = _nextLocalMidnight(dayStart); @@ -1316,6 +1321,7 @@ List calendarDays( } } debugSegmentedDayCount++; + if (onlyTarget) ana.cardioRecordObservations = recording && dayLabel == targetDay; // The main sleep that ENDS in this calendar day: search from the previous // local noon through this local midnight, then let the sleep selector pick // the overnight main block from any naps / split fragments it sees. The old @@ -1583,6 +1589,9 @@ List calendarDays( )); dayStart = dayEnd; } + } finally { + ana.cardioRecordObservations = recording; + } return days; } diff --git a/test/ed2a_e15_restricted_target_walk_test.dart b/test/ed2a_e15_restricted_target_walk_test.dart index 331d4a43a..61135f276 100644 --- a/test/ed2a_e15_restricted_target_walk_test.dart +++ b/test/ed2a_e15_restricted_target_walk_test.dart @@ -113,7 +113,7 @@ void main() { reason: 'nonfolding production staging must skip the irrelevant earlier day'); }); - test('mayFold always walks earlier days and folds once (guard; D36 excluded)', () async { + test('mayFold skips unused earlier days and folds the target once (D36)', () async { await edFresh('ed2a_e15_fold.db'); await edRows(sub.tsSec.first, sub.tsSec.last + 1, hr: 55, az: 1); final edge = DateTime(2020, 1, 3, 12).millisecondsSinceEpoch ~/ 1000; @@ -123,12 +123,12 @@ void main() { ana.cardioUserProfile = null; ana.cardioRecordObservations = true; ana.resetCardioObservations(); - final full = prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: false); - final observations = ana.takeCardioObservations() - ..sort((a, b) => b.epochs.compareTo(a.epochs)); - expect(observations, isNotEmpty); - final o = observations.first; - final fullObservation = jsonEncode({'epochs': o.epochs, + final full = prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: false, + recordTargetOnly: true); + final observations = ana.takeCardioObservations(); + expect(observations, hasLength(1)); + final o = observations.single; + final targetObservation = jsonEncode({'epochs': o.epochs, 'hr_floor_p5': o.hrFloorP5, 'hr_floor_p25': o.hrFloorP25, 'hr_sleep_median': o.hrSleepMedian, 'hr_arousal': o.hrArousal, 'rmssd_med': o.rmssdMed, 'rmssd_mad': o.rmssdMad, @@ -139,10 +139,10 @@ void main() { (fold, count, obs) => walked.add((fold, count, obs)); final candidate = await e.debugSleepCandidate(edDay); expect(jsonEncode(candidate.toJson()), jsonEncode(full.toJson())); - expect(walked.single.$3, fullObservation, reason: "preserve the full walk's actual fold input"); + expect(walked.single.$3, targetObservation, reason: "D36 folds the target's own observation"); expect(walked.single.$1, isTrue, reason: 'fixture reaches a folding pass'); - expect(walked.single.$2, 2, - reason: 'the longest observation can still belong to the earlier day upstream'); + expect(walked.single.$2, 1, + reason: 'D36 permits the output-neutral E15 skip on folding passes'); expect(walked.single.$3, isNotNull, reason: 'fixture must produce a real observation'); final profile = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; final p = jsonDecode(profile as String) as Map; diff --git a/test/ed2b_d35_ownership_endpoint_test.dart b/test/ed2b_d35_ownership_endpoint_test.dart new file mode 100644 index 000000000..07eb87861 --- /dev/null +++ b/test/ed2b_d35_ownership_endpoint_test.dart @@ -0,0 +1,85 @@ +@Timeout(Duration(minutes: 5)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/adapters/signals.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/coverage_resolver.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +void main() { + setUpAll(edInit); + setUp(() => edFresh('ed2b_d35.db')); + tearDown(edDrop); + + for (final signal in [InputSignal.hr1Hz, InputSignal.rrIntervals]) { + test('$signal inclusive endpoint is covered by the ownership read and spans', () async { + final end = edStart + 600; + final db = await LocalDb.instance; + // This interval starts exactly at the inclusive last sample. A query + // ending at `end` silently omits it, even if resolution is later fixed. + await db.insert('device_coverage', {'device_id': '', 'signal': signal.name, + 'start_ts': end, 'end_ts': end + 1}); + final (ownership, _) = await DerivationEngine().debugResolveOwnership(edStart, end); + expect(spanAt(ownership[signal]!, end)?.deviceId, '', + reason: 'both coverage query and resolution must extend through to+1'); + expect(ownership[signal]!.last.end, end + 1); + }); + } + + test('non-owner last-second HR does not leak into a single-second load', () async { + await edRows(edStart, edStart + 601, hr: 61); + await edRows(edStart + 600, edStart + 601, hr: 199, id: 'unranked'); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': edStart, 'end_ts': edStart + 601}); + final e = DerivationEngine()..debugDisableSubstrateRetention = true; + final last = await e.debugLoadOwnedSubstrate(edStart + 600, edStart + 600, day: edDay); + expect(last.hr, [61], reason: 'a zero-width half-open resolver must not disable the filter'); + }); + + test('non-owner last-second RR beats are excluded by the same inclusive ownership', () async { + await edRows(edStart, edStart + 601, hr: 61); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'rrIntervals', + 'start_ts': edStart, 'end_ts': edStart + 601}); + for (final id in ['', 'unranked']) { + await db.insert('decoded_rr', {'device_id': id, + 'ts_ms': (edStart + 600) * 1000, 'rec_ts': edStart + 600, + 'rr_ts_ms': (edStart + 600) * 1000, 'beat_index': 0, + 'rr_ms': id.isEmpty ? 900 : 1500}); + } + final last = await DerivationEngine().debugLoadOwnedSubstrate( + edStart + 600, edStart + 600, day: edDay); + expect(last.rrMs, [900], reason: 'the other device must not add a beat at to'); + }); + + test('same endpoint loaded alone or inside a larger range has identical owner data', () async { + await edRows(edStart, edStart + 601, hr: 61); + await edRows(edStart + 600, edStart + 601, hr: 199, id: 'unranked'); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': edStart, 'end_ts': edStart + 601}); + final e = DerivationEngine()..debugDisableSubstrateRetention = true; + final wide = await e.debugLoadOwnedSubstrate(edStart, edStart + 601, day: edDay); + final alone = await e.debugLoadOwnedSubstrate(edStart + 600, edStart + 600, day: edDay); + expect(jsonEncode(alone.toJson()), + jsonEncode(wide.slice(edStart + 600, edStart + 601).toJson())); + }); + + test('interior ownership stays primary and absent owner stays absent (guard)', () async { + await edRows(edStart, edStart + 600, hr: 61); + await edRows(edStart, edStart + 600, hr: 199, id: 'unranked'); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': edStart, 'end_ts': edStart + 600}); + final sub = await DerivationEngine().debugLoadOwnedSubstrate(edStart, edStart + 600, day: edDay); + expect(sub.hr, everyElement(61)); + final (ownership, _) = await DerivationEngine().debugResolveOwnership(edStart - 60, edStart - 1); + expect(ownership[InputSignal.hr1Hz], everyElement(predicate((s) => s.deviceId == null))); + }); +} diff --git a/test/ed2b_d36_target_profile_test.dart b/test/ed2b_d36_target_profile_test.dart new file mode 100644 index 000000000..843f67f51 --- /dev/null +++ b/test/ed2b_d36_target_profile_test.dart @@ -0,0 +1,195 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/sleep_profile_policy.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +final _mid = DateTime(2020, 1, 2).millisecondsSinceEpoch ~/ 1000; +int _h(int hour) => _mid + hour * 3600; + +// Longer previous night, shorter target, real motion and HR contrast. No +// forced/manual window: those deliberately cannot train the profile. +Substrate _night({bool target = true, bool earlierEvening = false}) { + final ts = [], hr = []; + final ax = [], ay = [], az = []; + for (var t = _h(-36); t < _h(30); t++) { + final still = (t >= _h(earlierEvening ? -12 : -26) && + t < _h(earlierEvening ? -2 : -17)) || + (target && t >= _h(-1) && t < _h(5)); + ts.add(t); + hr.add(still ? 52 + t % 7 : 80 + (t ~/ 60) % 20 + t % 3); + ax.add(still ? 0 : .3 * math.sin(t * .21)); + ay.add(still ? 0 : .2 * math.cos(t * .13)); + az.add(still ? 1 : 1 + .05 * math.sin(t * .07)); + } + return Substrate.fromJson({'ts_sec': ts, 'hr': hr, 'ax': ax, 'ay': ay, + 'az': az, 'device_family': 'gen4'}); +} + +String _observation(ana.SleepNightObservation o) => jsonEncode({ + 'epochs': o.epochs, 'hr_floor_p5': o.hrFloorP5, 'hr_floor_p25': o.hrFloorP25, + 'hr_sleep_median': o.hrSleepMedian, 'hr_arousal': o.hrArousal, + 'rmssd_med': o.rmssdMed, 'rmssd_mad': o.rmssdMad, + 'enmo_still_cut': o.enmoStillCut, 'enmo_move_cut': o.enmoMoveCut, + 'lfhf_med': o.lfhfMed, 'rk_med': o.rkMed}); + +Future _save(Substrate s) async { + final b = (await LocalDb.instance).batch(); + for (var i = 0; i < s.length; i++) { + b.insert('decoded_onehz', {'device_id': '', 'ts_ms': s.tsSec[i] * 1000, + 'rec_ts': s.tsSec[i], 'counter': i, 'hr': s.hr[i], 'ax': s.ax[i], + 'ay': s.ay[i], 'az': s.az[i], 'device_family': 'gen4'}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + await b.commit(noResult: true); +} + +void main() { + setUpAll(edInit); + setUp(() => edFresh('ed2b_d36.db')); + tearDown(() async { + ana.cardioRecordObservations = false; + ana.cardioUserProfile = null; + ana.resetCardioObservations(); + await edDrop(); + }); + + for (final target in [true, false]) { + test('recordTargetOnly: ${target ? "one target observation" : "no target means none"}', () { + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + final sub = _night(target: target); + // An existing exact restricted walk supplies an independent oracle; the + // full walk must retain history while excluding its training side effects. + calendarDays(sub, targetDay: edDay, skipUnusedDays: true); + final expected = ana.takeCardioObservations().map(_observation).toList(); + expect(expected.length, target ? 1 : 0, reason: 'fixture sanity'); + ana.resetCardioObservations(); + final days = calendarDays(sub, targetDay: edDay, skipUnusedDays: false, + recordTargetOnly: true); + expect(days.length, greaterThan(1), reason: 'history walk remains real'); + expect(ana.takeCardioObservations().map(_observation).toList(), expected, + reason: 'earlier and later walked nights must never train under the target ID'); + expect(ana.cardioRecordObservations, isTrue); + }); + } + + test('production mayFold worker returns the target observation, never the longest walked night', () async { + // Production's folding pass must use the same exact target-only walk as + // the nonfolding path, and report that target's own training input. + final sub = _night(); + await _save(sub); + final e = DerivationEngine(); + final range = e.debugTargetDayWindow(edDay); + final search = await e.debugLoadSubstrate(range.$1, range.$2, day: edDay); + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: true); + final target = ana.takeCardioObservations(); + expect(target.length, 1, reason: 'the target has a usable real night'); + final observed = []; + e.debugOnStagingWalk = (fold, count, observation) { + expect(fold, isTrue); + expect(count, 1, reason: 'target-only folding permits the E15 skip during training'); + observed.add(observation); + }; + await e.debugSleepCandidate(edDay); + expect(observed, [_observation(target.single)]); + }); + + test('production absent target never folds the earlier walked evening under its ID', () async { + await _save(_night(target: false, earlierEvening: true)); + final observations = []; + final e = DerivationEngine()..debugOnStagingWalk = + (fold, _, observation) { + expect(fold, isTrue); + observations.add(observation); + }; + final candidate = await e.debugSleepCandidate(edDay); + expect(candidate.present, isFalse, reason: 'the only sleep ended before target midnight'); + expect(observations, [null], reason: 'no target observation means no training'); + expect(await LocalDb.baseline('sleep_user_profile'), isNull); + }); + + test('already folded profile and folded_days survive version adoption byte-for-byte (guard)', () async { + await _save(_night()); + final profile = jsonEncode({...const ana.SleepUserProfile().toJson(), + 'nights': 7, 'folded_days': ['2019-12-31', edDay]}); + await LocalDb.putBaseline('sleep_user_profile', profile); + final before = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; + await DerivationEngine().debugSleepCandidate(edDay); + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], before, + reason: 'D36 is forward-only; existing accumulated evidence is not reset or rebuilt'); + }); + + test('fold flags restored when segmentation throws (guard)', () { + for (final initial in [true, false]) { + ana.cardioRecordObservations = initial; + expect(() => calendarDays(_night(), targetDay: edDay, recordTargetOnly: true, + priorSleep: [for (var i = 1; i <= 14; i++) + (startSec: _h(-24 * i - 1), endSec: _h(-24 * i + 5), + dayKey: '2019-12-${i.toString().padLeft(2, '0')}')], + tzOffsetAt: (_) => throw StateError('offset failure')), throwsStateError); + expect(ana.cardioRecordObservations, initial); + } + }); + + test('disabled recording remains disabled on successful target walk (guard)', () { + ana.cardioRecordObservations = false; + ana.resetCardioObservations(); + calendarDays(_night(), targetDay: edDay, recordTargetOnly: true); + expect(ana.cardioRecordObservations, isFalse); + expect(ana.takeCardioObservations(), isEmpty); + }); + + + test('actual transactional folds preserve explicit deterministic order and idempotent replay (guard)', () async { + final inputs = [ + ('2020-01-01', _observation(const ana.SleepNightObservation( + epochs: 480, hrFloorP5: 51, hrSleepMedian: 56))), + (edDay, _observation(const ana.SleepNightObservation( + epochs: 720, hrFloorP5: 58, hrSleepMedian: 63))), + ]; + final profiles = []; + for (var repeat = 0; repeat < 2; repeat++) { + final db = await LocalDb.instance; + await db.delete('baselines', where: 'key = ?', whereArgs: ['sleep_user_profile']); + final order = []; + final e = DerivationEngine()..debugOnProfileFold = order.add; + for (final input in inputs) { + await e.debugFoldObservation(input.$1, input.$2); + } + expect(order, inputs.map((input) => input.$1).toList()); + final before = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; + for (final input in inputs.reversed) { + await e.debugFoldObservation(input.$1, input.$2); + } + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], before, + reason: 'already folded nights leave the whole profile byte-identical'); + final profile = jsonDecode(before as String) as Map; + profile.remove('updated_at_ms'); // operational fold timestamp only + profiles.add(jsonEncode(profile)); + } + expect(profiles.last, profiles.first); + }); + + test('folded day bookkeeping is deterministic regardless of insertion order (guard)', () { + final a = SleepProfilePolicy.withFoldedDays({'nights': 8}, + {'2020-01-01', '2019-12-30'}, edDay); + final b = SleepProfilePolicy.withFoldedDays({'nights': 8}, + {'2019-12-30', '2020-01-01'}, edDay); + expect(jsonEncode(a), jsonEncode(b)); + expect(a['folded_days'], ['2019-12-30', '2020-01-01', edDay]); + }); +} diff --git a/test/multidevice_coverage_derive_test.dart b/test/multidevice_coverage_derive_test.dart index 99d555102..6fb3c277c 100644 --- a/test/multidevice_coverage_derive_test.dart +++ b/test/multidevice_coverage_derive_test.dart @@ -134,10 +134,10 @@ Future>> _expectedOwnership() async { for (final sig in const [InputSignal.hr1Hz, InputSignal.rrIntervals]) { final raw = await LocalDb.signalPriority(sig); out[sig] = resolveOwnership( - coverage: await LocalDb.coverageIntervals(sig, unionFrom, unionTo), + coverage: await LocalDb.coverageIntervals(sig, unionFrom, unionTo + 1), priority: raw.isEmpty ? const [LocalDb.kPrimaryDeviceId] : raw, from: unionFrom, - to: unionTo, + to: unionTo + 1, signal: sig, ); } From db9436be5d75d675583c3f3127bb319064db5ce3 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 11:22:29 -0600 Subject: [PATCH 03/11] derive: share cardio rows across staging paths; keep the night's RR in memory; hash shared lists once per pass - One CardioRowCache serves every staging path: the engine keeps the encoded bytes, workers decode, merge and prune the rows. - The night's corrected RR stays in a RAM owner between passes, keyed by the day's input revisions, store generation and algorithm. - Shared lists are fingerprinted once per pass; the identity memo is cleared between passes. Outputs are unchanged. --- lib/compute/cardio_rows_cache.dart | 46 ++++ lib/compute/derivation_engine.dart | 139 +++++++++++- lib/compute/derive_prepare.dart | 4 +- lib/compute/ed2b_stream_diagnostics.dart | 26 +-- lib/compute/night_rr_state.dart | 154 +++++++++++++ lib/compute/onehz_pipeline.dart | 83 +++++-- lib/compute/substrate.dart | 6 +- test/ed2b_cardio_cache_test.dart | 277 +++++++++++++++++++++++ 8 files changed, 689 insertions(+), 46 deletions(-) create mode 100644 lib/compute/cardio_rows_cache.dart create mode 100644 lib/compute/night_rr_state.dart create mode 100644 test/ed2b_cardio_cache_test.dart diff --git a/lib/compute/cardio_rows_cache.dart b/lib/compute/cardio_rows_cache.dart new file mode 100644 index 000000000..db39b7844 --- /dev/null +++ b/lib/compute/cardio_rows_cache.dart @@ -0,0 +1,46 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/onehz.dart' as ana; + +/// Caller-owned bytes and provenance. Row objects exist only in the worker. +class CardioRowsChunk { + CardioRowsChunk(this.bytes, this.loSec, this.hiSec, this.revisions, + this.generation, this.algorithm); + final Uint8List bytes; + final int loSec, hiSec; + final Map revisions; + int generation, algorithm; + + bool overlaps(int lo, int hi) => hiSec >= lo && loSec <= hi; + + bool matches(Map current, int gen, int algo) => + generation == gen && algorithm == algo && + current.entries.every((e) => + !revisions.containsKey(e.key) || revisions[e.key] == e.value); +} + +/// Worker only. Damage or an unknown codec version costs a cold staging. +ana.CardioRowCache decodeCardioRows(List chunks) { + final rows = ana.CardioRowCache(); + for (final bytes in chunks) { + try { + rows.merge(ana.CardioRowCache.fromJson( + (jsonDecode(utf8.decode(bytes)) as Map).cast())); + } on FormatException { + // Refuse the whole chunk. + } on TypeError { + // A structurally invalid JSON value is also a miss. + } + } + return rows; +} + +/// Keep at most one search window plus slack and 8,000 row variants. A limit +/// only evicts reusable work; analytics always recomputes a missing row. +Uint8List encodeCardioRows(ana.CardioRowCache rows, int lo, int hi) { + var kept = 0; + rows.retainWhere((k) => + k.startSec >= lo && k.startSec <= hi && kept++ < 8000); + return Uint8List.fromList(utf8.encode(jsonEncode(rows.toJson()))); +} diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index a90949e20..fa5b28e4f 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -75,6 +75,8 @@ import 'sleep_profile_policy.dart'; import 'ed2b_checkpoint_diagnostics.dart'; import 'staged_sleep_cache.dart'; import 'ed2b_stream_diagnostics.dart'; +import 'cardio_rows_cache.dart'; +import 'night_rr_state.dart'; import 'substrate_retention.dart'; import 'derive_prepare.dart'; import 'onehz_pipeline.dart'; @@ -3126,7 +3128,7 @@ class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; - // ED-2b RED observations. Controls stay inert until their GREEN phase. + // ED-2b observations. Checkpoint/union controls await their GREEN phase. final debugCheckpoints = Ed2bCheckpointDiagnostics(); @visibleForTesting void Function(String day)? debugOnProfileFold; @@ -3134,14 +3136,14 @@ class DerivationEngine { Future debugFoldObservation(String day, String observation) => _foldObservationIntoProfile(day, observation); @visibleForTesting - // ED-2b RED: optional observations follow the real day-bundle worker, - // carrying counters only. No night stream is retained or enabled here. + // Optional observations follow the real day-bundle worker and its RAM owner. Ed2bStreamDiagnostics? debugNightStreams; @visibleForTesting void Function(Map input, Map output)? debugOnNightBundle; @visibleForTesting - Uint8List? debugCardioRowBytes; + Uint8List? get debugCardioRowBytes => + _cardioRowChunks.isEmpty ? null : _cardioRowChunks.last.bytes; @visibleForTesting bool debugDisableCardioRows = false; @visibleForTesting @@ -3149,10 +3151,16 @@ class DerivationEngine { @visibleForTesting int debugCardioRowMisses = 0; @visibleForTesting - List get debugCardioRowWorkerOperations => const []; + List get debugCardioRowWorkerOperations => _cardioRowOperations.toList(); @visibleForTesting void debugRekeyCardioRows({required int inputRevision, - required int generation, required int algorithm}) {} + required int generation, required int algorithm}) { + for (final c in _cardioRowChunks) { + c.generation = generation; + c.algorithm = algorithm; + c.revisions.updateAll((_, fp) => (inputRevision, fp.$2, generation)); + } + } @visibleForTesting bool debugDisableUnionLoads = false; @visibleForTesting @@ -3214,6 +3222,11 @@ class DerivationEngine { } final _retained = SubstrateRetention(); + final _cardioRowChunks = []; + final _cardioRowOperations = {}; + final _nightStates = {}; + static const _cardioRowsSlackSec = 3600; + static const _cardioRowsMaxBytes = 8 * 1024 * 1024; int _retentionEpoch = 0; int? _retentionPassEpoch; bool _retentionInBackground = false; @@ -3226,6 +3239,9 @@ class DerivationEngine { void _clearRetainedSubstrate() { _retentionEpoch++; _retained.clear(); + _cardioRowChunks.clear(); + _nightStates.clear(); + debugNightStreams?.nightState.clear(); } void _beginRetentionPass() { @@ -3236,12 +3252,18 @@ class DerivationEngine { void _endRetentionPass() { _retained.retainDays(_retentionPreparedDays); + _nightStates.removeWhere((day, state) { + if (_retentionPreparedDays.contains(day)) return false; + state.clear(); + return true; + }); _retentionPreparedDays.clear(); _retentionPassEpoch = null; } void _dropRetainedDay(String day) { _retained.remove(day); + _nightStates.remove(day)?.clear(); _retentionPreparedDays.remove(day); } @visibleForTesting @@ -4598,8 +4620,10 @@ class DerivationEngine { // a night that was never scored. final (searchOwnership, searchPriority) = await _resolveOwnership(range.$1, range.$2); + final rowsEpoch = _retentionEpoch; List>? revisions; - if (!debugDisableStagedSleepCache) { + if (!debugDisableStagedSleepCache || + (!debugDisableCardioRows && _canRetainSubstrate)) { try { revisions = []; final first = DateTime.fromMillisecondsSinceEpoch(range.$1 * 1000); @@ -4687,7 +4711,7 @@ class DerivationEngine { : null; if (ownNight != null) vendorNights = [...vendorNights, ownNight]; String? stagingKey; - if (revisions != null) { + if (revisions != null && !debugDisableStagedSleepCache) { try { stagingKey = stagingInputKey( fromSec: range.$1, @@ -4715,7 +4739,24 @@ class DerivationEngine { // the context the worker closure ships, and the engine's callbacks (the // app's own fields) are not sendable. final stageTimeout = _perDayTimeout; - Future<(String, String?, int)> stage() => _runIsolateCancellable(() { + final cacheRows = !debugDisableCardioRows && _canRetainSubstrate && + rowsEpoch == _retentionEpoch && revisions != null; + final rowRevisions = { + for (final r in revisions ?? const >[]) + r[0] as String: (r[1] as int, r[2] as int, r[3] as int), + }; + final rowGeneration = LocalDb.storeGeneration; + final rowsLo = range.$1 - _cardioRowsSlackSec; + final rowsHi = range.$2 + _cardioRowsSlackSec; + _cardioRowChunks.removeWhere((c) => + c.generation != rowGeneration || c.algorithm != kAlgoVersion || + (c.overlaps(rowsLo, rowsHi) && + !c.matches(rowRevisions, rowGeneration, kAlgoVersion))); + final sentChunks = cacheRows ? [for (final c in _cardioRowChunks) + if (c.overlaps(rowsLo, rowsHi)) c] : []; + final rowBytes = [for (final c in sentChunks) c.bytes]; + Future<(String, String?, int, Uint8List?, int, int, List)> stage() => + _runIsolateCancellable(() { try { final p = profileJson == null ? null @@ -4737,6 +4778,14 @@ class DerivationEngine { ana.cardioRecordObservations = true; ana.resetCardioObservations(); debugSegmentedDayCount = 0; + final rowOperations = []; + final ana.CardioRowCache? rowCache; + if (cacheRows) { + rowCache = decodeCardioRows(rowBytes); + rowOperations.addAll(['decode:worker', 'merge:worker']); + } else { + rowCache = null; + } final candidate = prepareSleepSessionCandidate( searchSub, targetDay: dayId, @@ -4745,6 +4794,7 @@ class DerivationEngine { override: override, priorSleep: priorSleep, vendorNights: vendorNights, + rowCache: rowCache, ); // Fold the TARGET sleep (most epochs of its own observations) into the // rolling profile — done here in the worker because the observations live @@ -4786,7 +4836,12 @@ class DerivationEngine { } } } - return (jsonEncode(candidate.toJson()), observationJson, debugSegmentedDayCount); + final rowsOut = rowCache == null ? null : + encodeCardioRows(rowCache, rowsLo, rowsHi); + if (rowCache != null) rowOperations.add('prune:worker'); + return (jsonEncode(candidate.toJson()), observationJson, + debugSegmentedDayCount, rowsOut, rowCache?.hits ?? 0, + rowCache?.misses ?? 0, rowOperations); }, stageTimeout, label: 'sleep-staging $dayId'); final hit = stagingKey == null ? null : _stagedSleepCache.lookup( dayId, kAlgoVersion, stagingKey, mayFold: mayFold); @@ -4801,6 +4856,26 @@ class DerivationEngine { final result = await stage(); candidateJson = result.$1; observationJson = result.$2; + debugCardioRowHits += result.$5; + debugCardioRowMisses += result.$6; + _cardioRowOperations.addAll(result.$7); + if (_canRetainSubstrate && rowsEpoch == _retentionEpoch && + LocalDb.storeGeneration == rowGeneration && !debugDisableCardioRows) { + _cardioRowChunks.removeWhere(sentChunks.contains); + if (result.$4 != null && result.$4!.length <= _cardioRowsMaxBytes) { + _cardioRowChunks.add(CardioRowsChunk(result.$4!, rowsLo, rowsHi, + rowRevisions, rowGeneration, kAlgoVersion)); + } + // Bound both disjoint windows and repeated fingerprint variants. + final newest = _cardioRowChunks.fold(rowsHi, + (hi, c) => math.max(hi, c.hiSec)); + _cardioRowChunks.removeWhere((c) => c.hiSec < newest - 4 * 24 * 3600); + while (_cardioRowChunks.length > 4 || + _cardioRowChunks.fold(0, (n, c) => n + c.bytes.length) > + _cardioRowsMaxBytes) { + _cardioRowChunks.removeAt(0); + } + } debugOnStagingWalk?.call(mayFold, result.$3, observationJson); if (stagingKey != null) { _stagedSleepCache.store(dayId, StagedSleepEntry( @@ -6434,14 +6509,54 @@ class DerivationEngine { // Cancellable: on timeout the isolate is KILLED, not merely abandoned to // keep burning a core behind the worker pool's back. + final nightEpoch = _retentionEpoch; + final retainNight = _canRetainSubstrate && !suppliedSubstrate; final diagnostics = debugNightStreams; + // Keep DB reads and engine fields outside the shipped closure. A changed + // revision rejects even a byte-identical replacement; raw prefix proofs in + // the worker additionally detect moved windows and in-memory mutations. + NightRrState? nightState; + if ((retainNight && sleepSub.rrMs.isNotEmpty) || diagnostics != null) { + nightState = _nightStates.remove(day.date) ?? NightRrState(); + try { + var revision = 0; + var generation = LocalDb.storeGeneration; + final first = DateTime.fromMillisecondsSinceEpoch( + day.sleepOnsetSec * 1000); + final last = day.sleepOffsetSec > day.sleepOnsetSec + ? day.sleepOffsetSec - 1 : day.sleepOnsetSec; + for (var date = DateTime(first.year, first.month, first.day); + date.millisecondsSinceEpoch ~/ 1000 <= last; + date = DateTime(date.year, date.month, date.day + 1)) { + final fp = await LocalDb.inputDayFingerprint(dayLabelOf(date)); + revision += fp.revisionSum; + generation = fp.generation; + } + nightState.context(inputRevision: revision, generation: generation, + algorithm: kAlgoVersion); + } catch (_) { + nightState.clear(); // Unavailable provenance never certifies reuse. + } + diagnostics?.nightState = nightState; + } final workerResult = await _runIsolateCancellable( - () => (deriveDayBundle(withHistory, streamDiagnostics: diagnostics), diagnostics), + () => (deriveDayBundle(withHistory, streamDiagnostics: diagnostics, + nightState: nightState), nightState, diagnostics), _perDayTimeout, label: 'day-bundle ${day.date}', ); final bundle = workerResult.$1; - if (diagnostics != null) debugNightStreams = workerResult.$2; + if (diagnostics != null) debugNightStreams = workerResult.$3; + final returnedNight = workerResult.$2; + if (returnedNight != null && retainNight && _canRetainSubstrate && + nightEpoch == _retentionEpoch) { + _nightStates[day.date] = returnedNight; + while (_nightStates.length > math.min(2, _retained.maxDays)) { + _nightStates.remove(_nightStates.keys.first)?.clear(); + } + } else { + returnedNight?.clear(); + } debugOnNightBundle?.call(withHistory, bundle); // Readiness came back absent for TODAY specifically (not a historical // backfill day, which would just be noise) — log why. This ran inside diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index 912f9b853..d398018e5 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -415,7 +415,7 @@ PreparedDerivationPayload prepareDerivationPayload( List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], bool skipUnusedDays = false, - ana.CardioRowCache? rowCache, // ED-2b RED: unused until E6 adoption. + ana.CardioRowCache? rowCache, bool recordTargetOnly = false, }) { if (sub.isEmpty || sub.lastTs == null) { @@ -484,7 +484,7 @@ SleepSessionCandidate prepareSleepSessionCandidate( List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], bool skipUnusedDays = false, - ana.CardioRowCache? rowCache, // ED-2b RED: unused until E6 adoption. + ana.CardioRowCache? rowCache, bool recordTargetOnly = false, }) { // An UNCLAIMED device night (our rows saw under half of it) ending today IS diff --git a/lib/compute/ed2b_stream_diagnostics.dart b/lib/compute/ed2b_stream_diagnostics.dart index 69b2a14a9..cd37f3d27 100644 --- a/lib/compute/ed2b_stream_diagnostics.dart +++ b/lib/compute/ed2b_stream_diagnostics.dart @@ -1,10 +1,11 @@ -// ED-2b RED observations only. This object retains no analytics state and never -// changes a result. Work is counted beside the existing batch calls. GREEN can -// replace those calls with streams while preserving these measurement meanings. +import 'night_rr_state.dart'; + +// Work is counted beside the real owner and the remaining batch metric calls. class Ed2bStreamDiagnostics { int passes = 0; - int processedNightBeats = 0; - int nightRebuilds = 0; + NightRrState nightState = NightRrState(); + int get processedNightBeats => nightState.processedNightBeats; + int get nightRebuilds => nightState.nightRebuilds; final Map _batchBeats = {}; final Map _batchCalls = {}; bool disabled = false; @@ -16,14 +17,15 @@ class Ed2bStreamDiagnostics { void beginPass() { passes++; } void endPass() { onBeforePassReturn?.call(); } void context({required int inputRevision, required int generation, - required int algorithm}) {} // RED: no retained state to invalidate. + required int algorithm}) => nightState.context(inputRevision: inputRevision, + generation: generation, algorithm: algorithm); // Total input work: GREEN counts only newly appended/settled beats here, // regardless of whether the operation uses a batch call or an addSlice. - int get retainedNightBeats => 0; - int get settledNightBeats => 0; - int get identityMemoEntries => 0; - int get fullListDigests => 0; - int get maximumListDigestsPerIdentity => 0; + int get retainedNightBeats => nightState.retainedNightBeats; + int get settledNightBeats => nightState.settledNightBeats; + int get identityMemoEntries => nightState.identityMemoEntries; + int get fullListDigests => nightState.fullListDigests; + int get maximumListDigestsPerIdentity => nightState.maximumListDigestsPerIdentity; int retainedStreamBeats(String stream) => 0; int processedBeats(String stream) => _batchBeats[stream] ?? 0; int slicesProcessed(String stream) => _batchCalls[stream] ?? 0; @@ -31,8 +33,6 @@ class Ed2bStreamDiagnostics { void observeNightCorrection(int rawBeats, List nn, List times, double cleanFraction) { - processedNightBeats += rawBeats; - if (rawBeats > 0) nightRebuilds++; onCorrected?.call(nn, times, cleanFraction); } diff --git a/lib/compute/night_rr_state.dart b/lib/compute/night_rr_state.dart new file mode 100644 index 000000000..d17adc59a --- /dev/null +++ b/lib/compute/night_rr_state.dart @@ -0,0 +1,154 @@ +import 'dart:collection'; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/onehz.dart' as ana; + +typedef NightRr = ({List nn, List nnTimesMs, + double cleanFraction, int droppedCount, int correctedCount}); +typedef NightListProof = ({int beats, int times, (int, int) nn, (int, int) ts}); + +NightRr nightRrBatch(List rr, List times) { + final c = ana.correctRr(rr, rrTsMs: times); + return (nn: c.nn, nnTimesMs: c.nnTimesMs, cleanFraction: c.cleanFraction, + droppedCount: c.droppedCount, correctedCount: c.correctedCount); +} + +/// One RAM night, copied into a worker and returned with its result. Raw +/// prefix copies prove append-only input; settled output is never re-corrected. +class NightRrState { + static const maxRetainedBeats = 200000; + ana.RrCorrector? _corrector; + final List _rr = [], _times = [], _nn = [], _nnTimes = []; + (int, int, int)? _context; + (String, int)? _night; + final _memos = {}; + final _digests = _PassListDigests(); + int processedNightBeats = 0, nightRebuilds = 0; + + int get retainedNightBeats => _rr.length; + int get settledNightBeats => _nn.length; + int get identityMemoEntries => _digests.entries; + int get fullListDigests => _digests.computed; + int get maximumListDigestsPerIdentity => _digests.maximumPerIdentity; + + void beginPass() => _digests.clear(); + void endPass() => _digests.clear(); + + void context({required int inputRevision, required int generation, + required int algorithm}) { + final next = (inputRevision, generation, algorithm); + if (_context != next) clear(); + _context = next; + } + + void clear() { + _resetNight(); + _digests.clear(); + } + + void _resetNight() { + _corrector = null; + _rr.clear(); _times.clear(); _nn.clear(); _nnTimes.clear(); + _night = null; + _memos.clear(); + } + + /// Shared by the night metric memos. No digest outlives this pass's + /// identities; only its integer words may be kept as a dependency proof. + NightListProof dependencyProof(List nn, List times) => + (beats: nn.length, times: times.length, + nn: _digests.digest(nn), ts: _digests.digest(times)); + + NightRr correct(List rr, List times, + {required String day, required int onsetSec}) { + final night = (day, onsetSec); + var held = _rr.length; + var extendsPrefix = _night == night && rr.length >= held && + times.length == rr.length; + for (var i = 0; extendsPrefix && i < held; i++) { + extendsPrefix = identical(rr[i], _rr[i]) && identical(times[i], _times[i]); + } + if (!extendsPrefix) { + _resetNight(); + held = 0; + } + // Keep malformed/oversized input on the unchanged batch path. Eviction + // affects work only, never absence rules or numeric output. + if (rr.isEmpty || times.length != rr.length || + rr.length > maxRetainedBeats) { + _resetNight(); + processedNightBeats += rr.length; + return nightRrBatch(rr, times); + } + _night = night; + if (_corrector == null) nightRebuilds++; + final corrector = _corrector ??= ana.RrCorrector(); + if (rr.length > held) { + final addRr = held == 0 ? rr : rr.sublist(held); + final addTimes = held == 0 ? times : times.sublist(held); + final ana.RrSettled settled; + try { + settled = corrector.fold(addRr, tsMs: addTimes); + } on ArgumentError { + _resetNight(); + processedNightBeats += rr.length; + return nightRrBatch(rr, times); + } + processedNightBeats += addRr.length; + _rr.addAll(addRr); _times.addAll(addTimes); + _nn.addAll(settled.nn); _nnTimes.addAll(settled.nnTimes); + } + final tail = corrector.snapshot(); + return (nn: [..._nn, ...tail.tailNn], + nnTimesMs: [..._nnTimes, ...tail.tailNnTimes], + cleanFraction: tail.cleanFraction, droppedCount: tail.droppedCount, + correctedCount: tail.correctedCount); + } + + /// Exact batch-result reuse. Growing lists still run the ordinary batch + /// consumers; incremental metric streams are a separate adoption step. + T memo(String key, Object dependencies, T Function() compute) { + if (_corrector == null) return compute(); + final old = _memos[key]; + if (old != null && old.dependencies == dependencies) return old.value as T; + final result = compute(); + if (!_memos.containsKey(key) && _memos.length >= 8) { + _memos.remove(_memos.keys.first); + } + _memos[key] = (dependencies: dependencies, value: result); + return result; + } +} + +/// Short window lists are walked directly. Only long shared lists occupy +/// identity keys, and the pipeline releases all keys in finally. +class _PassListDigests { + final HashMap, (int, int, int, int)> _byList = HashMap.identity(); + final _bits = ByteData(8); + int computed = 0, maximumPerIdentity = 0; + int get entries => _byList.length; + void clear() => _byList.clear(); + + (int, int) digest(List list) { + final hit = _byList[list]; + if (hit != null && hit.$1 == list.length) return (hit.$2, hit.$3); + var a = 0x243f6a8885a308d3, b = 0x13198a2e03707344; + for (final value in list) { + _bits.setFloat64(0, value); + final word = _bits.getInt64(0); + a = _mix(a + word + 0x9e3779b97f4a7c15); + b = _mix((b ^ word) * 0xd6e8feb86659fd93 + 0x632be59bd9b4e019); + } + computed++; + final count = (hit?.$4 ?? 0) + 1; + if (count > maximumPerIdentity) maximumPerIdentity = count; + if (list.length >= 2048) _byList[list] = (list.length, a, b, count); + return (a, b); + } + + static int _mix(int z) { + z = (z ^ (z >>> 30)) * 0xbf58476d1ce4e5b9; + z = (z ^ (z >>> 27)) * 0x94d049bb133111eb; + return z ^ (z >>> 31); + } +} diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 96233c53e..4f15610c8 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -33,6 +33,7 @@ import 'package:openstrap_analytics/onehz.dart'; import 'inputs/garmin_inputs.dart' show isWearableFamily, kHrCadenceSec, kOneHzRunGapSec; import 'ed2b_stream_diagnostics.dart'; +import 'night_rr_state.dart'; import 'hr_max.dart' show estimatedMaxHr, @@ -548,8 +549,23 @@ class DayBundleInput { /// `Isolate.run(() => deriveDayBundle(input))` in production. Map deriveDayBundle(Map inputJson, { Ed2bStreamDiagnostics? streamDiagnostics, + NightRrState? nightState, }) { + final state = streamDiagnostics?.disabled == true + ? null : nightState ?? streamDiagnostics?.nightState; streamDiagnostics?.beginPass(); + state?.beginPass(); + try { + final bundle = _deriveDayBundle(inputJson, state, streamDiagnostics); + streamDiagnostics?.endPass(); + return bundle; + } finally { + state?.endPass(); + } +} + +Map _deriveDayBundle(Map inputJson, + NightRrState? nightState, Ed2bStreamDiagnostics? streamDiagnostics) { final d = DayBundleInput.fromJson(inputJson); // ── HR over the DAY (for the curve, strain zones, dip day-side) ─────────── @@ -595,12 +611,21 @@ Map deriveDayBundle(Map inputJson, { // HRV/RHR are rest/sleep-only per the catalog. Running correctRr+hrvTime over // the SLEEP RR (not the whole day) is what brings RMSSD back to physiological // tens-of-ms instead of the whole-day ~166 ms inflated value. - final corrected = correctRr(d.sleepRrMs, rrTsMs: d.sleepRrTsMs); + final corrected = nightState == null + ? nightRrBatch(d.sleepRrMs, d.sleepRrTsMs) + : nightState.correct(d.sleepRrMs, d.sleepRrTsMs, + day: d.date, onsetSec: d.sleepOnsetSec); streamDiagnostics?.observeNightCorrection(d.sleepRrMs.length, corrected.nn, corrected.nnTimesMs, corrected.cleanFraction); final nn = corrected.nn; final nnTimes = corrected.nnTimesMs; final artifactFraction = (1.0 - corrected.cleanFraction).clamp(0.0, 1.0); + final nnProof = nightState?.dependencyProof(nn, nnTimes); + T nightMemo(String key, Object? extra, T Function() compute) => + nnProof == null ? compute() : + nightState!.memo(key, (nnProof, extra), compute); + Map nightJson(Map json) => + nightState == null ? json : _copyNightJson(json); // Σ RR ÷ wall span of the raw sleep RR: above 1.10 the stream holds more // beat-time than elapsed (duplicated or interleaved beats), and every RMSSD // below refuses it. Null under 10 min of span. @@ -659,13 +684,15 @@ Map deriveDayBundle(Map inputJson, { final sleepSessionRmssd = sleepSessionRmssdMetric.present ? sessionDetail!.rmssd : null; final hrvF = nn.length >= 20 - ? hrvFreq(nn, nnTimes, artifactFraction: artifactFraction) + ? nightMemo('frequency', artifactFraction, () { + streamDiagnostics?.observeBatch('frequency', nn.length); + return hrvFreq(nn, nnTimes, artifactFraction: artifactFraction); + }) : Metric.absent( tier: Tier.high, inputs_used: const ['rr_cleaned'], note: needInputNote('nn_beats', have: nn.length, need: 20), ); - if (nn.length >= 20) streamDiagnostics?.observeBatch('frequency', nn.length); // Nocturnal RHR over the SLEEP HR. NO SLEEP ⇒ NO RESTING HR. // // This used to fall back to the whole-day HR when no sleep was scored, and @@ -713,9 +740,12 @@ Map deriveDayBundle(Map inputJson, { // HR dip: day-side = waking HR outside the sleep window; night-side = sleep HR. final dayOnly = _dayHrOutsideSleep(d); final dip = hrDip(dayOnly, sleepHr); - final dc = decelerationCapacity(nn); - final ac = accelerationCapacity(nn); - streamDiagnostics?.observeBatch('prsa', nn.length); + final prsa = nightMemo('prsa', null, () { + streamDiagnostics?.observeBatch('prsa', nn.length); + return (decelerationCapacity(nn), accelerationCapacity(nn)); + }); + final dc = prsa.$1; + final ac = prsa.$2; // Baevsky Stress Index over the sleep NN — resting autonomic tension (a // transparent RR-histogram metric; no ML). Daily resting-stress indicator. // nnTimesMs lets it segment at a charging/off-wrist gap instead of letting @@ -725,21 +755,25 @@ Map deriveDayBundle(Map inputJson, { // ── RESPIRATION (sleep-windowed) ─────────────────────────────────────────── final resp = nn.length >= 30 - ? rsaRespRate(nn, nnTimes, artifactFraction: artifactFraction) + ? nightMemo('rsa', artifactFraction, () { + streamDiagnostics?.observeBatch('rsa', nn.length); + return rsaRespRate(nn, nnTimes, artifactFraction: artifactFraction); + }) : Metric.absent( tier: Tier.estimate, inputs_used: const ['rr_cleaned'], note: needInputNote('nn_beats', have: nn.length, need: 30), ); final cvhr = nn.length >= 60 - ? cvhrApneaScreen(nn, nnTimes, artifactFraction: artifactFraction) + ? nightMemo('cvhr', artifactFraction, () { + streamDiagnostics?.observeBatch('cvhr', nn.length); + return cvhrApneaScreen(nn, nnTimes, artifactFraction: artifactFraction); + }) : Metric.absent( tier: Tier.estimate, inputs_used: const ['rr_cleaned'], note: needInputNote('nn_beats', have: nn.length, need: 60), ); - if (nn.length >= 30) streamDiagnostics?.observeBatch('rsa', nn.length); - if (nn.length >= 60) streamDiagnostics?.observeBatch('cvhr', nn.length); // ── 24/7 IRREGULAR-RHYTHM SCREEN (day-span RR; not a diagnosis) ──────────── // Runs over the WHOLE-DAY cleaned RR (not just sleep) so an arrhythmia screen // isn't limited to the sleep window. Hard-gated on beat count + artifact inside @@ -759,8 +793,11 @@ Map deriveDayBundle(Map inputJson, { // ── BREATHING-RATE VARIABILITY (per-window RSA over the sleep NN) ────────── // Window the cleaned sleep NN into ~30-min bins, take each bin's RSA resp rate, // then BRV = dispersion + Theil-Sen trend of those per-window rates. - final respWindows = respPerWindow(nn, nnTimes, - artifactFraction: artifactFraction, diagnostics: streamDiagnostics); + final cachedRespWindows = nightMemo('rsa_bins', artifactFraction, () => + respPerWindow(nn, nnTimes, artifactFraction: artifactFraction, + diagnostics: streamDiagnostics)); + final respWindows = nightState == null + ? cachedRespWindows : cachedRespWindows.toList(); streamDiagnostics?.onRespWindows?.call(respWindows); final brv = respWindows.length >= 3 ? breathingRateVariability(respWindows) @@ -1329,11 +1366,11 @@ Map deriveDayBundle(Map inputJson, { 'min_diffs_per_window': sessionDetail?.minDiffsPerWindow, }, 'rmssd_nocturnal': robustRmssd.toJson(), - 'hrv_freq': hrvF.toJson((v) => v.toJson()), + 'hrv_freq': nightJson(hrvF.toJson((v) => v.toJson())), 'resting_hr': rhr.toJson((v) => v.toJson()), 'hr_dip': dip.toJson((v) => v.toJson()), - 'prsa_dc': dc.toJson((v) => v.toJson()), - 'prsa_ac': ac.toJson((v) => v.toJson()), + 'prsa_dc': nightJson(dc.toJson((v) => v.toJson())), + 'prsa_ac': nightJson(ac.toJson((v) => v.toJson())), 'readiness_lnrmssd': lnReadiness.toJson((v) => v.toJson()), 'readiness_composite': { ...composite.toJson((v) => v.toJson()), @@ -1438,8 +1475,8 @@ Map deriveDayBundle(Map inputJson, { }; final respiration = { - 'rsa': resp.toJson((v) => v.toJson()), - 'cvhr_apnea': cvhr.toJson((v) => v.toJson()), + 'rsa': nightJson(resp.toJson((v) => v.toJson())), + 'cvhr_apnea': nightJson(cvhr.toJson((v) => v.toJson())), 'odi': odi.toJson((v) => v.toJson()), }; @@ -1708,7 +1745,6 @@ Map deriveDayBundle(Map inputJson, { ), }; - streamDiagnostics?.endPass(); return { 'date': d.date, 'day_confidence': _round(d.dayConfidence, 4), @@ -1997,6 +2033,17 @@ List respPerWindow( return out; } +// Metric serializers can expose lists owned by the cached value or envelope. +// Detach these small JSON blocks before returning them to a caller. +Map _copyNightJson(Map json) { + Object? copy(Object? value) { + if (value is Map) return value.map((k, v) => MapEntry(k, copy(v))); + if (value is List) return [for (final v in value) copy(v)]; + return value; + } + return json.map((k, v) => MapEntry(k, copy(v))); +} + /// Wrap a value sub-map in the {value,confidence,tier,inputs_used} envelope the /// serve seam reads via `.value`. Null inner → honest "—". Map _envelope( diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index 8aefbb876..8e1403f06 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -1267,7 +1267,7 @@ List calendarDays( List vendorNights = const [], String? targetDay, bool skipUnusedDays = false, - ana.CardioRowCache? rowCache, // ED-2b RED: unused until E6 adoption. + ana.CardioRowCache? rowCache, // D36: other walked nights must not train under the target's ID. // Restore the caller's recording flag on every exit. bool recordTargetOnly = false, @@ -1394,6 +1394,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, forcedWindow: (onsetSec: ov.onsetSec, offsetSec: ov.offsetSec), @@ -1417,6 +1418,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, habitualMidsleepSec: habitualMidsleepSec, @@ -1450,6 +1452,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, forcedWindow: @@ -1484,6 +1487,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, forcedWindow: (onsetSec: vn.onsetSec, offsetSec: vn.offsetSec), diff --git a/test/ed2b_cardio_cache_test.dart b/test/ed2b_cardio_cache_test.dart new file mode 100644 index 000000000..1dfb0dead --- /dev/null +++ b/test/ed2b_cardio_cache_test.dart @@ -0,0 +1,277 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/compute/vendor_sleep.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +const _day = '2020-01-02'; +int _sec(int hour) => DateTime(2020, 1, 2, hour).millisecondsSinceEpoch ~/ 1000; + +// RR plus accelerometry forces cardio rows through the real stager; a sleepy +// HR-only fixture could pass output guards without ever exercising the cache. +Substrate _night({int endHour = 7, int fromHour = -2}) { + final ts = [], hr = []; + final ax = [], ay = [], az = []; + final rr = [], rt = []; + var beat = _sec(fromHour) * 1000.0; + for (var t = _sec(fromHour); t < _sec(endHour); t++) { + final asleep = t >= _sec(-1) && t < _sec(6); + final bpm = asleep ? 52 + t % 7 : 85 + t % 13; + ts.add(t); hr.add(bpm); + ax.add(asleep ? 0 : .3 * math.sin(t * .21)); + ay.add(asleep ? 0 : .2 * math.cos(t * .13)); + az.add(asleep ? 1 : 1 + .05 * math.sin(t * .07)); + while (beat < (t + 1) * 1000) { + final ibi = 60000 / bpm + (asleep ? 40 : 12) * math.sin(t * .35); + rt.add(beat); rr.add(ibi); beat += ibi; + } + } + return Substrate(tsSec: ts, hr: hr, ax: ax, ay: ay, az: az, + rrMs: rr, rrTsMs: rt, skinTemp: List.filled(ts.length, 3000), + skinContact: List.filled(ts.length, 1), spo2Red: List.filled(ts.length, 1), + spo2Ir: List.filled(ts.length, 1), deviceFamily: 'gen4'); +} + +String _candidate(Substrate sub, ana.CardioRowCache? cache, + {SleepWindowOverride? override}) => jsonEncode(prepareSleepSessionCandidate( + sub, targetDay: _day, rowCache: cache, override: override).toJson()); + +Future _save(Substrate sub) async { + final db = await LocalDb.instance; final b = db.batch(); + for (var i = 0; i < sub.tsSec.length; i++) { + final t = sub.tsSec[i]; + b.insert('decoded_onehz', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'counter': i, 'hr': sub.hr[i], 'ax': sub.ax[i], + 'ay': sub.ay[i], 'az': sub.az[i], 'device_family': 'gen4'}); + } + final indexes = {}; + for (var i = 0; i < sub.rrMs.length; i++) { + final t = sub.rrTsMs[i] ~/ 1000; + final index = indexes.update(t, (v) => v + 1, ifAbsent: () => 0); + b.insert('decoded_rr', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'rr_ts_ms': sub.rrTsMs[i], 'rr_ms': sub.rrMs[i], + 'beat_index': index}); + } + await b.commit(noResult: true); +} + +void _nearPrsa(Object? got, Object? want, [String path = r'$']) { + if (got is double && want is double) { + final tolerance = path.endsWith('.confidence') ? 1e-12 : 1e-9 + 1e-9 * want.abs(); + expect((got - want).abs(), lessThanOrEqualTo(tolerance), reason: path); + } else if (want is Map) { + final g = got as Map; expect(g.keys.toList(), want.keys.toList()); + for (final k in want.keys) { _nearPrsa(g[k], want[k], '$path.$k'); } + } else if (want is List) { + final g = got as List; expect(g.length, want.length); + for (var i = 0; i < want.length; i++) { _nearPrsa(g[i], want[i], '$path[$i]'); } + } else { expect(got, want, reason: path); } +} + +void _checkNightOutput(Map input, Map output) { + final cold = deriveDayBundle(input); + final a = Map.from(output['clinical'] as Map); + final b = Map.from(cold['clinical'] as Map); + for (final k in ['prsa_dc', 'prsa_ac']) { _nearPrsa(a.remove(k), b.remove(k)); } + expect(jsonEncode({...output, 'clinical': a}), jsonEncode({...cold, 'clinical': b})); +} + +void main() { + for (final path in ['candidate auto', 'candidate manual', 'candidate confirmed', 'payload', 'calendar']) { + test('E6 $path shares the supplied CardioRowCache and stays byte-identical', () { + final sub = _night(); final cache = ana.CardioRowCache(); + String run(ana.CardioRowCache? rows) { + if (path == 'candidate auto') { return _candidate(sub, rows); } + if (path == 'candidate confirmed' || path == 'candidate manual') { return _candidate(sub, rows, + override: SleepWindowOverride(dayId: _day, onsetSec: _sec(0), + offsetSec: _sec(5), source: path == 'candidate manual' ? 'manual' : 'confirmed')); } + if (path == 'payload') { return jsonEncode(prepareDerivationPayload(sub, + targetDay: _day, rowCache: rows).days.map((d) => d.toJson()).toList()); } + return jsonEncode(calendarDays(sub, rowCache: rows) + .map((d) => d.sleep.toJson()).toList()); + } + final plain = run(null); expect(run(cache), plain); expect(run(cache), plain); + expect(cache.length, greaterThan(0), reason: 'every actual staging call must populate shared rows'); + expect(cache.hits, greaterThan(0)); + }); + } + + test('E6 HR-led fallback shares the cache through its forced stager', () { + // Include waking HR before the previous local noon so the fallback uses + // an actual daytime baseline rather than a flat night percentile. + final original = _night(fromHour: -13); + final sub = Substrate.fromJson({...original.toJson(), + // Keep valid moving vectors: all-zero missing accel would leave the + // forced stager with no feature rows to cache even after E6 lands. + 'ax': [for (final t in original.tsSec) .3 * math.sin(t * .21)], + 'ay': [for (final t in original.tsSec) .2 * math.cos(t * .13)], + 'az': [for (final t in original.tsSec) 1 + .05 * math.sin(t * .07)]}); + final plain = prepareSleepSessionCandidate(sub, targetDay: _day); + expect(plain.present, isTrue); + expect(plain.sleepSource, 'auto_fallback', reason: 'fixture must reach the HR-led forced-window call'); + final cache = ana.CardioRowCache(); + expect(_candidate(sub, cache), jsonEncode(plain.toJson())); + expect(_candidate(sub, cache), jsonEncode(plain.toJson())); + expect(cache.hits, greaterThan(0)); + }); + + test('E6 vendor-owned forced staging shares the same auto-stage row cache', () { + final sub = _night(); + final plain = prepareSleepSessionCandidate(sub, targetDay: _day); + expect(plain.present, isTrue); + final start = plain.sleepOnsetSec, end = plain.sleepOffsetSec - 1; + final step = (end - start) ~/ 3; + final vendor = VendorNight(deviceId: '', source: 'gen4', decodedAtSec: _sec(7), + epochs: [VendorEpoch(start, start + step, 'light'), + VendorEpoch(start + step, start + step * 2, 'deep'), + VendorEpoch(start + step * 2, end, 'rem')]); + String run(ana.CardioRowCache? cache) { + final got = prepareSleepSessionCandidate(sub, targetDay: _day, + vendorNights: [vendor], rowCache: cache); + expect(got.sleepSource, 'vendor_staged', reason: 'fixture must reach the vendor forced-window call'); + return jsonEncode(got.toJson()); + } + final cold = run(null), cache = ana.CardioRowCache(); + expect(run(cache), cold); expect(run(cache), cold); + expect(cache.hits, greaterThan(0)); + }); + + test('E6 an append reuses settled rows and preserves cold candidate bytes', () { + final cache = ana.CardioRowCache(); + _candidate(_night(endHour: 3), cache); + final full = _night(); final got = _candidate(full, cache); + expect(got, _candidate(full, null)); + expect(cache.hits, greaterThan(0), reason: 'earlier epochs survive a growing night'); + }); + + test('guard: #99 malformed cardio cache refuses whole payload', () { + for (final bad in [{'version': 99, 'rows': []}, + {'version': 2, 'rows': 'invalid'}, + {'version': 2, 'rows': [{'startSec': 1}]}]) { + expect(() => ana.CardioRowCache.fromJson(bad), throwsFormatException); + } + }); + + group('E6 engine owns encoded bytes and workers own row objects', () { + setUpAll(edInit); + setUp(() async { await edFresh('ed2b_cardio.db'); await _save(_night()); }); + tearDown(edDrop); + + test('real engine worker decodes, merges and prunes the shared row cache', () async { + final e = DerivationEngine()..debugDisableStagedSleepCache = true; + final first = await e.debugSleepCandidate(_day); + final second = await e.debugSleepCandidate(_day); + expect(jsonEncode(second.toJson()), jsonEncode(first.toJson())); + expect(e.debugCardioRowBytes, isA(), reason: 'UI isolate retains only the encoded payload'); + expect(e.debugCardioRowHits, greaterThan(0)); + expect(e.debugCardioRowWorkerOperations, containsAll(['decode:worker', 'merge:worker', 'prune:worker'])); + expect(e.debugCardioRowWorkerOperations.where((s) => s.endsWith(':caller')), isEmpty); + final cold = DerivationEngine()..debugDisableStagedSleepCache = true + ..debugDisableCardioRows = true; + expect(jsonEncode((await cold.debugSleepCandidate(_day)).toJson()), + jsonEncode(first.toJson())); + }); + + for (final key in ['input revision', 'generation', 'algorithm']) { + test('$key rejects bytes then replacement cache hits with cold-exact staging', () async { + final e = DerivationEngine()..debugDisableStagedSleepCache = true; + final first = await e.debugSleepCandidate(_day); + final firstMisses = e.debugCardioRowMisses; + if (key == 'input revision') { + final db = await LocalDb.instance; + final row = (await db.query('decoded_onehz', limit: 1)).first; + await db.update('decoded_onehz', {'hr': row['hr']}, + where: 'device_id = ? AND ts_ms = ?', whereArgs: [row['device_id'], row['ts_ms']]); + } else if (key == 'generation') { + await LocalDb.close(); await LocalDb.instance; + } else { + final fp = await LocalDb.inputDayFingerprint(_day); + e.debugRekeyCardioRows(inputRevision: fp.revisionSum, + generation: fp.generation, algorithm: 115); + } + final changed = await e.debugSleepCandidate(_day); + expect(jsonEncode(changed.toJson()), jsonEncode(first.toJson())); + expect(e.debugCardioRowMisses, greaterThan(firstMisses), + reason: 'new provenance must reject existing encoded rows'); + final misses = e.debugCardioRowMisses, hits = e.debugCardioRowHits; + await e.debugSleepCandidate(_day); + expect(e.debugCardioRowMisses, misses); + expect(e.debugCardioRowHits, greaterThan(hits), reason: 'new provenance must become reusable'); + }); + } + test('E7-E11 real runDays retains night streams across worker dispatches', () async { + await LocalDb.putSleepOverride(dayId: _day, onsetTs: _sec(1), + offsetTs: _sec(1) + 1200, source: 'confirmed'); + var observed = 0; + final e = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + e.debugOnNightBundle = (input, output) { + observed++; + _checkNightOutput(input, output); + }; + expect(await e.runDays(edProfile, {_day}), 1); + expect(observed, greaterThan(0)); + final before = e.debugNightStreams!; + expect(before.processedNightBeats, greaterThan(0)); + final work = {for (final name in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) + name: before.processedBeats(name)}; + expect(await e.runDays(edProfile, {_day}), 1); + expect(e.debugNightStreams!.processedNightBeats, before.processedNightBeats, + reason: 'the production dispatch must return and retain the RAM night owner'); + for (final entry in work.entries) { + expect(work[entry.key], greaterThan(0)); + expect(e.debugNightStreams!.processedBeats(entry.key), entry.value); + } + expect(e.debugNightStreams!.fullListDigests, greaterThan(0)); + expect(e.debugNightStreams!.maximumListDigestsPerIdentity, 1); + expect(e.debugNightStreams!.identityMemoEntries, 0); + }); + + for (final change in ['revision', 'generation', 'algorithm']) { + test('E7-E11 engine $change misses then retains the replacement worker state', () async { + await LocalDb.putSleepOverride(dayId: _day, onsetTs: _sec(1), + offsetTs: _sec(1) + 1200, source: 'confirmed'); + final e = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics() + ..debugOnNightBundle = _checkNightOutput; + expect(await e.runDays(edProfile, {_day}), 1); + final before = e.debugNightStreams!.processedNightBeats; + if (change == 'revision') { + final db = await LocalDb.instance; + final row = (await db.query('decoded_onehz', where: 'rec_ts = ?', + whereArgs: [_sec(1) + 60], limit: 1)).first; + await db.update('decoded_onehz', {'hr': row['hr']}, + where: 'device_id = ? AND ts_ms = ?', whereArgs: [row['device_id'], row['ts_ms']]); + } else if (change == 'generation') { + await LocalDb.close(); await LocalDb.instance; + } else { + final fp = await LocalDb.inputDayFingerprint(_day); + e.debugNightStreams!.context(inputRevision: fp.revisionSum, + generation: fp.generation, algorithm: 115); + } + expect(await e.runDays(edProfile, {_day}), 1); + final miss = e.debugNightStreams!.processedNightBeats; + expect(miss, greaterThan(before)); + final metricWork = {for (final s in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) + s: e.debugNightStreams!.processedBeats(s)}; + expect(await e.runDays(edProfile, {_day}), 1); + expect(e.debugNightStreams!.processedNightBeats, miss); + for (final s in metricWork.keys) { + expect(e.debugNightStreams!.processedBeats(s), metricWork[s]); + } + expect(e.debugNightStreams!.identityMemoEntries, 0); + }); + } + }); +} From a8eb242fe0fffd1f448fee966a408970ebe41da2 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 11:22:29 -0600 Subject: [PATCH 04/11] derive: stream the night's HRV frequency, RSA, CVHR, RSA bins and PRSA across passes Built on the night RR owner and analytics #99's streams: a pass feeds only the beats it has not seen. CVHR drops the tail-only fallback because #99 carries the fresh-stream fix; PRSA keeps sequential slices. Results equal one batch call on the same input, within #99's documented tolerances. --- lib/compute/ed2b_stream_diagnostics.dart | 14 +- lib/compute/night_prsa.dart | 32 ++ lib/compute/night_resp_bins.dart | 73 +++++ lib/compute/night_rr_state.dart | 72 ++++- lib/compute/onehz_pipeline.dart | 10 +- test/ed2b_streams_test.dart | 357 +++++++++++++++++++++++ 6 files changed, 548 insertions(+), 10 deletions(-) create mode 100644 lib/compute/night_prsa.dart create mode 100644 lib/compute/night_resp_bins.dart create mode 100644 test/ed2b_streams_test.dart diff --git a/lib/compute/ed2b_stream_diagnostics.dart b/lib/compute/ed2b_stream_diagnostics.dart index cd37f3d27..76ff21597 100644 --- a/lib/compute/ed2b_stream_diagnostics.dart +++ b/lib/compute/ed2b_stream_diagnostics.dart @@ -26,10 +26,12 @@ class Ed2bStreamDiagnostics { int get identityMemoEntries => nightState.identityMemoEntries; int get fullListDigests => nightState.fullListDigests; int get maximumListDigestsPerIdentity => nightState.maximumListDigestsPerIdentity; - int retainedStreamBeats(String stream) => 0; - int processedBeats(String stream) => _batchBeats[stream] ?? 0; - int slicesProcessed(String stream) => _batchCalls[stream] ?? 0; - int streamRestarts(String stream) => 0; + int retainedStreamBeats(String stream) => nightState.retainedStreamBeats(stream); + int processedBeats(String stream) => nightState.processedBeats(stream) + + (_batchBeats[stream] ?? 0); + int slicesProcessed(String stream) => nightState.slicesProcessed(stream) + + (_batchCalls[stream] ?? 0); + int streamRestarts(String stream) => nightState.streamRestarts(stream); void observeNightCorrection(int rawBeats, List nn, List times, double cleanFraction) { @@ -37,7 +39,7 @@ class Ed2bStreamDiagnostics { } void observeBatch(String stream, int beats) { - _batchCalls[stream] = slicesProcessed(stream) + 1; - _batchBeats[stream] = processedBeats(stream) + beats; + _batchCalls[stream] = (_batchCalls[stream] ?? 0) + 1; + _batchBeats[stream] = (_batchBeats[stream] ?? 0) + beats; } } diff --git a/lib/compute/night_prsa.dart b/lib/compute/night_prsa.dart new file mode 100644 index 000000000..90fc6f136 --- /dev/null +++ b/lib/compute/night_prsa.dart @@ -0,0 +1,32 @@ +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart'; + +final _dc = prsaAggregator(deceleration: true); +final _ac = prsaAggregator(deceleration: false); + +/// Settled night NN in reference order. Provisional tails only touch copies. +class NightPrsa { + Object? _dcState = _dc.empty(), _acState = _ac.empty(); + + void fold(List nn) { + final column = DoubleColumn(Float64List.fromList(nn)); + _dcState = _dc.addSlice(_dcState, column, 0, column.length); + _acState = _ac.addSlice(_acState, column, 0, column.length); + } + + (Metric, Metric) snapshot(List tailNn) { + final tail = DoubleColumn(Float64List.fromList(tailNn)); + Metric finish( + Aggregator> agg, + Object? state) { + final copy = agg.decodeState(agg.encodeState(state)); + return agg.finish(agg.addSlice(copy, tail, 0, tail.length)); + } + // Sequential addSlice preserves anchor order; never combine partial sums. + // #99 prsa_aggregator.dart declares profile/capacity abs+rel 1e-9, + // confidence abs 1e-12 parity with the batch functions. + return (finish(_dc, _dcState), finish(_ac, _acState)); + } +} diff --git a/lib/compute/night_resp_bins.dart b/lib/compute/night_resp_bins.dart new file mode 100644 index 000000000..4ac421c0e --- /dev/null +++ b/lib/compute/night_resp_bins.dart @@ -0,0 +1,73 @@ +import 'package:openstrap_analytics/onehz.dart'; + +/// Thirty-minute bins anchored at the first corrected beat, like the batch. +/// Keep closed spectral states so every snapshot applies the current night's +/// artifact gate through analytics, including to bins closed on a prior pass. +class NightRespBins { + NightRespBins(); + double? _origin; + int _openIndex = 0, _openCount = 0; + RsaRespStream _open = RsaRespStream(); + final List<({int count, RsaRespStream stream})> _closed = []; + + int _bin(double time, double origin) => ((time - origin) / 1800000.0).floor(); + + void _appendRate(List out, Metric metric) { + final rate = metric.present ? metric.value!.brpm : null; + if (rate != null) out.add(rate); + } + + /// Returns the number of nonempty bin slices actually folded. + int fold(List nn, List times) { + var from = 0, slices = 0; + void feed(int end) { + if (end == from) return; + _open.fold(nn.sublist(from, end), times.sublist(from, end)); + _openCount += end - from; + slices++; + } + for (var i = 0; i < times.length; i++) { + final index = _bin(times[i], _origin ??= times[i]); + if (index == _openIndex) continue; + feed(i); + if (_openCount >= 60) _closed.add((count: _openCount, stream: _open)); + _open = RsaRespStream(); + _openCount = 0; + _openIndex = index; + from = i; + } + feed(nn.length); + return slices; + } + + List snapshot(List tailNn, List tailTimes, + {required double artifactFraction}) { + if (_origin == null && tailNn.isEmpty) return []; + final origin = _origin ?? tailTimes.first; + final out = []; + for (final bin in _closed) { + _appendRate(out, bin.stream.snapshot(artifactFraction: artifactFraction)); + } + var i = 0; + while (i < tailNn.length && _bin(tailTimes[i], origin) == _openIndex) { + i++; + } + if (_openCount + i >= 60) { + _appendRate(out, _open.snapshot(tailNnMs: tailNn.sublist(0, i), + tailNnTimesMs: tailTimes.sublist(0, i), artifactFraction: artifactFraction)); + } + while (i < tailNn.length) { + final index = _bin(tailTimes[i], origin); + var end = i + 1; + while (end < tailNn.length && _bin(tailTimes[end], origin) == index) { + end++; + } + if (end - i >= 60) { + _appendRate(out, rsaRespRate(tailNn.sublist(i, end), + tailTimes.sublist(i, end), artifactFraction: artifactFraction)); + } + i = end; + } + return out; + } +} diff --git a/lib/compute/night_rr_state.dart b/lib/compute/night_rr_state.dart index d17adc59a..a9345fb98 100644 --- a/lib/compute/night_rr_state.dart +++ b/lib/compute/night_rr_state.dart @@ -3,6 +3,9 @@ import 'dart:typed_data'; import 'package:openstrap_analytics/onehz.dart' as ana; +import 'night_prsa.dart'; +import 'night_resp_bins.dart'; + typedef NightRr = ({List nn, List nnTimesMs, double cleanFraction, int droppedCount, int correctedCount}); typedef NightListProof = ({int beats, int times, (int, int) nn, (int, int) ts}); @@ -23,8 +26,21 @@ class NightRrState { (String, int)? _night; final _memos = {}; final _digests = _PassListDigests(); + List _tailNn = const [], _tailTimes = const []; + ana.HrvFreqStream? _frequency; + ana.RsaRespStream? _rsa; + ana.CvhrStream? _cvhr; + NightPrsa? _prsa; + NightRespBins? _rsaBins; + final Map _fed = {}, _processed = {}, _slices = {}, _restarts = {}; int processedNightBeats = 0, nightRebuilds = 0; + bool get canStream => _corrector != null; + // Settled beats represented by each stream, including compacted summaries. + int retainedStreamBeats(String key) => _fed[key] ?? 0; + int processedBeats(String key) => _processed[key] ?? 0; + int slicesProcessed(String key) => _slices[key] ?? 0; + int streamRestarts(String key) => _restarts[key] ?? 0; int get retainedNightBeats => _rr.length; int get settledNightBeats => _nn.length; int get identityMemoEntries => _digests.entries; @@ -51,6 +67,10 @@ class NightRrState { _rr.clear(); _times.clear(); _nn.clear(); _nnTimes.clear(); _night = null; _memos.clear(); + _tailNn = const []; _tailTimes = const []; + _frequency = null; _rsa = null; _cvhr = null; + _prsa = null; _rsaBins = null; + _fed.clear(); } /// Shared by the night metric memos. No digest outlives this pass's @@ -99,14 +119,15 @@ class NightRrState { _nn.addAll(settled.nn); _nnTimes.addAll(settled.nnTimes); } final tail = corrector.snapshot(); + _tailNn = tail.tailNn; _tailTimes = tail.tailNnTimes; return (nn: [..._nn, ...tail.tailNn], nnTimesMs: [..._nnTimes, ...tail.tailNnTimes], cleanFraction: tail.cleanFraction, droppedCount: tail.droppedCount, correctedCount: tail.correctedCount); } - /// Exact batch-result reuse. Growing lists still run the ordinary batch - /// consumers; incremental metric streams are a separate adoption step. + /// Exact result reuse. Growing lists snapshot incremental metric streams; + /// unchanged lists also avoid re-evaluating their provisional tails. T memo(String key, Object dependencies, T Function() compute) { if (_corrector == null) return compute(); final old = _memos[key]; @@ -118,6 +139,53 @@ class NightRrState { _memos[key] = (dependencies: dependencies, value: result); return result; } + + void _feed(String key, int Function(List, List) fold) { + final held = _fed[key] ?? 0; + if (!_fed.containsKey(key)) { + _restarts[key] = streamRestarts(key) + 1; + _fed[key] = 0; + } + if (held == _nn.length) return; + final slices = fold(_nn.sublist(held), _nnTimes.sublist(held)); + _processed[key] = processedBeats(key) + _nn.length - held; + _slices[key] = slicesProcessed(key) + slices; + _fed[key] = _nn.length; + } + + ana.Metric nightHrvFreq(double artifactFraction) { + final stream = _frequency ??= ana.HrvFreqStream(); + _feed('frequency', (nn, times) { stream.fold(nn, times); return 1; }); + return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, + artifactFraction: artifactFraction); + } + + ana.Metric nightResp(double artifactFraction) { + final stream = _rsa ??= ana.RsaRespStream(); + _feed('rsa', (nn, times) { stream.fold(nn, times); return 1; }); + return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, + artifactFraction: artifactFraction); + } + + ana.Metric nightCvhr(double artifactFraction) { + final stream = _cvhr ??= ana.CvhrStream(); + _feed('cvhr', (nn, times) { stream.fold(nn, times); return 1; }); + // #99 handles a fresh stream's entirely provisional night too. + return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, + artifactFraction: artifactFraction); + } + + (ana.Metric, ana.Metric) nightPrsa() { + final stream = _prsa ??= NightPrsa(); + _feed('prsa', (nn, times) { stream.fold(nn); return 1; }); + return stream.snapshot(_tailNn); + } + + List nightRespWindows(double artifactFraction) { + final stream = _rsaBins ??= NightRespBins(); + _feed('rsa_bins', stream.fold); + return stream.snapshot(_tailNn, _tailTimes, artifactFraction: artifactFraction); + } } /// Short window lists are walked directly. Only long shared lists occupy diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 4f15610c8..54a946d4f 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -620,6 +620,7 @@ Map _deriveDayBundle(Map inputJson, final nn = corrected.nn; final nnTimes = corrected.nnTimesMs; final artifactFraction = (1.0 - corrected.cleanFraction).clamp(0.0, 1.0); + final streams = nightState?.canStream == true ? nightState : null; final nnProof = nightState?.dependencyProof(nn, nnTimes); T nightMemo(String key, Object? extra, T Function() compute) => nnProof == null ? compute() : @@ -685,6 +686,7 @@ Map _deriveDayBundle(Map inputJson, sleepSessionRmssdMetric.present ? sessionDetail!.rmssd : null; final hrvF = nn.length >= 20 ? nightMemo('frequency', artifactFraction, () { + if (streams != null) return streams.nightHrvFreq(artifactFraction); streamDiagnostics?.observeBatch('frequency', nn.length); return hrvFreq(nn, nnTimes, artifactFraction: artifactFraction); }) @@ -741,6 +743,7 @@ Map _deriveDayBundle(Map inputJson, final dayOnly = _dayHrOutsideSleep(d); final dip = hrDip(dayOnly, sleepHr); final prsa = nightMemo('prsa', null, () { + if (streams != null) return streams.nightPrsa(); streamDiagnostics?.observeBatch('prsa', nn.length); return (decelerationCapacity(nn), accelerationCapacity(nn)); }); @@ -756,6 +759,7 @@ Map _deriveDayBundle(Map inputJson, // ── RESPIRATION (sleep-windowed) ─────────────────────────────────────────── final resp = nn.length >= 30 ? nightMemo('rsa', artifactFraction, () { + if (streams != null) return streams.nightResp(artifactFraction); streamDiagnostics?.observeBatch('rsa', nn.length); return rsaRespRate(nn, nnTimes, artifactFraction: artifactFraction); }) @@ -766,6 +770,7 @@ Map _deriveDayBundle(Map inputJson, ); final cvhr = nn.length >= 60 ? nightMemo('cvhr', artifactFraction, () { + if (streams != null) return streams.nightCvhr(artifactFraction); streamDiagnostics?.observeBatch('cvhr', nn.length); return cvhrApneaScreen(nn, nnTimes, artifactFraction: artifactFraction); }) @@ -794,8 +799,9 @@ Map _deriveDayBundle(Map inputJson, // Window the cleaned sleep NN into ~30-min bins, take each bin's RSA resp rate, // then BRV = dispersion + Theil-Sen trend of those per-window rates. final cachedRespWindows = nightMemo('rsa_bins', artifactFraction, () => - respPerWindow(nn, nnTimes, artifactFraction: artifactFraction, - diagnostics: streamDiagnostics)); + streams != null ? streams.nightRespWindows(artifactFraction) : + respPerWindow(nn, nnTimes, artifactFraction: artifactFraction, + diagnostics: streamDiagnostics)); final respWindows = nightState == null ? cachedRespWindows : cachedRespWindows.toList(); streamDiagnostics?.onRespWindows?.call(respWindows); diff --git a/test/ed2b_streams_test.dart b/test/ed2b_streams_test.dart new file mode 100644 index 000000000..adb511f05 --- /dev/null +++ b/test/ed2b_streams_test.dart @@ -0,0 +1,357 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:isolate'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/aggregate.dart' as agg; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/night_resp_bins.dart'; + +// Deliberately no fork DayCalculationState, mode or revision framework. The +// optional diagnostics observe the real upstream batch calls until GREEN. +Map _input({int beats = 950, bool gaps = false, + bool artifacts = false}) { + const start = 1577930400; + final rr = [], times = []; + var t = start * 1000.0; + for (var i = 0; i < beats; i++) { + final x = 970 + 38 * math.sin(i * .071) + 24 * math.sin(i * 1.31); + rr.add(artifacts && i >= 430 && i < 465 ? 2400.0 : x); + times.add(t); + t += x; + if (gaps && i == 510) t += 90000; + } + return DayBundleInput(date: '2020-01-02', dayTsSec: const [], + dayHr: const [], sleepTsSec: const [], sleepHr: const [], + sleepRrMs: rr, sleepRrTsMs: times, sleepSkinTemp: const [], + sleepJson: const {}, hypnoStages: const [], sleepOnsetSec: start, + sleepOffsetSec: t ~/ 1000 + 1, profile: const {}, + deviceFamily: 'gen4').toJson(); +} + +Map _prefix(Map input, int beats) => { + ...input, + 'sleep_rr_ms': (input['sleep_rr_ms'] as List).take(beats).toList(), + 'sleep_rr_ts_ms': (input['sleep_rr_ts_ms'] as List).take(beats).toList(), +}; + +String _wire(Object? value) => jsonEncode(value); +List _rr(Map x) => x['sleep_rr_ms'] as List; +List _times(Map x) => x['sleep_rr_ts_ms'] as List; + +void _near(Object? got, Object? want, [String path = r'$']) { + if (got is num && want is num) { + // #99 PRSA profile/capacity abs+relative 1e-9; confidence abs 1e-12. + final tol = path.endsWith('.confidence') ? 1e-12 : + 1e-9 + 1e-9 * want.abs(); + expect((got - want).abs(), lessThanOrEqualTo(tol), reason: path); + } else if (want is Map) { + expect(got, isA(), reason: path); + final g = got as Map; + expect(g.keys.toList(), want.keys.toList(), reason: path); + for (final k in want.keys) { _near(g[k], want[k], '$path.$k'); } + } else if (want is List) { + expect(got, isA(), reason: path); + final g = got as List; + expect(g.length, want.length, reason: path); + for (var i = 0; i < want.length; i++) { _near(g[i], want[i], '$path[$i]'); } + } else { expect(got, want, reason: path); } +} + +void _oracles(Map input, Map bundle) { + final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); + final nn = c.nn, ts = c.nnTimesMs; + final af = (1 - c.cleanFraction).clamp(0.0, 1.0); + final h = ana.HrvFreqStream(), r = ana.RsaRespStream(), v = ana.CvhrStream(); + for (var i = 0; i < nn.length; i += 73) { + final end = math.min(i + 73, nn.length); + final a = nn.sublist(i, end), b = ts.sublist(i, end); + h.fold(a, b); r.fold(a, b); v.fold(a, b); + } + final clinical = bundle['clinical'] as Map; + final respiration = bundle['respiration'] as Map; + if (nn.length >= 20) { + final batch = ana.hrvFreq(nn, ts, artifactFraction: af).toJson((x) => x.toJson()); + expect(_wire(h.snapshot(artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + expect(_wire(clinical['hrv_freq']), _wire(batch)); + } + if (nn.length >= 30) { + final batch = ana.rsaRespRate(nn, ts, artifactFraction: af).toJson((x) => x.toJson()); + expect(_wire(r.snapshot(artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + expect(_wire(respiration['rsa']), _wire(batch)); + } + if (nn.length >= 60) { + final batch = ana.cvhrApneaScreen(nn, ts, artifactFraction: af).toJson((x) => x.toJson()); + expect(_wire(v.snapshot(artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + expect(_wire(respiration['cvhr_apnea']), _wire(batch)); + // #99 contains 914bb6a: an entirely provisional tail on a FRESH stream. + expect(_wire(ana.CvhrStream().snapshot(tailNnMs: nn, + tailNnTimesMs: ts, artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + } + for (final deceleration in [true, false]) { + final a = agg.prsaAggregator(deceleration: deceleration); + var state = a.empty(); + final column = agg.DoubleColumn(Float64List.fromList(nn)); + for (var i = 0; i < nn.length; i += 37) { + state = a.addSlice(state, column, i, math.min(i + 37, nn.length)); + } + final batch = (deceleration ? ana.decelerationCapacity(nn) : + ana.accelerationCapacity(nn)).toJson((x) => x.toJson()); + _near(a.finish(state).toJson((x) => x.toJson()), batch); + _near(clinical[deceleration ? 'prsa_dc' : 'prsa_ac'], batch); + } +} + +void _bundleSame(Map got, Map cold) { + // #99 documents PRSA capacity/profile abs+relative 1e-9 and confidence + // abs 1e-12. Every other bundle byte, including notes and absence, is exact. + final gc = Map.from(got['clinical'] as Map); + final cc = Map.from(cold['clinical'] as Map); + for (final k in ['prsa_dc', 'prsa_ac']) { _near(gc.remove(k), cc.remove(k)); } + expect(_wire({...got, 'clinical': gc}), _wire({...cold, 'clinical': cc})); +} + +void _exact(Map input, Ed2bStreamDiagnostics diagnostics) { + final cold = deriveDayBundle(input); + final got = deriveDayBundle(input, streamDiagnostics: diagnostics); + _bundleSame(got, cold); + _oracles(input, got); +} + +void main() { + for (final style in ['smooth', 'gaps + capped artifact run']) { + final input = _input(gaps: style != 'smooth', artifacts: style != 'smooth'); + test('E7 $style chunked night NN is batch-exact and folds each beat once', () { + final d = Ed2bStreamDiagnostics(); + var observed = false; + d.onCorrected = (nn, times, fraction) { + observed = true; + final c = ana.correctRr(_rr(current), rrTsMs: _times(current)); + expect(_wire(nn), _wire(c.nn)); expect(_wire(times), _wire(c.nnTimesMs)); + expect(fraction, c.cleanFraction); + }; + // Cuts straddle the 248-beat capped corrector's provisional tail and + // the >32 artifact run. No cut is rounded to an epoch boundary. + for (final n in [1, 91, 247, 248, 249, 431, 464, 465, 701, 950]) { + current = _prefix(input, n); + _exact(current, d); + } + expect(observed, isTrue); + expect(d.processedNightBeats, _rr(input).length, + reason: 'RAM corrector must process only the appended raw beats'); + expect(d.retainedNightBeats, greaterThan(0)); + expect(d.nightRebuilds, 1); + }); + + for (final stream in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) { + final contract = {'frequency': 'E8', 'rsa': 'E8', 'cvhr': 'E9', + 'prsa': 'E10', 'rsa_bins': 'E10'}[stream]; + test('$contract $style $stream streams across growing and repeated passes', () { + final d = Ed2bStreamDiagnostics(); + for (final n in [300, 641, 950]) { _exact(_prefix(input, n), d); } + final work = d.processedBeats(stream); + expect(work, greaterThan(0), reason: 'real metric path must be exercised'); + _exact(input, d); + expect(d.processedBeats(stream), work, + reason: 'repeat must reuse settled stream state'); + expect(d.retainedStreamBeats(stream), greaterThan(0)); + expect(d.streamRestarts(stream), 1); + }); + } + } + + for (final feature in ['rr', 'frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) { + for (final key in ['input revision', 'store generation', 'algorithm']) { + test('E7-E10 $feature refuses changed $key, equals cold, then reuses replacement', () { + final d = Ed2bStreamDiagnostics(); + final input = _input(); + d.context(inputRevision: 1, generation: 1, algorithm: 114); + _exact(input, d); + final before = feature == 'rr' ? d.processedNightBeats : d.processedBeats(feature); + d.context(inputRevision: key == 'input revision' ? 2 : 1, + generation: key == 'store generation' ? 2 : 1, + algorithm: key == 'algorithm' ? 115 : 114); + _exact(input, d); + final miss = feature == 'rr' ? d.processedNightBeats : d.processedBeats(feature); + expect(miss, greaterThan(before), reason: 'changed context must reject retained work'); + _exact(input, d); + expect(feature == 'rr' ? d.processedNightBeats : d.processedBeats(feature), miss, + reason: 'the replacement context must subsequently hit'); + }); + } + } + + for (final mutation in ['old value', 'old timestamp', 'shorter prefix', 'shifted onset']) { + test('E7-E10 $mutation rebuilds the night and preserves every bundle byte', () { + final d = Ed2bStreamDiagnostics(); + final input = _input(); _exact(input, d); + final changed = _prefix(input, mutation == 'shorter prefix' ? 700 : 950); + if (mutation == 'old value') (_rr(changed))[100] += 7; + if (mutation == 'old timestamp') (_times(changed))[100] += .125; + if (mutation == 'shifted onset') { + changed['sleep_rr_ms'] = _rr(changed).sublist(10); + changed['sleep_rr_ts_ms'] = _times(changed).sublist(10); + changed['sleep_onset_sec'] = (changed['sleep_onset_sec'] as int) + 10; + } + _exact(changed, d); + final miss = d.processedNightBeats; + _exact(changed, d); + expect(d.processedNightBeats, miss, reason: 'changed prefix must be retained after rebuild'); + expect(d.nightRebuilds, 2); + }); + } + + test('E7 RAM night owner survives a real isolate hop without changed output', () async { + final d = Ed2bStreamDiagnostics(); final input = _input(); + deriveDayBundle(input, streamDiagnostics: d); + final before = d.processedNightBeats; + final hopped = await Isolate.run(() { + final bundle = deriveDayBundle(input, streamDiagnostics: d); + return (bundle, d); + }); + _bundleSame(hopped.$1, deriveDayBundle(input)); + expect(hopped.$2.processedNightBeats, before); + expect(hopped.$2.retainedNightBeats, greaterThan(0)); + }); + + test('guard: #99 capped RR settles and snapshots equal batch at every chunk', () { + final input = _input(gaps: true, artifacts: true); + final stream = ana.RrCorrector(); + final settled = [], times = []; + var previous = 0; + for (final n in [1, 91, 247, 248, 249, 431, 464, 465, 701, 950]) { + final next = stream.fold(_rr(input).sublist(previous, n), + tsMs: _times(input).sublist(previous, n)); + settled.addAll(next.nn); times.addAll(next.nnTimes); + final tail = stream.snapshot(); + final batch = ana.correctRr(_rr(input).sublist(0, n), + rrTsMs: _times(input).sublist(0, n)); + expect(_wire([...settled, ...tail.tailNn]), _wire(batch.nn)); + expect(_wire([...times, ...tail.tailNnTimes]), _wire(batch.nnTimesMs)); + previous = n; + } + }); + + test('E11 exceptional exit releases occupied identity memo and retry is fresh', () { + final d = Ed2bStreamDiagnostics(); final input = _input(beats: 2300); + var digestsAtExit = 0, memoAtExit = 0; + d.onBeforePassReturn = () { + digestsAtExit = d.fullListDigests; memoAtExit = d.identityMemoEntries; + throw StateError('injected after the actual analytics pass'); + }; + expect(() => deriveDayBundle(input, streamDiagnostics: d), throwsStateError); + expect(digestsAtExit, greaterThan(0), reason: 'must exercise actual hash work before failure'); + expect(memoAtExit, greaterThan(0), reason: 'must fail while the pass owns real identity keys'); + expect(d.identityMemoEntries, 0, reason: 'finally must release lists after exceptions'); + d.onBeforePassReturn = null; + _rr(input)[90] += 13; _exact(input, d); + expect(d.identityMemoEntries, 0); + }); + + for (final key in ['input revision', 'generation', 'algorithm']) { + test('E11 $key never reuses a preceding pass identity digest', () { + final input = _input(beats: 2300), d = Ed2bStreamDiagnostics(); + d.context(inputRevision: 1, generation: 1, algorithm: 114); + _exact(input, d); final first = d.fullListDigests; + d.context(inputRevision: key == 'input revision' ? 2 : 1, + generation: key == 'generation' ? 2 : 1, + algorithm: key == 'algorithm' ? 115 : 114); + _rr(input)[89] += 9; _exact(input, d); + expect(d.fullListDigests, greaterThan(first)); + expect(d.maximumListDigestsPerIdentity, 1); + expect(d.identityMemoEntries, 0); + }); + } + + test('E10 RSA bins cross 30-minute edges; sequential PRSA is batch-toleranced', () { + final input = _input(beats: 4000, gaps: true); + final d = Ed2bStreamDiagnostics(); + List? actualBins; + d.onRespWindows = (rates) { actualBins = rates; }; + _exact(_prefix(input, 1861), d); _exact(input, d); + final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); + final bins = , List)>{}; + for (var i = 0; i < c.nn.length; i++) { + final key = ((c.nnTimesMs[i] - c.nnTimesMs.first) / 1800000).floor(); + final b = bins.putIfAbsent(key, () => ([], [])); + b.$1.add(c.nn[i]); b.$2.add(c.nnTimesMs[i]); + } + final rates = []; + for (final b in bins.values) { + if (b.$1.length < 60) continue; + final stream = ana.RsaRespStream(); + for (var i = 0; i < b.$1.length; i += 79) { + final end = math.min(i + 79, b.$1.length); + stream.fold(b.$1.sublist(i, end), b.$2.sublist(i, end)); + } + final batch = ana.rsaRespRate(b.$1, b.$2, artifactFraction: 0); + expect(_wire(stream.snapshot(artifactFraction: 0).toJson((v) => v.toJson())), + _wire(batch.toJson((v) => v.toJson()))); + final rate = batch.present ? batch.value!.brpm : null; + if (rate != null) rates.add(rate); + } + expect(_wire(actualBins), _wire(rates)); + expect(bins.length, greaterThanOrEqualTo(3)); + expect(d.slicesProcessed('rsa_bins'), greaterThanOrEqualTo(3)); + final before = d.processedBeats('rsa_bins'); _exact(input, d); + expect(d.processedBeats('rsa_bins'), before); + expect(d.retainedStreamBeats('rsa_bins'), greaterThan(0)); + expect(d.retainedStreamBeats('prsa'), greaterThan(0)); + }); + + test('E11 shared long night lists are hashed once each per pass and released', () { + final d = Ed2bStreamDiagnostics(); final input = _input(beats: 2300); + _exact(input, d); + expect(d.fullListDigests, greaterThan(0), reason: 'real dependency proofs must be measured'); + expect(d.maximumListDigestsPerIdentity, 1); + expect(d.identityMemoEntries, 0, reason: 'identity keys must not pin worker arrays between passes'); + final before = d.fullListDigests; + // Reuse the SAME input identities, mutate content between passes. A digest + // surviving endPass would conceal the changed old prefix. + _rr(input)[100] += 11; _exact(input, d); + expect(d.fullListDigests, greaterThan(before)); + expect(d.maximumListDigestsPerIdentity, 1); expect(d.identityMemoEntries, 0); + }); + + test('guard: diagnostics enabled or disabled never changes bundle bytes', () { + final input = _input(gaps: true, artifacts: true); + final d = Ed2bStreamDiagnostics()..disabled = true; + _exact(input, d); d.disabled = false; _exact(input, d); + }); + + test('guard: empty and thin inputs preserve absence and no identity retention', () { + for (final n in [0, 1, 19, 29, 59]) { + final input = _input(beats: n); final d = Ed2bStreamDiagnostics(); + final got = deriveDayBundle(input, streamDiagnostics: d); + expect(_wire(got), _wire(deriveDayBundle(input))); + expect(d.identityMemoEntries, 0); + if (n == 0) expect(d.processedNightBeats, 0); + } + }); + test('E10 closed RSA bins reapply the night artifact gate after append', () { + final input = _input(beats: 6200); + final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); + final bins = NightRespBins(); + bins.fold(c.nn.sublist(0, 4000), c.nnTimesMs.sublist(0, 4000)); + List batch(double artifacts) => respPerWindow(c.nn, c.nnTimesMs, + artifactFraction: artifacts); + // A positive control ensures the artifact gate, rather than absent signal, + // is the only reason a high-artifact snapshot has no rates. + expect(batch(0), isNotEmpty); + bins.fold(c.nn.sublist(4000), c.nnTimesMs.sublist(4000)); + expect(bins.snapshot(const [], const [], artifactFraction: 0), batch(0)); + expect(batch(.5), isEmpty); + expect(bins.snapshot(const [], const [], artifactFraction: .5), batch(.5)); + }); + +} + +// Assigned immediately before each synchronous derive; used only to inspect +// the callback's actual corrected arrays without adding them to output JSON. +Map current = {}; From cfc41ebc50db64ffd422cb7dd7a419912e1488be Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 11:22:29 -0600 Subject: [PATCH 05/11] derive: load the search, day, nap and night windows once per pass One union load per pass, cut into the windows in workers, borrowing overlap from the retained day data. Exact given the to+1 ownership endpoint. --- lib/compute/derivation_engine.dart | 199 +++++++++++++++++-- lib/compute/derive_prepare.dart | 37 +++- lib/compute/substrate_retention.dart | 13 +- lib/compute/substrate_union.dart | 271 ++++++++++++++++++++++++++ test/ed2b_e14_union_loading_test.dart | 128 ++++++++++++ 5 files changed, 632 insertions(+), 16 deletions(-) create mode 100644 lib/compute/substrate_union.dart create mode 100644 test/ed2b_e14_union_loading_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index fa5b28e4f..0e04cd2d8 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -78,6 +78,7 @@ import 'ed2b_stream_diagnostics.dart'; import 'cardio_rows_cache.dart'; import 'night_rr_state.dart'; import 'substrate_retention.dart'; +import 'substrate_union.dart'; import 'derive_prepare.dart'; import 'onehz_pipeline.dart'; import 'step_cadence.dart'; @@ -2994,6 +2995,13 @@ class _PrepareStats { int rows = 0; } +class _PrepareUnion { + RetainedSubstrate? load; + Substrate? search; + Map> ownership = const {}; + Map> priority = const {}; +} + /// Run [worker] over [items] with at most [concurrency] running at once. Each /// of up to [concurrency] "lanes" pulls the next unclaimed item as soon as /// it's free — a mix of fast (empty/mostly-empty day) and slow (heavy @@ -3128,7 +3136,7 @@ class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; - // ED-2b observations. Checkpoint/union controls await their GREEN phase. + // ED-2b observations. Checkpoint controls await their GREEN phase. final debugCheckpoints = Ed2bCheckpointDiagnostics(); @visibleForTesting void Function(String day)? debugOnProfileFold; @@ -3164,9 +3172,11 @@ class DerivationEngine { @visibleForTesting bool debugDisableUnionLoads = false; @visibleForTesting - int get debugUnionWorkerCuts => 0; + int get debugUnionWorkerCuts => _unionWorkerCuts; @visibleForTesting - void debugRekeyUnionAlgorithm(int algorithm) {} + void debugRekeyUnionAlgorithm(int algorithm) { + for (final union in _retained.unions) { union.version = algorithm; } + } @visibleForTesting void Function(int from, int to, String load)? debugOnSubstrateLoad; @visibleForTesting @@ -3222,6 +3232,8 @@ class DerivationEngine { } final _retained = SubstrateRetention(); + final _unionLock = _AsyncLock(); + int _unionWorkerCuts = 0; final _cardioRowChunks = []; final _cardioRowOperations = {}; final _nightStates = {}; @@ -4295,6 +4307,7 @@ class DerivationEngine { static const int _rawDecodeBatchSize = 2000; static const int _maxDayRawRows = 500000; static const int _maxDayRawPages = 300; + static const int _maxUnionBeats = 4 * _maxDayRawRows; Future _prepareTargetDay(String dayId) async { try { @@ -4320,7 +4333,8 @@ class DerivationEngine { // is telemetry-only). Each day now gets its own local accumulator, // merged into the shared running max exactly once, below. final stats = _PrepareStats(); - final candidate = await _sleepCandidateForDay(dayId, stats: stats); + final union = debugDisableUnionLoads ? null : _PrepareUnion(); + final candidate = await _sleepCandidateForDay(dayId, stats: stats, union: union); final dayStart = _localDayLabelToSec(dayId); final dayEnd = _localNextDayLabelToSec(dayId); @@ -4344,8 +4358,29 @@ class DerivationEngine { if (candidate.present) candidate.sleepOffsetSec, ].reduce(math.max); - final (ownership, priority) = await _resolveOwnership(unionFrom, unionTo); + final (ownership, priority) = union?.load != null && + union!.load!.from == unionFrom && union.load!.to == unionTo + ? (Map>.of(union.ownership), + Map>.of(union.priority)) + : await _resolveOwnership(unionFrom, unionTo); final ceilingInput = await _ceilingInputRevision(dayId); + if (union != null && (union.load == null || + unionFrom < union.load!.from! || unionTo > union.load!.to!)) { + // A banked candidate needs no search load. A live search already in + // the holder can extend to a wider selected night by borrowing itself. + final from = union.load == null + ? math.min(dayStart, candidate.present ? candidate.sleepOnsetSec : dayStart) + : unionFrom; + final to = union.load == null + ? math.max(dayEnd - 1 + napBoundaryBufferSec, + candidate.present ? candidate.sleepOffsetSec - 1 : 0) + : unionTo; + union.load = await _loadUnion(dayId, from, to, ownership, stats: stats); + } + final windows = await _cutUnionWindows(union?.load, [ + (dayStart, dayEnd - 1 + napBoundaryBufferSec), + (candidate.sleepOnsetSec, candidate.sleepOffsetSec - 1), + ], ownership, dayStart: dayStart, dayEnd: dayEnd); // Load the day PLUS the nap boundary buffer in ONE pass (each // _loadSubstrateRange spawns its own isolate, so a second load would @@ -4353,7 +4388,7 @@ class DerivationEngine { // this, the live decoded path — run()/runDays()/rescanRecent(), i.e. every // non-import day — fell back to napSub == daySub and went on bisecting // naps at midnight. - var napSub = await _loadSubstrateRange( + var napSub = windows.$1[0] ?? await _loadSubstrateRange( dayStart, dayEnd - 1 + napBoundaryBufferSec, dayId: dayId, @@ -4367,22 +4402,23 @@ class DerivationEngine { // "Saw" is the calendar day, the same test the scope and the re-derive // use (`decodedRecTsMaxByDay`): band rows only in the next morning's nap // buffer do not keep the wearable off a day it alone covers. - final wearable = - napSub.slice(dayStart, dayEnd).isEmpty ? await activeWearable() : null; + final wearable = (windows.$2 ?? napSub.slice(dayStart, dayEnd)).isEmpty + ? await activeWearable() : null; if (wearable != null) { await _resolveSparseOwnership( unionFrom, unionTo, wearable.$1, ownership, priority); napSub = await wearableSubstrate( wearable, dayStart, dayEnd - 1 + napBoundaryBufferSec); } - final daySub = napSub.slice(dayStart, dayEnd); + final daySub = wearable == null && windows.$2 != null + ? windows.$2! : napSub.slice(dayStart, dayEnd); Substrate sleepSub = Substrate.empty; if (candidate.present && candidate.sleepOffsetSec > candidate.sleepOnsetSec) { sleepSub = wearable != null ? await wearableSubstrate(wearable, candidate.sleepOnsetSec, candidate.sleepOffsetSec - 1) - : await _loadSubstrateRange( + : windows.$1[1] ?? await _loadSubstrateRange( candidate.sleepOnsetSec, candidate.sleepOffsetSec - 1, dayId: dayId, @@ -4572,6 +4608,7 @@ class DerivationEngine { Future _sleepCandidateForDay( String dayId, { _PrepareStats? stats, + _PrepareUnion? union, }) async { // A user sleep override is the source of truth — never serve the cached auto // candidate, and don't cache the override result (so a later edit / clear is @@ -4638,7 +4675,24 @@ class DerivationEngine { revisions = null; // Unavailable provenance costs a staging, never a hit. } } - final searchSub = await _loadSubstrateRange( + Substrate? cutSearch; + if (union != null) { + final from = range.$1; + final to = _localNextDayLabelToSec(dayId) - 1 + napBoundaryBufferSec; + final (owned, ranked) = await _resolveOwnership(from, to); + union.ownership = owned; + union.priority = ranked; + union.load = await _loadUnion(dayId, from, to, owned, stats: stats, + search: union, window: (from: range.$1, to: range.$2, + ownership: searchOwnership)); + cutSearch = union.search; + union.search = null; + if (cutSearch == null) { + final cuts = await _cutUnionWindows(union.load, [range], searchOwnership); + cutSearch = cuts.$1.single; + } + } + final searchSub = cutSearch ?? await _loadSubstrateRange( range.$1, range.$2, dayId: dayId, @@ -4706,7 +4760,7 @@ class DerivationEngine { // ([_prepareTargetDay]): band rows from the previous afternoon, before // the ring went on, must not leave the ring's night unstaged. final ownNight = override == null && - searchSub.slice(_localDayLabelToSec(dayId), range.$2 + 1).isEmpty + (searchSub.lastTs ?? -1) < _localDayLabelToSec(dayId) ? await wearableNight(range.$1, range.$2) : null; if (ownNight != null) vendorNights = [...vendorNights, ownNight]; @@ -5115,6 +5169,116 @@ class DerivationEngine { return (startSec: startSec, endSec: endSec); } + /// The E13 owner holds one union per day. Only the loading turn is serial: + /// a neighbouring day can borrow its completed overlap while staging runs. + Future _loadUnion(String dayId, int from, int to, + Map> ownership, {_PrepareStats? stats, + _PrepareUnion? search, SubstrateUnionWindow? window}) { + final epoch = _retentionEpoch; + return _unionLock.run(() async { + int generation; + Map revisions; + try { + final db = await LocalDb.instance; + generation = LocalDb.storeGeneration; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [from ~/ 900, to ~/ 900]); + revisions = {for (final r in rows) + (r['bucket'] as num).toInt(): (r['rev'] as num).toInt()}; + } catch (_) { + _dropRetainedDay(dayId); + return null; + } + final context = substrateLoadContext(from, to, generation, ownership); + final bases = _canRetainSubstrate ? [for (final base in _retained.unions) + if (base.version == kAlgoVersion && base.generation == generation && + base.context == substrateLoadContext(base.from!, base.to!, + generation, base.ownership)) base] : []; + final stretches = reusableUnionStretches(bases, from, to, revisions, ownership); + final timeout = _perDayTimeout; + final whole = stretches.length == 1 && + stretches.single.$1 == stretches.single.$3.from && + stretches.single.$2 == stretches.single.$3.to; + final borrowed = whole ? [stretches.single.$3] : + stretches.isEmpty ? [] : + await _runIsolateCancellable(borrowUnionJob(stretches, ownership), + timeout, label: 'union-borrow $dayId'); + if (!whole) _unionWorkerCuts += borrowed.whereType().length; + final parts = []; + Future read(int a, int b) async { + debugOnSubstrateLoad?.call(a, b, 'union'); + final (sub, extras) = await _readSubstrateRange(a, b, dayId: dayId, + stats: stats, ownership: ownership, retain: true, + unionWindow: a == from && b == to ? window : null); + if (extras.isEmpty) return null; + final cut = extras.remove('union_window') as Substrate?; + if (cut != null) { + search?.search = cut; + _unionWorkerCuts++; + } + return RetainedSubstrate(version: kAlgoVersion, context: context, + revisions: revisions, sub: sub, extras: extras, cut: a, + from: a, to: b, generation: generation, ownership: ownership); + } + var pos = from; + for (var i = 0; i < stretches.length; i++) { + final part = borrowed[i]; + if (part == null) continue; + if (part.from! > pos) { + final gap = await read(pos, part.from! - 1); + if (gap == null) return null; + parts.add(gap); + } + parts.add(part); + pos = part.to! + 1; + } + if (pos <= to) { + final gap = await read(pos, to); + if (gap == null) return null; + parts.add(gap); + } + var loaded = parts.length == 1 ? parts.single : + await _runIsolateCancellable( + joinUnionJob(parts, context, revisions, ownership), timeout, + label: 'union-join $dayId'); + loaded ??= await read(from, to); + if (loaded == null) return null; + final rows = (loaded.extras['raw_rows_by_bucket'] as Map) + .values.fold(0, (int n, v) => n + v); + _enforcePrepareBudget(dayId: dayId, fromRecTs: from, toRecTs: to, + rangePages: (rows / _rawDecodeBatchSize).ceil(), rangeRows: rows); + if (_canRetainSubstrate && epoch == _retentionEpoch && + LocalDb.storeGeneration == generation && + loaded.sub.length <= _maxDayRawRows && + loaded.sub.rrMs.length <= _maxUnionBeats) { + // A sole borrowed part still carries its original, wider context. + loaded = RetainedSubstrate(version: kAlgoVersion, context: context, + revisions: revisions, sub: loaded.sub, extras: loaded.extras, + cut: from, from: from, to: to, generation: generation, + ownership: ownership); + _retained.store(dayId, 'union', loaded); + } + return loaded; + }); + } + + /// The calendar day deliberately keeps the buffered load's provenance and + /// beat-time slice semantics, just as the separate-window path does. + Future<(List, Substrate?)> _cutUnionWindows( + RetainedSubstrate? load, List<(int, int)> ranges, + Map> ownership, + {int? dayStart, int? dayEnd}) async { + if (load == null) return ([for (final _ in ranges) null], null); + final timeout = _perDayTimeout; + final cuts = await _runIsolateCancellable( + cutUnionJob(load, ranges, ownership, dayStart, dayEnd), + timeout, label: 'union-windows'); + _unionWorkerCuts += cuts.$1.whereType().length; + if (cuts.$2 != null) _unionWorkerCuts++; + return cuts; + } + Future _loadSubstrateRange( int fromRecTs, int toRecTs, { @@ -5185,6 +5349,7 @@ class DerivationEngine { Map> ownership = const {}, RetainedSubstrate? seed, bool retain = false, + SubstrateUnionWindow? unionWindow, }) async { if (toRecTs < fromRecTs) return (Substrate.empty, const {}); final port = ReceivePort(); @@ -5226,6 +5391,11 @@ class DerivationEngine { final payload = ((message['payload'] as Map?) ?? const {}) .cast(); extras = ((message['retain'] as Map?) ?? const {}).cast(); + final window = message['union_window']; + if (window is Map) { + extras = {...extras, + 'union_window': Substrate.fromTransfer(window.cast())}; + } if (!result.isCompleted) result.complete(Substrate.fromTransfer(payload)); } return; @@ -5257,7 +5427,10 @@ class DerivationEngine { } try { final worker = await ready.future; - worker.send({'type': 'config', 'mode': 'substrate', 'retain': retain}); + worker.send({'type': 'config', 'mode': 'substrate', 'retain': retain, + if (unionWindow != null) ...{'union_window': unionWindow, + 'load_from': fromRecTs, 'load_to': toRecTs, 'ownership': ownership}, + }); if (seed != null) worker.send(seed.seed(fromRecTs)); int? afterRecTs; int? afterCursor; diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index d398018e5..12845ca04 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -6,6 +6,8 @@ import 'package:openstrap_analytics/onehz.dart' as ana; import '../ble/adapters/signals.dart'; import '../data/coverage_resolver.dart'; import 'substrate.dart'; +import 'substrate_retention.dart'; +import 'substrate_union.dart'; import 'vendor_sleep.dart'; class PreparedDerivationDay { @@ -317,6 +319,9 @@ void derivationPrepareWorker(SendPort mainSendPort) { final state = _PrepareAccumulator(); String? targetDay; var mode = 'prepared_day'; + SubstrateUnionWindow? unionWindow; + var loadFrom = 0, loadTo = -1; + Map> loadOwnership = const {}; mainSendPort.send(port.sendPort); void handle(Object? message) { @@ -348,6 +353,10 @@ void derivationPrepareWorker(SendPort mainSendPort) { final cfgMode = message['mode']?.toString(); if (cfgMode != null && cfgMode.isNotEmpty) mode = cfgMode; state.retain = message['retain'] == true; + unionWindow = message['union_window'] as SubstrateUnionWindow?; + loadFrom = message['load_from'] as int? ?? 0; + loadTo = message['load_to'] as int? ?? -1; + loadOwnership = message['ownership'] as Map>? ?? const {}; return; } if (type == 'seed') { @@ -358,11 +367,20 @@ void derivationPrepareWorker(SendPort mainSendPort) { try { final substrate = state.buildSubstrate(); if (mode == 'substrate') { + final extras = state.retainable ? state.retainExtras() : null; + final request = unionWindow; + final window = request == null || extras == null ? null : + cutSubstrateUnion(RetainedSubstrate(version: 0, context: '', + revisions: const {}, sub: substrate, extras: extras, cut: loadFrom, + from: loadFrom, to: loadTo, ownership: loadOwnership), + request.from, request.to, request.ownership, + retainMetadata: false)?.sub; mainSendPort.send({ 'type': 'result', 'kind': 'substrate', 'payload': substrate.toJson(), - if (state.retainable) 'retain': state.retainExtras(), + 'retain': ?extras, + if (window != null) 'union_window': window.toJson(), }); } else { final payload = prepareDerivationPayload( @@ -661,6 +679,7 @@ class _PrepareAccumulator { final List _rrStart = []; final Map _buckets = {}; Float64List? _rrTsRaw; + final List> _provenance = []; void seed(Map message) { List ints(String k) => message[k] as List; @@ -689,6 +708,11 @@ class _PrepareAccumulator { rrTsMs.addAll(dbls('rr_ts_raw').getRange(0, beats)); rrMs.addAll(dbls('rr_ms').getRange(0, beats)); _rrStart.addAll(starts.getRange(0, n)); + for (final run in (message['provenance'] as List?) ?? const []) { + final r = (run as List).cast(); + if ((r[0] as int) >= n) break; + _provenance.add(List.from(r)); + } for (final wire in message['meta'] as List) { final m = wire as List; final bucket = m[0] as int; @@ -706,6 +730,7 @@ class _PrepareAccumulator { Map retainExtras() => { 'rr_start': Int32List.fromList(_rrStart), 'rr_ts_raw': _rrTsRaw!, + 'provenance': _provenance, 'meta': [for (final e in _buckets.entries) [e.key, e.value.families.toList(), e.value.deviceIds.toList(), e.value.skinUnit]], @@ -787,6 +812,16 @@ class _PrepareAccumulator { if (retain) { _rrStart.add(rrMs.length); _buckets.putIfAbsent(recTs ~/ 900, _BucketAcc.new).note(row); + final family = row?['device_family']; + final id = row?['device_id']; + final unit = row?['skin_temp_raw'] is num ? 'raw' : + (row?['skin_temp_c'] is num ? 'c' : null); + final f = family is String && family.isNotEmpty ? family : null; + final d = id is String ? id : null; + final last = _provenance.isEmpty ? null : _provenance.last; + if (last == null || last[1] != f || last[2] != d || last[3] != unit) { + _provenance.add([tsSec.length, f, d, unit]); + } } tsSec.add(recTs); // NULL (absent), a beat-only second (no row at all) and an impossible diff --git a/lib/compute/substrate_retention.dart b/lib/compute/substrate_retention.dart index af77a2b6a..2cfdd50fe 100644 --- a/lib/compute/substrate_retention.dart +++ b/lib/compute/substrate_retention.dart @@ -8,7 +8,8 @@ import 'substrate.dart'; class RetainedSubstrate { RetainedSubstrate({required this.version, required this.context, required this.revisions, required this.sub, required this.extras, - required this.cut}); + required this.cut, this.from, this.to, this.generation, + this.ownership = const {}}); int version; final String context; @@ -16,6 +17,8 @@ class RetainedSubstrate { final Substrate sub; final Map extras; final int cut; + final int? from, to, generation; + final Map> ownership; /// A changed closed bucket requires a cold load. Otherwise rebuild the tail /// from [cut], or return the whole load when every bucket is unchanged. @@ -35,7 +38,7 @@ class RetainedSubstrate { }; } -/// At most three loads per day: search, buffered day and selected sleep. +/// At most three loads per day: separate windows, or their shared union. /// Both days and loads are ordered least recently used first. class SubstrateRetention { final _days = >{}; @@ -49,6 +52,12 @@ class SubstrateRetention { } } Set get days => _days.keys.toSet(); + Iterable get unions sync* { + for (final loads in _days.values) { + final union = loads['union']; + if (union != null) yield union; + } + } RetainedSubstrate? lookup(String day, String load) { final loads = _days.remove(day); diff --git a/lib/compute/substrate_union.dart b/lib/compute/substrate_union.dart new file mode 100644 index 000000000..c52be1148 --- /dev/null +++ b/lib/compute/substrate_union.dart @@ -0,0 +1,271 @@ +import 'dart:typed_data'; + +import '../ble/adapters/signals.dart'; +import '../data/coverage_resolver.dart'; +import 'substrate.dart'; +import 'substrate_retention.dart'; + +typedef SubstrateUnionWindow = ({int from, int to, + Map> ownership}); + +/// Compare the loader's decisions, including null-owner (unfiltered) spans. +/// Wider resolver windows can seed hysteresis differently. +bool sameSubstrateOwnership(Map> a, + Map> b, int from, int to) { + List clip(List spans) { + final out = []; + for (final s in spans) { + if (s.deviceId == null || s.end <= from || s.start > to) continue; + final lo = s.start < from ? from : s.start; + final hi = s.end > to + 1 ? to + 1 : s.end; + if (out.isNotEmpty && out.last.end == lo && + out.last.deviceId == s.deviceId) { + out[out.length - 1] = + (start: out.last.start, end: hi, deviceId: s.deviceId); + } else { + out.add((start: lo, end: hi, deviceId: s.deviceId)); + } + } + return out; + } + for (final signal in InputSignal.values) { + final x = clip(a[signal] ?? const []), y = clip(b[signal] ?? const []); + if (x.length != y.length) return false; + for (var i = 0; i < x.length; i++) { + if (x[i] != y[i]) return false; + } + } + return true; +} + +int _lowerBound(List ts, int value) { + var lo = 0, hi = ts.length; + while (lo < hi) { + final mid = (lo + hi) >> 1; + if (ts[mid] < value) { lo = mid + 1; } else { hi = mid; } + } + return lo; +} + +List> _runs(RetainedSubstrate load) => + (load.extras['provenance'] as List).cast>(); +String? _unit(List> runs) { + for (final r in runs) { + if (r[3] != null) return r[3] as String; + } + return null; +} +(Set, Set) _sources(List> runs) { + final families = {}, ids = {}; + for (final r in runs) { + if (r[1] != null) families.add(r[1] as String); + if (r[2] != null) ids.add(r[2] as String); + } + return (families, ids); +} + +/// Worker-only: reproduce a decoded load by record second, restarting the +/// beat running maximum from the original placements. Substrate.slice instead +/// selects beats by their repaired time and keeps the parent's provenance. +RetainedSubstrate? cutSubstrateUnion(RetainedSubstrate load, int from, int to, + Map> ownership, {bool retainMetadata = true}) { + if (load.from == null || from < load.from! || to > load.to! || to < from || + load.extras['provenance'] == null || + !sameSubstrateOwnership(load.ownership, ownership, from, to)) { + return null; + } + final s = load.sub, n = s.length; + final starts = load.extras['rr_start'] as List; + final raw = load.extras['rr_ts_raw'] as List; + if (starts.length != n || raw.length != s.rrMs.length) return null; + final lo = _lowerBound(s.tsSec, from), hi = _lowerBound(s.tsSec, to + 1); + final r0 = lo < n ? starts[lo] : s.rrMs.length; + final r1 = hi < n ? starts[hi] : s.rrMs.length; + final runs = _runs(load), selected = >[]; + for (var i = 0; i < runs.length; i++) { + final start = runs[i][0] as int; + final end = i + 1 < runs.length ? runs[i + 1][0] as int : n; + if (end <= lo || start >= hi) continue; + selected.add([start < lo ? 0 : start - lo, ...runs[i].skip(1)]); + } + // The union has already suppressed other temperature units. A later + // window that would choose another unit must take the ordinary loader. + final unit = _unit(selected); + if (unit != null && unit != _unit(runs)) return null; + final (families, ids) = _sources(selected); + final axis = Float64List(r1 - r0)..setRange(0, r1 - r0, raw, r0); + monotonizeBeatAxis(axis); + final columns = s.toJson(); + final out = { + for (final key in _intColumns) + key: (columns[key] as List).sublist(lo, hi), + for (final key in _motionColumns) + key: (columns[key] as List).sublist(lo, hi), + 'rr_ts_ms': axis, 'rr_ms': s.rrMs.sublist(r0, r1), + 'device_family': families.length == 1 ? families.first : null, + 'device_ids': ids.toList(), + }; + return RetainedSubstrate(version: load.version, context: load.context, + revisions: load.revisions, from: from, to: to, + generation: load.generation, ownership: ownership, + sub: Substrate.fromTransfer(out), cut: from, + extras: !retainMetadata ? const {} : { + 'rr_start': Int32List.fromList([ + for (var i = lo; i < hi; i++) starts[i] - r0]), + 'rr_ts_raw': Float64List(r1 - r0)..setRange(0, r1 - r0, raw, r0), + 'provenance': selected, + 'meta': [for (final m in load.extras['meta'] as List) + if ((m as List)[0] as int >= from ~/ 900 && + m[0] as int <= to ~/ 900) m], + 'raw_rows_by_bucket': { + for (final e in ((load.extras['raw_rows_by_bucket'] as Map?) ?? + const {}).entries) + if (e.key >= from ~/ 900 && e.key <= to ~/ 900) e.key: e.value}, + }); +} + +const _intColumns = ['ts_sec', 'hr', 'spo2_red', 'spo2_ir', 'skin_temp', + 'skin_contact', 'step_count', 'hr_valid', 'band_sleep_state']; +const _motionColumns = ['ax', 'ay', 'az']; + +/// Worker-only: join disjoint loads and repair their raw beat axis once across +/// the seams. Refuse independently selected temperature units that cannot be +/// reconstructed from already suppressed columns. +RetainedSubstrate? joinSubstrateUnions(List parts, + String context, Map revisions, + Map> ownership) { + if (parts.isEmpty) return null; + final units = {for (final p in parts) _unit(_runs(p))}..remove(null); + if (units.length > 1) return null; + var n = 0, beats = 0; + for (var i = 0; i < parts.length; i++) { + if (i > 0 && parts[i].from != parts[i - 1].to! + 1) return null; + n += parts[i].sub.length; + beats += parts[i].sub.rrMs.length; + } + final columns = { + for (final key in _intColumns) key: Int64List(n), + for (final key in _motionColumns) key: Float64List(n), + 'rr_ms': Float64List(beats), + }; + final raw = Float64List(beats), starts = Int32List(n); + final runs = >[], meta = >{}; + final rawRows = {}; + var at = 0, rat = 0; + for (final p in parts) { + final source = p.sub.toJson(), pn = p.sub.length, rn = p.sub.rrMs.length; + for (final key in [..._intColumns, ..._motionColumns]) { + (columns[key] as List).setRange(at, at + pn, source[key] as List); + } + (columns['rr_ms'] as List).setRange(rat, rat + rn, p.sub.rrMs); + raw.setRange(rat, rat + rn, p.extras['rr_ts_raw'] as List); + final rs = p.extras['rr_start'] as List; + for (var i = 0; i < pn; i++) { starts[at + i] = rs[i] + rat; } + for (final r in _runs(p)) { + final last = runs.isEmpty ? null : runs.last; + if (last == null || last[1] != r[1] || last[2] != r[2] || last[3] != r[3]) { + runs.add([(r[0] as int) + at, ...r.skip(1)]); + } + } + for (final m in p.extras['meta'] as List) { + final wire = m as List; + final target = meta.putIfAbsent(wire[0] as int, + () => [wire[0], {}, {}, null]); + (target[1] as Set).addAll((wire[1] as List).cast()); + (target[2] as Set).addAll((wire[2] as List).cast()); + target[3] ??= wire[3]; + } + for (final e in (p.extras['raw_rows_by_bucket'] as Map).entries) { + rawRows[e.key] = (rawRows[e.key] ?? 0) + e.value; + } + at += pn; rat += rn; + } + final axis = Float64List.fromList(raw); + monotonizeBeatAxis(axis); + final (families, ids) = _sources(runs); + columns.addAll({'rr_ts_ms': axis, + 'device_family': families.length == 1 ? families.first : null, + 'device_ids': ids.toList()}); + return RetainedSubstrate(version: parts.first.version, context: context, + revisions: revisions, from: parts.first.from, to: parts.last.to, + generation: parts.first.generation, ownership: ownership, + sub: Substrate.fromTransfer(columns), cut: parts.first.from!, + extras: {'rr_start': starts, 'rr_ts_raw': raw, 'provenance': runs, + 'raw_rows_by_bucket': rawRows, + 'meta': [for (final m in meta.values) + [m[0], (m[1] as Set).toList(), (m[2] as Set).toList(), m[3]]], + }); +} + +/// Revision/ownership proofs are small; column cuts and joins run in workers. +List<(int, int, RetainedSubstrate)> reusableUnionStretches( + Iterable bases, int from, int to, + Map revisions, Map> ownership) { + final candidates = <(int, int, RetainedSubstrate)>[]; + for (final base in bases) { + final lo = from > base.from! ? from : base.from!; + final hi = to < base.to! ? to : base.to!; + int? start; + void close(int end) { + if (start == null) return; + final a = start! < lo ? lo : start!; + final b = end > hi ? hi : end; + start = null; + if (b < a || (a != base.from && a % 900 != 0) || + (b != base.to && (b + 1) % 900 != 0)) { return; } + if (sameSubstrateOwnership(base.ownership, ownership, a, b)) { + candidates.add((a, b, base)); + } + } + for (var b = lo ~/ 900; b <= hi ~/ 900; b++) { + if ((revisions[b] ?? 0) == (base.revisions[b] ?? 0)) { + start ??= b * 900; + } else { close(b * 900 - 1); } + } + close(hi); + } + final out = <(int, int, RetainedSubstrate)>[]; + var pos = from; + while (pos <= to) { + (int, int, RetainedSubstrate)? best; + int? next; + for (final c in candidates) { + if (c.$1 <= pos && c.$2 >= pos && + (best == null || c.$2 > best.$2)) { best = c; } + if (c.$1 > pos && (next == null || c.$1 < next)) next = c.$1; + } + if (best == null) { + if (next == null) break; + pos = next; + } else { + if (pos != best.$3.from && pos % 900 != 0) { + pos = (pos ~/ 900 + 1) * 900; + continue; + } + out.add((pos, best.$2, best.$3)); + pos = best.$2 + 1; + } + } + return out; +} + +// Factories keep dispatch closures in a scope containing only plain inputs. +List Function() borrowUnionJob( + List<(int, int, RetainedSubstrate)> stretches, + Map> ownership) => () => [ + for (final s in stretches) cutSubstrateUnion(s.$3, s.$1, s.$2, ownership)]; + +RetainedSubstrate? Function() joinUnionJob(List parts, + String context, Map revisions, + Map> ownership) => () => + joinSubstrateUnions(parts, context, revisions, ownership); + +(List, Substrate?) Function() cutUnionJob(RetainedSubstrate load, + List<(int, int)> ranges, Map> ownership, + int? dayStart, int? dayEnd) => () { + final windows = [for (final r in ranges) + cutSubstrateUnion(load, r.$1, r.$2, ownership, retainMetadata: false)?.sub]; + final day = dayStart == null || dayEnd == null ? null : + windows.first?.slice(dayStart, dayEnd); + return (windows, day); +}; diff --git a/test/ed2b_e14_union_loading_test.dart b/test/ed2b_e14_union_loading_test.dart new file mode 100644 index 000000000..c62758a8c --- /dev/null +++ b/test/ed2b_e14_union_loading_test.dart @@ -0,0 +1,128 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +String _payload(PreparedDerivationDay day) => jsonEncode({ + 'prepared': day.toJson(), 'bundle': edBundle(day)}); + +Future _night() async { + final start = DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000; + await edRows(start, start + 10 * 3600, hr: 55, az: 1); + await edRows(edStart, edStart + 1200, hr: 85); + final batch = (await LocalDb.instance).batch(); + // Multiple beats, beat-only seconds and backwards raw placements straddle + // every revision bucket; worker cuts must restart the running maximum. + for (var t = start; t < start + 10 * 3600; t += 17) { + for (var i = 0; i < 2; i++) { + batch.insert('decoded_rr', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'rr_ts_ms': t * 1000, 'beat_index': i, + 'beat_ts_ms': t * 1000 - (t % 11 == 0 ? 9000 : i * 400), + 'rr_ms': 900 + t % 90}, conflictAlgorithm: ConflictAlgorithm.replace); + } + } + await batch.commit(noResult: true); +} + +DerivationEngine _cold() => DerivationEngine() + ..debugDisableUnionLoads = true + ..debugDisableSubstrateRetention = true + ..debugDisableStagedSleepCache = true; + +void main() { + setUpAll(edInit); + setUp(() async { await edFresh('ed2b_e14.db'); await _night(); }); + tearDown(edDrop); + + test('E14 search, calendar, nap and night come from one union load with worker cuts', () async { + final loads = <(int, int, String)>[]; + final e = DerivationEngine()..debugDisableSubstrateRetention = true + ..debugOnSubstrateLoad = (a, b, label) => loads.add((a, b, label)); + final day = await e.debugPrepareDay(edDay); + expect(day, isNotNull); + expect(day!.sleepSub.isEmpty, isFalse, reason: 'fixture must reach the staged-night cut'); + final cold = await _cold().debugPrepareDay(edDay); + expect(_payload(day), _payload(cold!)); + expect(loads.length, 1, + reason: 'overlapping search/day/night rows must be loaded once as a union'); + expect(e.debugUnionWorkerCuts, greaterThanOrEqualTo(3), + reason: 'UI transfers encoded substrate; workers cut all requested windows'); + }); + + test('E14 neighbouring days borrow overlap rather than loading it twice', () async { + final next = DateTime(2020, 1, 3, 5).millisecondsSinceEpoch ~/ 1000; + await edRows(next, next + 1200, hr: 57); + final e = DerivationEngine(); + await e.debugPrepareDay(edDay); + final before = e.snapshot()['raw_rows'] as int; + final warm = await e.debugPrepareDay('2020-01-03'); + final cold = await _cold().debugPrepareDay('2020-01-03'); + expect(_payload(warm!), _payload(cold!)); + final reread = (e.snapshot()['raw_rows'] as int) - before; + final coldEngine = _cold(); + await coldEngine.debugPrepareDay('2020-01-03'); + expect(reread, lessThan(coldEngine.snapshot()['raw_rows'] as int), + reason: 'neighbour union must retain its earlier shared hours'); + }); + + final changes = Function(DerivationEngine)>{ + 'input revision': (_) => edRows(edStart, edStart + 1, hr: 91), + 'late RR': (_) async { + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': edStart * 1000, 'rec_ts': edStart, 'rr_ts_ms': edStart * 1000, + 'beat_index': 0, 'rr_ms': 850}, conflictAlgorithm: ConflictAlgorithm.replace); + }, + 'generation': (_) async { await LocalDb.close(); await LocalDb.instance; }, + 'algorithm': (e) async { e.debugRekeyUnionAlgorithm(kAlgoVersion - 1); }, + }; + for (final change in changes.entries) { + test('E14 ${change.key}: refuse stale union, cold bytes exact, then reuse worker cuts', () async { + final e = DerivationEngine()..debugDisableStagedSleepCache = true; + await e.debugPrepareDay(edDay); + final before = e.snapshot()['raw_rows'] as int; + await change.value(e); + final warm = await e.debugPrepareDay(edDay); + final cold = await _cold().debugPrepareDay(edDay); + expect(_payload(warm!), _payload(cold!)); + final refreshed = e.snapshot()['raw_rows'] as int; + expect(refreshed, greaterThan(before), + reason: 'changed provenance must reload rather than hit a stale union'); + final cuts = e.debugUnionWorkerCuts; + await e.debugPrepareDay(edDay); + expect(e.snapshot()['raw_rows'], refreshed, + reason: 'unchanged replacement provenance must reuse the refreshed union'); + expect(e.debugUnionWorkerCuts, greaterThan(cuts), + reason: 'valid refreshed union supplies worker cuts after invalidation'); + }); + } + + test('E14 D35 endpoint: union cuts agree with a separately owned single second', () async { + final db = await LocalDb.instance; + final e = DerivationEngine(); + final range = e.debugTargetDayWindow(edDay); + final end = range.$2; + await edRows(end, end + 1, hr: 61); + await edRows(end, end + 1, hr: 199, id: 'non-owner'); + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': range.$1, 'end_ts': end + 1}); + final day = await e.debugPrepareDay(edDay); + expect(day!.daySub.hr.last, 61); + final last = await DerivationEngine().debugLoadOwnedSubstrate(end, end, day: edDay); + expect(last.hr, [61], reason: 'D35 is a prerequisite for exact union cuts'); + expect(last.hr, day.daySub.slice(end, end + 1).hr); + }); + + test('E14 disabled union diagnostic is behavior-neutral (guard)', () async { + final ordinary = await DerivationEngine().debugPrepareDay(edDay); + final disabled = await _cold().debugPrepareDay(edDay); + expect(_payload(ordinary!), _payload(disabled!)); + }); +} From 541a1dad9cacf472bf466b86454fab3b6af0c8da Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 11:22:30 -0600 Subject: [PATCH 06/11] storage: night and cardio checkpoints at schema rung 62, valid across restarts - day_checkpoint keeps a versioned, checksummed night/cardio state for at most three days (8 MiB cap per state), written after the day result commits. Resuming from it equals a cold derive byte for byte; a bad, old or foreign checkpoint is refused whole and the night is recomputed. - store_identity holds a random id. Wipe, a committed import or restore, salvage/rebuild and a replaced file get a new id and clear checkpoints, so a checkpoint survives an app restart but never a different store. RAM owners still key on the per-open generation; an algorithm change never adopts a disk checkpoint. - Rung 62 is additive and idempotent, repeated by open repair, with no scan or backfill. day_checkpoint is left out of diagnostic exports. Preserve restore skip forwarding, main's race-table classification, and ED-2a's revision tables at rung 61. Checkpoint migration tests use 61 to 62. --- lib/compute/derivation_engine.dart | 189 +++++- lib/compute/night_checkpoint.dart | 115 ++++ lib/compute/night_prsa.dart | 16 + lib/compute/night_resp_bins.dart | 36 ++ lib/compute/night_rr_state.dart | 167 +++++ lib/compute/staged_sleep_cache.dart | 6 +- lib/data/db.dart | 124 +++- lib/data/recent_slice.dart | 2 + lib/data/restore_classification.dart | 2 + test/data_ownership_test.dart | 10 +- test/ed2a_e18_session_reuse_test.dart | 13 +- test/ed2a_input_revision_test.dart | 8 +- test/ed2b_cardio_cache_test.dart | 3 + test/ed2b_checkpoints_test.dart | 854 ++++++++++++++++++++++++++ test/ed2b_streams_test.dart | 6 +- test/import_bulk_restore_test.dart | 5 +- 16 files changed, 1513 insertions(+), 43 deletions(-) create mode 100644 lib/compute/night_checkpoint.dart create mode 100644 test/ed2b_checkpoints_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 0e04cd2d8..8032e74de 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -36,6 +36,7 @@ import 'readiness_calibration_backfill.dart' show backfillReadinessCalibration, seedRecentReadinessZ; import 'package:crypto/crypto.dart' show sha1; +import 'package:sqflite/sqflite.dart' show ConflictAlgorithm; import 'package:flutter/foundation.dart'; import 'findings.dart'; import 'nap_overlap_cleanup.dart'; @@ -77,6 +78,7 @@ import 'staged_sleep_cache.dart'; import 'ed2b_stream_diagnostics.dart'; import 'cardio_rows_cache.dart'; import 'night_rr_state.dart'; +import 'night_checkpoint.dart'; import 'substrate_retention.dart'; import 'substrate_union.dart'; import 'derive_prepare.dart'; @@ -3136,7 +3138,7 @@ class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; - // ED-2b observations. Checkpoint controls await their GREEN phase. + // ED-2b observations of the worker-owned caches and checkpoint path. final debugCheckpoints = Ed2bCheckpointDiagnostics(); @visibleForTesting void Function(String day)? debugOnProfileFold; @@ -3237,6 +3239,8 @@ class DerivationEngine { final _cardioRowChunks = []; final _cardioRowOperations = {}; final _nightStates = {}; + final _checkpointPlans = {}; + int? _checkpointGeneration; static const _cardioRowsSlackSec = 3600; static const _cardioRowsMaxBytes = 8 * 1024 * 1024; int _retentionEpoch = 0; @@ -3253,6 +3257,9 @@ class DerivationEngine { _retained.clear(); _cardioRowChunks.clear(); _nightStates.clear(); + _checkpointPlans.clear(); + debugCheckpoints.lastEncodedPayload = null; + debugCheckpoints.lastDecodedPayload = null; debugNightStreams?.nightState.clear(); } @@ -3271,11 +3278,13 @@ class DerivationEngine { }); _retentionPreparedDays.clear(); _retentionPassEpoch = null; + _checkpointPlans.clear(); } void _dropRetainedDay(String day) { _retained.remove(day); _nightStates.remove(day)?.clear(); + _checkpointPlans.remove(day); _retentionPreparedDays.remove(day); } @visibleForTesting @@ -4658,6 +4667,31 @@ class DerivationEngine { final (searchOwnership, searchPriority) = await _resolveOwnership(range.$1, range.$2); final rowsEpoch = _retentionEpoch; + _checkpointPlans.remove(dayId); + NightCheckpointPlan? checkpointPlan; + if (!debugCheckpoints.disabled && _canRetainSubstrate) { + try { + final db = await LocalDb.instance; + final generation = LocalDb.storeGeneration; + if (_checkpointGeneration != null && _checkpointGeneration != generation) { + // Invalidate cached state while preserving the owners' work counters. + for (final state in _nightStates.values) { state.clear(); } + _cardioRowChunks.clear(); + _checkpointPlans.clear(); + debugCheckpoints.lastEncodedPayload = null; + debugCheckpoints.lastDecodedPayload = null; + } + _checkpointGeneration = generation; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [range.$1 ~/ 900, range.$2 ~/ 900]); + checkpointPlan = NightCheckpointPlan(range.$1, range.$2, + {for (final r in rows) r['bucket'] as int: r['rev'] as int}, + generation, rowsEpoch, await LocalDb.storeIdentity(executor: db)); + } catch (_) { + // Missing provenance costs a cold pass, never a partial adoption. + } + } List>? revisions; if (!debugDisableStagedSleepCache || (!debugDisableCardioRows && _canRetainSubstrate)) { @@ -4765,12 +4799,13 @@ class DerivationEngine { : null; if (ownNight != null) vendorNights = [...vendorNights, ownNight]; String? stagingKey; - if (revisions != null && !debugDisableStagedSleepCache) { + if ((revisions != null && !debugDisableStagedSleepCache) || checkpointPlan != null) { try { - stagingKey = stagingInputKey( + final stagingWearable = override == null ? await activeWearable() : null; + String inputKey(bool prefixContext) => stagingInputKey( fromSec: range.$1, toSec: range.$2, - revisions: revisions, + revisions: revisions ?? const [], sub: searchSub, ownership: searchOwnership, priority: searchPriority, @@ -4778,10 +4813,21 @@ class DerivationEngine { profileJson: profileJson, priorSleep: priorSleep, vendorNights: vendorNights, - activeWearable: override == null ? await activeWearable() : null, + activeWearable: stagingWearable, + prefixContext: prefixContext, ); + if (revisions != null && !debugDisableStagedSleepCache) stagingKey = inputKey(false); + if (checkpointPlan != null) { + // Prefix reuse omits revision aggregates. The durable store id + // survives restart and changes when wipe/restore replaces inputs. + checkpointPlan.context = sha1.convert(utf8.encode(jsonEncode([ + checkpointPlan.storeId, inputKey(true), + ]))).toString(); + checkpointPlan.boundary = ((searchSub.lastTs ?? 0) + 1) ~/ 900 * 900; + } } catch (_) { // A context that cannot be keyed still takes the ordinary worker path. + checkpointPlan = null; } } // Cancellable + TIMED OUT. This site previously used a bare `Isolate.run` @@ -4809,6 +4855,46 @@ class DerivationEngine { final sentChunks = cacheRows ? [for (final c in _cardioRowChunks) if (c.overlaps(rowsLo, rowsHi)) c] : []; final rowBytes = [for (final c in sentChunks) c.bytes]; + if (checkpointPlan != null && checkpointPlan.boundary > range.$1 && + checkpointPlan.context != null) { + if (_nightStates[dayId]?.canStream != true && + _nightStates[dayId]?.acceptsCheckpointAlgorithm(kAlgoVersion) != false) { + try { + debugCheckpoints.reads++; + final row = await LocalDb.dayCheckpoint(dayId, kAlgoVersion, + maxBytes: maxNightCheckpointBytes); + if (row == null) { + debugCheckpoints.misses++; + } else { + final context = checkpointPlan.context!; + final from = checkpointPlan.from, through = checkpointPlan.through; + final inputRevisions = checkpointPlan.revisions; + final searchRr = searchSub.rrMs, searchTimes = searchSub.rrTsMs; + final restored = await _runIsolateCancellable( + () => decodeNightCheckpoint(row, dayId, kAlgoVersion, context, + from, through, inputRevisions, + searchRr: searchRr, searchTimes: searchTimes), _perDayTimeout, + label: 'checkpoint-read $dayId'); + if (restored.rejection != null) { + debugCheckpoints.rejections.add(restored.rejection!); + debugCheckpoints.misses++; + } else if (LocalDb.storeGeneration == checkpointPlan.generation && + rowsEpoch == _retentionEpoch) { + checkpointPlan.restored = restored.night; + debugCheckpoints.lastDecodedPayload = restored.payload; + if (cacheRows && restored.cardio != null) rowBytes.add(restored.cardio!); + } + } + } catch (_) { + debugCheckpoints.misses++; + } + } + checkpointPlan.cardio = rowBytes; + _checkpointPlans[dayId] = checkpointPlan; + while (_checkpointPlans.length > 3) { + _checkpointPlans.remove(_checkpointPlans.keys.first); + } + } Future<(String, String?, int, Uint8List?, int, int, List)> stage() => _runIsolateCancellable(() { try { @@ -4908,6 +4994,9 @@ class DerivationEngine { } else { debugStagingComputations++; final result = await stage(); + if (checkpointPlan != null && result.$4 != null) { + checkpointPlan.cardio = [result.$4!]; + } candidateJson = result.$1; observationJson = result.$2; debugCardioRowHits += result.$5; @@ -6685,6 +6774,14 @@ class DerivationEngine { final nightEpoch = _retentionEpoch; final retainNight = _canRetainSubstrate && !suppliedSubstrate; final diagnostics = debugNightStreams; + final checkpointPlan = _checkpointPlans.remove(day.date); + final checkpointEnabled = !debugCheckpoints.disabled && retainNight && + checkpointPlan?.context != null && checkpointPlan!.epoch == nightEpoch && + checkpointPlan.generation == LocalDb.storeGeneration && + day.sleepOnsetSec >= checkpointPlan.from && + day.sleepOffsetSec <= checkpointPlan.through + 1; + var nightRevision = 0, nightGeneration = LocalDb.storeGeneration; + var nightProvenance = false; // Keep DB reads and engine fields outside the shipped closure. A changed // revision rejects even a byte-identical replacement; raw prefix proofs in // the worker additionally detect moved windows and in-memory mutations. @@ -6707,18 +6804,61 @@ class DerivationEngine { } nightState.context(inputRevision: revision, generation: generation, algorithm: kAlgoVersion); + nightRevision = revision; + nightGeneration = generation; + nightProvenance = true; } catch (_) { nightState.clear(); // Unavailable provenance never certifies reuse. } diagnostics?.nightState = nightState; } - final workerResult = await _runIsolateCancellable( - () => (deriveDayBundle(withHistory, streamDiagnostics: diagnostics, - nightState: nightState), nightState, diagnostics), + final canCheckpoint = checkpointEnabled && nightProvenance && + nightEpoch == _retentionEpoch && + LocalDb.storeGeneration == checkpointPlan.generation; + final restoredNight = canCheckpoint ? checkpointPlan.restored : null; + final checkpointContext = checkpointEnabled ? checkpointPlan.context! : ''; + final checkpointBoundary = checkpointEnabled ? checkpointPlan.boundary : 0; + final checkpointFrom = checkpointEnabled ? checkpointPlan.from : 0; + final checkpointVector = checkpointEnabled ? checkpointRevisions( + checkpointPlan.revisions, checkpointFrom, checkpointBoundary) : Uint8List(0); + final checkpointRows = checkpointEnabled ? checkpointPlan.cardio : []; + final checkpointDay = day.date; + final workerResult = await _runIsolateCancellable(() { + var state = nightState; + var resumed = false; + if (restoredNight != null && restoredNight.acceptsPrefix( + input.sleepRrMs, input.sleepRrTsMs, input.date, + input.sleepOnsetSec)) { + restoredNight.bindRestoredContext(nightRevision, nightGeneration, kAlgoVersion, + previous: state); + state = restoredNight; + resumed = true; + if (diagnostics != null) diagnostics.nightState = state; + } + final result = deriveDayBundle(withHistory, streamDiagnostics: diagnostics, + nightState: state); + NightCheckpointWrite? checkpoint; + if (canCheckpoint) { + try { + checkpoint = encodeNightCheckpoint(state, checkpointRows, checkpointDay, + kAlgoVersion, checkpointContext, checkpointFrom, checkpointBoundary, + checkpointVector); + } catch (_) { + // Scratch serialization cannot turn a successful derive into an error. + } + } + return (result, state, diagnostics, resumed, + restoredNight != null && !resumed, checkpoint); + }, _perDayTimeout, label: 'day-bundle ${day.date}', ); final bundle = workerResult.$1; + if (workerResult.$4) debugCheckpoints.hits++; + if (workerResult.$5) { + debugCheckpoints.rejections.add('checkpoint night prefix/window'); + debugCheckpoints.misses++; + } if (diagnostics != null) debugNightStreams = workerResult.$3; final returnedNight = workerResult.$2; if (returnedNight != null && retainNight && _canRetainSubstrate && @@ -7572,6 +7712,39 @@ class DerivationEngine { }, ); if (effectiveFinalized || effectivePartial) _dropRetainedDay(day.date); + if (!debugCheckpoints.disabled && !suppliedSubstrate) { + try { + final db = await LocalDb.instance; + if (effectiveFinalized || effectivePartial) { + await db.delete('day_checkpoint', where: 'day_id = ?', whereArgs: [day.date]); + } else if (checkpointEnabled && workerResult.$6 != null && + _canRetainSubstrate && nightEpoch == _retentionEpoch && + LocalDb.storeGeneration == checkpointPlan.generation) { + final checkpoint = workerResult.$6!; + await db.transaction((txn) async { + if (LocalDb.storeGeneration != checkpointPlan.generation || + await LocalDb.storeIdentity(executor: txn) != checkpointPlan.storeId) { + return; + } + await txn.insert('day_checkpoint', { + 'day_id': day.date, 'algo_version': kAlgoVersion, + 'fmt': nightCheckpointFormat, 'ctx_sig': checkpointContext, + 'cp_rec_ts': checkpointBoundary, 'rev_vec': checkpointVector, + 'state': checkpoint.bytes, 'night_ref': null, + 'computed_at': DateTime.now().millisecondsSinceEpoch, + }, conflictAlgorithm: ConflictAlgorithm.replace); + await txn.rawDelete('DELETE FROM day_checkpoint WHERE day_id NOT IN ' + '(SELECT day_id FROM day_checkpoint GROUP BY day_id ' + 'ORDER BY day_id DESC LIMIT 3) OR algo_version != ?', [kAlgoVersion]); + }); + debugCheckpoints.writes++; + debugCheckpoints.lastEncodedPayload = checkpoint.bytes; + debugCheckpoints.lastDecodedPayload = checkpoint.payload; + } + } catch (_) { + // Disposable scratch is never required to commit a day result. + } + } // AFTER the row: retired only once the row that proves it is durable. if (quietSettled) await LocalDb.deleteComputeFreshness(quietKey); // NOTE: the sweep's `history` snapshot is deliberately NOT updated here. diff --git a/lib/compute/night_checkpoint.dart b/lib/compute/night_checkpoint.dart new file mode 100644 index 000000000..d30e9491a --- /dev/null +++ b/lib/compute/night_checkpoint.dart @@ -0,0 +1,115 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; + +import 'cardio_rows_cache.dart'; +import 'night_rr_state.dart'; + +// Format 2 carries RSA v3 and closed-bin spectral states, not saved rates. +// Older closed windows cannot be certified under the current RSA floor gate. +const nightCheckpointFormat = 2; +const maxNightCheckpointBytes = 8 * 1024 * 1024; + +/// One request's provenance, read before loading substrate. No decoded rows. +class NightCheckpointPlan { + NightCheckpointPlan(this.from, this.through, this.revisions, this.generation, + this.epoch, this.storeId); + final int from, through, generation, epoch; + final String storeId; + final Map revisions; + String? context; + int boundary = 0; + List cardio = []; + NightRrState? restored; +} + +Uint8List checkpointRevisions(Map revisions, int from, int boundary) => + Uint8List.fromList(utf8.encode(jsonEncode([ + for (var b = from ~/ 900; b < boundary ~/ 900; b++) + [b, revisions[b] ?? 0], + ]))); + +typedef NightCheckpointRead = ({NightRrState? night, Uint8List? cardio, + Map? payload, String? rejection}); +typedef NightCheckpointWrite = ({Uint8List bytes, Map payload}); + +/// Worker only. SHA-256 covers both sections and all SQL provenance. No section +/// is returned unless every section and the complete revision vector validate. +NightCheckpointRead decodeNightCheckpoint(Map row, + String day, int algorithm, String context, int from, int through, + Map revisions, {List? searchRr, List? searchTimes}) { + try { + final boundary = row['cp_rec_ts']; + if (row['day_id'] != day || row['algo_version'] != algorithm || + row['fmt'] != nightCheckpointFormat || row['ctx_sig'] != context || + row['night_ref'] != null || boundary is! int || boundary % 900 != 0 || + boundary <= from || boundary > through + 1) { + throw const FormatException('checkpoint provenance'); + } + final expectedRevisions = checkpointRevisions(revisions, from, boundary); + if (base64Encode(row['rev_vec'] as List) != base64Encode(expectedRevisions)) { + throw const FormatException('checkpoint revisions'); + } + final bytes = row['state'] as List; + if (bytes.length <= 32 || bytes.length > maxNightCheckpointBytes) { + throw const FormatException('checkpoint length'); + } + final body = bytes.sublist(32); + if (base64Encode(sha256.convert(body).bytes) != base64Encode(bytes.sublist(0, 32))) { + throw const FormatException('checkpoint checksum'); + } + final payload = (jsonDecode(utf8.decode(body)) as Map).cast(); + if (payload.length != 4 || payload['v'] != nightCheckpointFormat || + jsonEncode(payload['provenance']) != jsonEncode([ + day, algorithm, context, boundary, base64Encode(expectedRevisions), + ])) { + throw const FormatException('checkpoint envelope'); + } + final night = NightRrState.fromCheckpoint( + (payload['night'] as Map).cast(), boundary); + if (((payload['night'] as Map)['tag'] as Map)['day'] != day || + (searchRr != null && (searchTimes == null || + !night.acceptsSearchPrefix(searchRr, searchTimes, day)))) { + throw const FormatException('checkpoint raw prefix'); + } + final cardioJson = (payload['cardio'] as Map).cast(); + final cardio = ana.CardioRowCache.fromJson(cardioJson); + if (cardio.length > 8000) throw const FormatException('checkpoint cardio size'); + var outside = false; + cardio.retainWhere((k) { + if (k.startSec < from - 3600 || k.startSec + k.epochSec > boundary) outside = true; + return true; + }); + if (outside) throw const FormatException('checkpoint cardio window'); + return (night: night, + cardio: Uint8List.fromList(utf8.encode(jsonEncode(cardioJson))), + payload: payload, rejection: null); + } catch (e) { + return (night: null, cardio: null, payload: null, + rejection: 'checkpoint refused: ${e.runtimeType}'); + } +} + +/// Worker only. Oversized or absent nights stay on the ordinary cold path. +NightCheckpointWrite? encodeNightCheckpoint(NightRrState? night, + List chunks, String day, int algorithm, String context, + int from, int boundary, Uint8List revisions) { + final nightJson = night?.checkpoint(boundary); + if (nightJson == null) return null; + final cardio = decodeCardioRows(chunks); + var kept = 0; + cardio.retainWhere((k) => k.startSec >= from - 3600 && + k.startSec + k.epochSec <= boundary && kept++ < 8000); + final payload = { + 'v': nightCheckpointFormat, + 'provenance': [day, algorithm, context, boundary, base64Encode(revisions)], + 'night': nightJson, 'cardio': cardio.toJson(), + }; + final body = utf8.encode(jsonEncode(payload)); + if (body.length + 32 > maxNightCheckpointBytes) return null; + final bytes = Uint8List.fromList([...sha256.convert(body).bytes, ...body]); + return (bytes: bytes, + payload: (jsonDecode(utf8.decode(body)) as Map).cast()); +} diff --git a/lib/compute/night_prsa.dart b/lib/compute/night_prsa.dart index 90fc6f136..14f9e2502 100644 --- a/lib/compute/night_prsa.dart +++ b/lib/compute/night_prsa.dart @@ -8,8 +8,24 @@ final _ac = prsaAggregator(deceleration: false); /// Settled night NN in reference order. Provisional tails only touch copies. class NightPrsa { + NightPrsa(); Object? _dcState = _dc.empty(), _acState = _ac.empty(); + Map toJson() => { + 'v': 1, 'dc': _dc.encodeState(_dcState), 'ac': _ac.encodeState(_acState), + }; + + factory NightPrsa.fromJson(Map json, int fed) { + if (json['v'] != 1) throw const FormatException('PRSA version'); + final dc = json['dc'], ac = json['ac']; + if (dc is! List || ac is! List || dc.first != fed || ac.first != fed) { + throw const FormatException('PRSA feed count'); + } + return NightPrsa() + .._dcState = _dc.decodeState(dc) + .._acState = _ac.decodeState(ac); + } + void fold(List nn) { final column = DoubleColumn(Float64List.fromList(nn)); _dcState = _dc.addSlice(_dcState, column, 0, column.length); diff --git a/lib/compute/night_resp_bins.dart b/lib/compute/night_resp_bins.dart index 4ac421c0e..728769538 100644 --- a/lib/compute/night_resp_bins.dart +++ b/lib/compute/night_resp_bins.dart @@ -10,6 +10,42 @@ class NightRespBins { RsaRespStream _open = RsaRespStream(); final List<({int count, RsaRespStream stream})> _closed = []; + Map toJson() => { + 'v': 2, 'origin': _origin, 'index': _openIndex, 'count': _openCount, + 'open': _open.toJson(), + 'closed': [for (final bin in _closed) + {'count': bin.count, 'state': bin.stream.toJson()}], + }; + + factory NightRespBins.fromJson(Map json, int fed) { + final origin = json['origin'], index = json['index'], count = json['count']; + final closed = json['closed'] as List; + if (json['v'] != 2 || index is! int || index < 0 || count is! int || + count < 0 || count > fed || closed.length > fed ~/ 60 || + (fed == 0 ? origin != null : origin is! num || !origin.isFinite)) { + throw const FormatException('RSA bins'); + } + final open = (json['open'] as Map).cast(); + if (open['n'] != count) throw const FormatException('RSA bin count'); + final result = NightRespBins() + .._origin = (origin as num?)?.toDouble() + .._openIndex = index + .._openCount = count + .._open = RsaRespStream.fromJson(open); + var total = count; + for (final bin in closed) { + final n = (bin as Map)['count']; + final state = (bin['state'] as Map).cast(); + if (n is! int || n < 60 || n > fed || state['n'] != n) { + throw const FormatException('closed RSA bin count'); + } + total += n; + if (total > fed) throw const FormatException('RSA bins feed count'); + result._closed.add((count: n, stream: RsaRespStream.fromJson(state))); + } + return result; + } + int _bin(double time, double origin) => ((time - origin) / 1800000.0).floor(); void _appendRate(List out, Metric metric) { diff --git a/lib/compute/night_rr_state.dart b/lib/compute/night_rr_state.dart index a9345fb98..10e4d682c 100644 --- a/lib/compute/night_rr_state.dart +++ b/lib/compute/night_rr_state.dart @@ -19,6 +19,7 @@ NightRr nightRrBatch(List rr, List times) { /// One RAM night, copied into a worker and returned with its result. Raw /// prefix copies prove append-only input; settled output is never re-corrected. class NightRrState { + NightRrState(); static const maxRetainedBeats = 200000; ana.RrCorrector? _corrector; final List _rr = [], _times = [], _nn = [], _nnTimes = []; @@ -47,6 +48,172 @@ class NightRrState { int get fullListDigests => _digests.computed; int get maximumListDigestsPerIdentity => _digests.maximumPerIdentity; + // A cleared owner still records its algorithm. A transition must rebuild; + // only a fresh owner or a same-algorithm restart may adopt disk state. + bool acceptsCheckpointAlgorithm(int algorithm) => + _context == null || _context!.$3 == algorithm; + + bool acceptsPrefix(List rr, List times, String day, int onset) { + if (_night != (day, onset) || rr.length < _rr.length || + rr.length != times.length) { + return false; + } + for (var i = 0; i < _rr.length; i++) { + if (!identical(rr[i], _rr[i]) || !identical(times[i], _times[i])) { + return false; + } + } + return true; + } + + bool acceptsSearchPrefix(List rr, List times, String day) { + if (_night?.$1 != day || rr.length != times.length || _times.isEmpty) { + return false; + } + var start = 0; + while (start < times.length && times[start] < _times.first) { start++; } + if (start + _rr.length > rr.length) return false; + for (var i = 0; i < _rr.length; i++) { + if (!identical(rr[start + i], _rr[i]) || !identical(times[start + i], _times[i])) { + return false; + } + } + return true; + } + + void bindRestoredContext(int revision, int generation, int algorithm, + {NightRrState? previous}) { + _context = (revision, generation, algorithm); + // Disk state carries no work counters. Adopting it after clearing a RAM + // owner must preserve that owner's cumulative observations. + if (previous == null) return; + processedNightBeats = previous.processedNightBeats; + nightRebuilds = previous.nightRebuilds; + _processed.addAll(previous._processed); + _slices.addAll(previous._slices); + _restarts.addAll(previous._restarts); + _digests.computed = previous._digests.computed; + _digests.maximumPerIdentity = previous._digests.maximumPerIdentity; + } + + /// Worker only. A checkpoint stops before the closed input boundary. Cutting + /// a live tail requires a separate fold; it must never rewind the live owner. + Map? checkpoint(int boundary) { + if (_corrector == null || _night == null) return null; + var n = 0; + while (n < _times.length && _times[n] < boundary * 1000) { n++; } + if (n == 0) return null; + final NightRrState state; + if (n == _rr.length) { + state = this; + } else { + state = NightRrState(); + state.correct(_rr.sublist(0, n), _times.sublist(0, n), + day: _night!.$1, onsetSec: _night!.$2); + if (_frequency != null) { + final stream = state._frequency = ana.HrvFreqStream(); + state._feed('frequency', (nn, ts) { stream.fold(nn, ts); return 1; }); + } + if (_rsa != null) { + final stream = state._rsa = ana.RsaRespStream(); + state._feed('rsa', (nn, ts) { stream.fold(nn, ts); return 1; }); + } + if (_cvhr != null) { + final stream = state._cvhr = ana.CvhrStream(); + state._feed('cvhr', (nn, ts) { stream.fold(nn, ts); return 1; }); + } + if (_prsa != null) { + final stream = state._prsa = NightPrsa(); + state._feed('prsa', (nn, ts) { stream.fold(nn); return 1; }); + } + if (_rsaBins != null) { + final stream = state._rsaBins = NightRespBins(); + state._feed('rsa_bins', stream.fold); + } + } + return { + 'v': 1, 'tag': {'day': state._night!.$1, 'onset': state._night!.$2, + 'count': state._rr.length}, + 'rr': state._rr, 'times': state._times, + 'corrector': state._corrector!.toJson(), + 'nn': state._nn, 'nnTimes': state._nnTimes, 'fed': state._fed, + 'frequency': state._frequency?.toJson(), 'rsa': state._rsa?.toJson(), + 'cvhr': state._cvhr?.toJson(), 'prsa': state._prsa?.toJson(), + 'rsa_bins': state._rsaBins?.toJson(), + }; + } + + factory NightRrState.fromCheckpoint(Map json, int boundary) { + List doubles(Object? v) { + final list = (v as List).map((x) => (x as num).toDouble()).toList(); + if (list.length > maxRetainedBeats || list.any((x) => !x.isFinite)) { + throw const FormatException('night column'); + } + return list; + } + final tag = json['tag'] as Map; + final rr = doubles(json['rr']), times = doubles(json['times']); + final nn = doubles(json['nn']), nnTimes = doubles(json['nnTimes']); + final correctorJson = (json['corrector'] as Map).cast(); + final count = correctorJson['maxArtifactRun'] == null + ? correctorJson['n'] : (correctorJson['capped'] as Map)['n']; + if (json['v'] != 1 || tag['day'] is! String || tag['onset'] is! int || + rr.isEmpty || rr.length != times.length || rr.length != tag['count'] || + count != rr.length || nn.length != nnTimes.length || nn.length > rr.length || + times.any((t) => t >= boundary * 1000) || + correctorJson['maxArtifactRun'] != 32 || correctorJson['alpha'] != 5.2 || + correctorJson['windowBeats'] != 91 || correctorJson['minThresholdMs'] != 100 || + correctorJson['reanchorGapMs'] != 1000) { + throw const FormatException('night shape/count'); + } + for (var i = 1; i < times.length; i++) { + if (times[i] < times[i - 1]) throw const FormatException('night times'); + } + for (var i = 1; i < nnTimes.length; i++) { + if (nnTimes[i] < nnTimes[i - 1]) throw const FormatException('NN times'); + } + final state = NightRrState() + .._night = (tag['day'] as String, tag['onset'] as int) + .._corrector = ana.RrCorrector.fromJson(correctorJson) + .._rr.addAll(rr).._times.addAll(times).._nn.addAll(nn).._nnTimes.addAll(nnTimes); + final fed = (json['fed'] as Map).cast(); + const keys = ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']; + if (fed.keys.any((k) => !keys.contains(k))) { + throw const FormatException('night stream key'); + } + for (final key in keys) { + final data = json[key]; + if (data == null) { + if (fed.containsKey(key)) throw const FormatException('missing stream'); + continue; + } + if (fed[key] != nn.length) throw const FormatException('stream feed count'); + final map = (data as Map).cast(); + if (key == 'prsa') { + state._prsa = NightPrsa.fromJson(map, fed[key]!); + } else if (key == 'rsa_bins') { + state._rsaBins = NightRespBins.fromJson(map, fed[key]!); + } else { + if (map['n'] != fed[key]) throw const FormatException('stream count'); + if (key == 'frequency' && map['oversample'] != 4.0) { + throw const FormatException('frequency config'); + } + if (key == 'cvhr') { + final defaults = ana.CvhrStream().toJson(); + for (final name in ['minWidthSec', 'maxWidthSec', 'depthWidthRatioMin', + 'envWindowSec', 'maxGapSec']) { + if (map[name] != defaults[name]) throw const FormatException('CVHR config'); + } + } + if (key == 'frequency') state._frequency = ana.HrvFreqStream.fromJson(map); + if (key == 'rsa') state._rsa = ana.RsaRespStream.fromJson(map); + if (key == 'cvhr') state._cvhr = ana.CvhrStream.fromJson(map); + } + } + state._fed.addAll(fed); + return state; + } + void beginPass() => _digests.clear(); void endPass() => _digests.clear(); diff --git a/lib/compute/staged_sleep_cache.dart b/lib/compute/staged_sleep_cache.dart index 30fe3d638..67174684f 100644 --- a/lib/compute/staged_sleep_cache.dart +++ b/lib/compute/staged_sleep_cache.dart @@ -76,6 +76,7 @@ String stagingInputKey({ required List<({int startSec, int endSec, String dayKey})> priorSleep, required List vendorNights, required (String, String)? activeWearable, + bool prefixContext = false, }) { // Count local midnights, including DST days, rather than dividing by 86400. var walked = 0; @@ -114,12 +115,13 @@ String stagingInputKey({ } final now = DateTime.now(); return jsonEncode([ - fromSec, toSec, revisions, + fromSec, toSec, prefixContext ? const [] : revisions, [for (final sig in InputSignal.values) [sig.name, [for (final o in ownership[sig] ?? const []) [o.start, o.end, o.deviceId]], priority[sig]]], sub.deviceFamily, (sub.deviceIds.toList()..sort()), - sub.length, sub.firstTs, sub.lastTs, sub.rrMs.length, + if (prefixContext) ...[null, null, null, null] + else ...[sub.length, sub.firstTs, sub.lastTs, sub.rrMs.length], now.timeZoneName, now.timeZoneOffset.inSeconds, zoneOffsets, override == null ? null : [override.dayId, override.onsetSec, override.offsetSec, override.source], diff --git a/lib/data/db.dart b/lib/data/db.dart index c225345cc..dbf819ae4 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -688,7 +688,7 @@ class LocalDb { /// pass it: sqflite throws `ArgumentError('onCreate must be null if no /// version is specified')` BEFORE opening anything when `onCreate` is given /// without `version` (sqflite_common database_mixin.dart). - static const int schemaVersion = 61; + static const int schemaVersion = 62; /// In-memory cache identity, advanced after open, wipe or committed import. /// It is local to this isolate; no cache survives a process restart. @@ -1540,6 +1540,10 @@ class LocalDb { await _createInputRev(db); await _createSessionCeilingRev(db); } + if (oldV < 62) { + await _createStoreIdentity(db); + await _createDayCheckpoint(db); + } }, onOpen: (db) async { await _repairOpenSchema(db); @@ -1662,12 +1666,68 @@ class LocalDb { // A decoded-table rebuild above drops its triggers; attach them last. await _createInputRev(db); await _createSessionCeilingRev(db); + await _createStoreIdentity(db); + await _createDayCheckpoint(db); // Views LAST — they depend on metric_series / day_result / baselines / sessions // / notifications all existing. DROP+CREATE so a shape change takes effect. await _ensureCoachViews(db); await _dropRawStore(db); } + // Stable across opens, unique to this file's current input history. Neither + // restore nor salvage imports it: input_rev is local and can repeat on reset. + static Future _createStoreIdentity(DatabaseExecutor db) async { + await db.execute('CREATE TABLE IF NOT EXISTS store_identity (' + 'singleton INTEGER PRIMARY KEY CHECK(singleton = 1), id TEXT NOT NULL)'); + await db.execute('INSERT OR IGNORE INTO store_identity(singleton, id) ' + 'VALUES(1, lower(hex(randomblob(16))))'); + } + + static Future storeIdentity({DatabaseExecutor? executor}) async => + (await (executor ?? await instance).rawQuery( + 'SELECT id FROM store_identity WHERE singleton = 1')).single['id'] as String; + + static Future _rotateStoreIdentity(DatabaseExecutor db) async { + await db.execute('INSERT OR REPLACE INTO store_identity(singleton, id) ' + 'VALUES(1, lower(hex(randomblob(16))))'); + await db.delete('day_checkpoint'); + } + + static Future _createDayCheckpoint(Database db) async { + await db.execute('CREATE TABLE IF NOT EXISTS day_checkpoint (' + 'day_id TEXT NOT NULL, algo_version INTEGER NOT NULL, ' + 'fmt INTEGER NOT NULL, ctx_sig TEXT NOT NULL, cp_rec_ts INTEGER NOT NULL, ' + 'rev_vec BLOB NOT NULL, state BLOB NOT NULL, night_ref TEXT, ' + 'computed_at INTEGER NOT NULL, PRIMARY KEY(day_id, algo_version))'); + } + + /// Chunk the disposable blob like substrate_archive: a whole night can + /// exceed Android's cursor window. A racing replacement fails codec checks. + static Future?> dayCheckpoint( + String day, int algorithm, {required int maxBytes}) async { + final db = await instance; + const where = 'day_id = ? AND algo_version = ?'; + final rows = await db.rawQuery('SELECT day_id, algo_version, fmt, ctx_sig, ' + 'cp_rec_ts, rev_vec, night_ref, computed_at, length(state) AS state_length ' + 'FROM day_checkpoint WHERE $where', [day, algorithm]); + if (rows.isEmpty) return null; + final row = Map.of(rows.single); + final length = row.remove('state_length') as int; + final bytes = Uint8List(length <= maxBytes ? length : 0); + row['state'] = bytes; + if (length > maxBytes) return row; + for (var off = 0; off < length; off += 128 * 1024) { + final parts = await db.rawQuery( + 'SELECT substr(state, ?, ?) AS part FROM day_checkpoint WHERE $where', + [off + 1, 128 * 1024, day, algorithm]); + if (parts.isEmpty) return null; + final part = parts.single['part'] as Uint8List; + if (part.length > length - off) return null; + bytes.setRange(off, off + part.length, part); + } + return row; + } + /// One revision counter per absolute 15-minute input bucket, shared by the /// canonical 1 Hz rows and sparse beats. Triggers cover every write path and /// roll back with the input transaction. No raw rows are read at creation. @@ -11517,6 +11577,9 @@ class LocalDb { } await db.transaction((txn) async { + // Scratch prefixes can include an evening owned by another day label. + // Clear every copy atomically with deletion so deleted raw stays deleted. + await _rotateStoreIdentity(txn); // `raw_archive` frames are kept, but a later archive re-drive must not // decode them back into a day the user deleted: record the windows. final prevRow = await txn.query('compute_freshness', @@ -11821,6 +11884,7 @@ class LocalDb { final t = r['name']; if (t is String) deleted += await txn.delete(t); } + await _rotateStoreIdentity(txn); }); _markStoreReplaced(); return deleted; @@ -12277,6 +12341,7 @@ class LocalDb { /// frame already here, or in the backup twice, lands once. Paged on rowid, /// like every other table the merge reads. Returns rows offered. static Future _mergeRawArchive(Database src, Database db, { + required Future Function(Transaction) beforePageCommit, required void Function() onPageCommitted, }) async { var copied = 0; @@ -12297,26 +12362,29 @@ class LocalDb { ); if (page.isEmpty) return; after = (page.last['_r'] as num).toInt(); - final batch = db.batch(); - for (final r in page) { - final hex = r['hex']; - if (hex is! String) continue; - batch.insert( - rawArchiveTable, - rawArchiveRow( - deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, - hex: hex, - counter: (r['counter'] as num?)?.toInt(), - packetType: (r['packet_type'] as num?)?.toInt() ?? 0, - recTs: (r['rec_ts'] as num?)?.toInt(), - capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, - reason: r['reason'] as String? ?? 'unknown', - ), - conflictAlgorithm: ConflictAlgorithm.ignore, - ); - copied++; - } - await batch.commit(noResult: true); + await db.transaction((txn) async { + await beforePageCommit(txn); + final batch = txn.batch(); + for (final r in page) { + final hex = r['hex']; + if (hex is! String) continue; + batch.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, + hex: hex, + counter: (r['counter'] as num?)?.toInt(), + packetType: (r['packet_type'] as num?)?.toInt() ?? 0, + recTs: (r['rec_ts'] as num?)?.toInt(), + capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, + reason: r['reason'] as String? ?? 'unknown', + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); + copied++; + } + await batch.commit(noResult: true); + }); onPageCommitted(); } } @@ -12368,13 +12436,19 @@ class LocalDb { only: only, tolerant: tolerant, skip: skip, + beforePageCommit: (txn) async { + if (!pageCommitted) await _rotateStoreIdentity(txn); + }, onPageCommitted: () => pageCommitted = true, ); } finally { // A later page, source close or follow-up write can fail after a durable // replacement. Invalidate on every such exit, but never on a rolled-back // first page. Keep the flag local so concurrent merges cannot share it. - if (pageCommitted) _markStoreReplaced(); + if (pageCommitted) { + _markStoreReplaced(); + await (await instance).delete('day_checkpoint'); + } } } @@ -12382,6 +12456,7 @@ class LocalDb { String path, { List? only, bool tolerant = false, + required Future Function(Transaction) beforePageCommit, required void Function() onPageCommitted, Set skip = const {}, }) async { @@ -12790,6 +12865,7 @@ class LocalDb { for (final upper in ends) { final first = lower == _kBeforeEveryRowid; await db.transaction((txn) async { + await beforePageCommit(txn); if (t == 'day_result' && first) { // Inside the first chunk's transaction, so a day the engine // finalizes while the restore runs is still protected. @@ -12913,6 +12989,7 @@ class LocalDb { if (t == 'raw_archive') { try { counts[t] = await _mergeRawArchive(src, db, + beforePageCommit: beforePageCommit, onPageCommitted: onPageCommitted); } catch (_) { if (!tolerant) rethrow; @@ -13117,6 +13194,9 @@ class LocalDb { } } await db.transaction((txn) async { + // Rotate atomically with the first committed page. A killed + // process must not leave old scratch valid after partial restore. + await beforePageCommit(txn); if (t == SubstrateArchive.table) { final r = await SubstrateArchive.restoreRows( txn, diff --git a/lib/data/recent_slice.dart b/lib/data/recent_slice.dart index 2840705d4..5da2df053 100644 --- a/lib/data/recent_slice.dart +++ b/lib/data/recent_slice.dart @@ -84,6 +84,8 @@ const Map kSliceTables = { 'derive_fingerprint': (SliceKeep.all, null, 'which days changed since they were last scored'), 'input_rev': (SliceKeep.all, null, 'per-bucket input revisions'), 'session_ceiling_rev': (SliceKeep.all, null, 'session ceiling revisions'), + 'store_identity': (SliceKeep.none, null, 'local checkpoint store identity'), + 'day_checkpoint': (SliceKeep.none, null, 'disposable night/cardio checkpoints'), 'sleep_nap': (SliceKeep.all, null, 'naps'), 'sleep_override': (SliceKeep.all, null, 'sleep times the user edited'), // Workouts and activity: one row each. diff --git a/lib/data/restore_classification.dart b/lib/data/restore_classification.dart index ee17dee63..3d9c04f4b 100644 --- a/lib/data/restore_classification.dart +++ b/lib/data/restore_classification.dart @@ -39,6 +39,8 @@ const kRestoreClassification = { 'compute_freshness': RestoreClass.localOnly, 'derive_fingerprint': RestoreClass.rebuildable, 'input_rev': RestoreClass.rebuildable, + 'day_checkpoint': RestoreClass.rebuildable, + 'store_identity': RestoreClass.localOnly, 'session_ceiling_rev': RestoreClass.rebuildable, 'sleep_nap': RestoreClass.merge, 'sleep_override': RestoreClass.merge, diff --git a/test/data_ownership_test.dart b/test/data_ownership_test.dart index 576523a81..aca6e952c 100644 --- a/test/data_ownership_test.dart +++ b/test/data_ownership_test.dart @@ -4,7 +4,8 @@ // The reset test is the load-bearing one. The old reset walked the derived // days and deleted those, which left about twenty tables standing behind a // dialog that said otherwise — so the assertion here is deliberately -// "EVERY table in sqlite_master is empty", not a list of table names. A list +// "EVERY user-data table in sqlite_master is empty", not a list of table names. +// The store_identity row is rotated on wipe to invalidate old checkpoints. A list // would have to be updated when a table is added, which is exactly the failure // mode being tested for. @@ -60,7 +61,7 @@ void main() { }); group('wipeAll', () { - test('leaves EVERY table empty, including the ones a list would forget', + test('leaves EVERY user-data table empty and rotates local store identity', () async { await _seedEverything(); final db = await LocalDb.instance; @@ -83,11 +84,14 @@ void main() { expect(n, greaterThan(0), reason: '$t was never seeded'); } + final storeId = await LocalDb.storeIdentity(); await LocalDb.wipeAll(); + expect(await LocalDb.storeIdentity(), isNot(storeId)); for (final t in tables) { final n = _count(await db.rawQuery('SELECT COUNT(*) AS n FROM $t')); - expect(n, 0, reason: '$t survived "Delete everything"'); + expect(n, t == 'store_identity' ? 1 : 0, + reason: '$t survived "Delete everything"'); } }); diff --git a/test/ed2a_e18_session_reuse_test.dart b/test/ed2a_e18_session_reuse_test.dart index b6dfa2702..aa4b1502b 100644 --- a/test/ed2a_e18_session_reuse_test.dart +++ b/test/ed2a_e18_session_reuse_test.dart @@ -47,7 +47,7 @@ void main() { await db.setVersion(60); await LocalDb.close(); db = await LocalDb.instance; - expect(await db.getVersion(), 61); + expect(await db.getVersion(), LocalDb.schemaVersion); expect(await db.query('sessions'), sessions); expect(await db.query('session_ceiling_rev'), isEmpty, reason: 'upgrade never scans or backfills existing sessions'); @@ -75,9 +75,18 @@ void main() { expect(await db.query('session_ceiling_rev'), hasLength(1)); expect(await db.query('input_rev'), isNotEmpty); final generation = LocalDb.storeGeneration; + final identity = await LocalDb.storeIdentity(); await LocalDb.wipeAll(); + expect(await LocalDb.storeIdentity(), isNot(identity)); for (final table in await LocalDb.tableNames()) { - expect(await db.query(table, limit: 1), isEmpty, reason: table); + if (table == 'store_identity') { + final rows = await db.query(table); + expect(rows, hasLength(1)); + expect(rows.single['singleton'], 1); + expect(rows.single['id'], matches(RegExp(r'^[0-9a-f]{32}$'))); + } else { + expect(await db.query(table, limit: 1), isEmpty, reason: table); + } } expect(LocalDb.storeGeneration, greaterThan(generation)); await edSession(); diff --git a/test/ed2a_input_revision_test.dart b/test/ed2a_input_revision_test.dart index 78823fb04..00fc1db97 100644 --- a/test/ed2a_input_revision_test.dart +++ b/test/ed2a_input_revision_test.dart @@ -155,10 +155,10 @@ void main() { expect(LocalDb.restoreTablesForTest, isNot(contains('input_rev'))); }); - test('rung 61 is reserved for additive input revisions', () async { - expect(LocalDb.schemaVersion, 61, reason: 'upstream is schema 60'); + test('rung 61 input revisions remain present with rung 62 checkpoints', () async { + expect(LocalDb.schemaVersion, 62, reason: 'checkpoint storage follows revisions at 61'); final db = await open(); - expect(await db.getVersion(), 61); + expect(await db.getVersion(), LocalDb.schemaVersion); expect(await _hasRevisions(db), isTrue); final tables = await LocalDb.tableNames(); expect(tables, isNot(contains('store_rev'))); @@ -175,7 +175,7 @@ void main() { expect(await db.rawQuery('PRAGMA table_info($table)'), legacyColumns[table], reason: 'rung 61 must not change raw columns/defaults'); } - expect(await db.getVersion(), 61); + expect(await db.getVersion(), LocalDb.schemaVersion); expect(await _hasRevisions(db), isTrue); final triggers = await db.rawQuery( "SELECT tbl_name, sql FROM sqlite_master WHERE type='trigger'", diff --git a/test/ed2b_cardio_cache_test.dart b/test/ed2b_cardio_cache_test.dart index 1dfb0dead..2339b257a 100644 --- a/test/ed2b_cardio_cache_test.dart +++ b/test/ed2b_cardio_cache_test.dart @@ -245,6 +245,9 @@ void main() { offsetTs: _sec(1) + 1200, source: 'confirmed'); final e = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics() ..debugOnNightBundle = _checkNightOutput; + // Isolate the RAM generation contract: disk checkpoints legitimately + // survive reopen and would otherwise hide the required RAM rebuild. + if (change == 'generation') e.debugCheckpoints.disabled = true; expect(await e.runDays(edProfile, {_day}), 1); final before = e.debugNightStreams!.processedNightBeats; if (change == 'revision') { diff --git a/test/ed2b_checkpoints_test.dart b/test/ed2b_checkpoints_test.dart new file mode 100644 index 000000000..41bc68409 --- /dev/null +++ b/test/ed2b_checkpoints_test.dart @@ -0,0 +1,854 @@ +// E16 RED contracts. The standalone night/cardio payload uses format 2; +// fork day/minute/awake accumulator formats and schema numbers do not transfer. +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:io'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; +import 'package:path_provider_platform_interface/path_provider_platform_interface.dart'; +import 'package:openstrap_edge/compute/night_checkpoint.dart'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/recent_slice.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common/sqflite_logger.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +class _CheckpointPaths extends PathProviderPlatform { + _CheckpointPaths(this.root); + final String root; + @override + Future getTemporaryPath() async => root; +} + +Map _legacyRsa(Map row, {required bool bins}) { + final bytes = row['state'] as List; + final payload = jsonDecode(utf8.decode(bytes.sublist(32))) as Map; + final night = payload['night'] as Map; + final state = bins ? (night['rsa_bins'] as Map)['open'] as Map : night['rsa'] as Map; + state['version'] = 2; + final body = utf8.encode(jsonEncode(payload)); + return {'state': Uint8List.fromList([...sha256.convert(body).bytes, ...body])}; +} + +Future _hasCheckpoint(Database db) async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='day_checkpoint'")) + .isNotEmpty; + +Future _storage() async { + final db = await LocalDb.instance; + expect(await _hasCheckpoint(db), isTrue, + reason: 'E16 needs disposable day_checkpoint storage at rung 62'); + expect(await db.query('store_identity'), hasLength(1), + reason: 'rung 62 and repair must also create the durable identity row'); + return db; +} + +Map _sampleRow({String day = edDay, int? version}) => { + 'day_id': day, 'algo_version': version ?? kAlgoVersion, 'fmt': nightCheckpointFormat, + 'ctx_sig': 'night-cardio-context', 'cp_rec_ts': edStart ~/ 900 * 900, + 'rev_vec': Uint8List.fromList([0, 1, 2, 3, 4, 5, 6, 7]), + 'state': Uint8List.fromList([0, 255, 9, 10, 13, 128]), + 'night_ref': null, 'computed_at': 7, +}; + +Future>> _rows(Database db) => db.query( + 'day_checkpoint', where: 'day_id = ? AND algo_version = ?', + whereArgs: [edDay, kAlgoVersion]); + +// Supply a real sleep window and RR, not an absent-night fixture that could +// satisfy persistence by writing an empty payload. Two closed 15-minute buckets +// plus a tail exercise the resume boundary. +const _nightSeconds = 2100, _manualOffsetSeconds = 2000; +// One beat per second, including both manual-window endpoints. Clamp to the +// seeded RR extent; later 1 Hz-only appends add no beats. +int get _manualBeatCount => math.min(_nightSeconds, _manualOffsetSeconds + 1); + +Future _night() async { + await edRows(edStart, edStart + _nightSeconds, hr: 60, az: 1); + await LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + _manualOffsetSeconds, source: 'manual'); + final db = await LocalDb.instance; + final batch = db.batch(); + for (var i = 0; i < _nightSeconds; i++) { + final ts = edStart + i; + batch.insert('decoded_rr', {'device_id': '', 'ts_ms': ts * 1000, + 'rec_ts': ts, 'beat_index': 0, 'rr_ts_ms': ts * 1000, + 'rr_ms': 950.0 + 35 * math.sin(i * .13) + 13 * math.sin(i * .041)}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + await batch.commit(noResult: true); +} + +Future _derive(DerivationEngine engine) async { + expect(await engine.runDays(edProfile, {edDay}), 1, + reason: 'the checkpoint assertions require a real persisted derivation'); +} + +void main() { + final originalName = LocalDb.dbName; + final originalZone = Platform.environment['TZ'] ?? 'UTC'; + setUpAll(edInit); + setUp(() async { + edSetZone('UTC'); + await edFresh('ed2b_checkpoints.db'); + LocalDb.lastRebuild = null; + }); + tearDown(() async { + DerivationEngine.debugHoldTonight = {}; + await edDrop(); + LocalDb.dbName = originalName; + edSetZone(originalZone); + }); + + test('E16 fresh storage uses rung 62 and a versioned composite key', () async { + expect(LocalDb.schemaVersion, 62, + reason: '61 belongs to ED-2a revisions; checkpoints use the next rung'); + final db = await _storage(); + expect(await db.getVersion(), 62); + expect(await db.query('store_identity'), hasLength(1)); + expect(await LocalDb.storeIdentity(), matches(RegExp(r'^[0-9a-f]{32}$'))); + final columns = {for (final row in await db.rawQuery( + 'PRAGMA table_info(day_checkpoint)')) row['name']: row}; + expect(columns.keys.toSet(), {'day_id', 'algo_version', 'fmt', 'ctx_sig', + 'cp_rec_ts', 'rev_vec', 'state', 'night_ref', 'computed_at'}); + expect(columns['day_id']!['pk'], 1); + expect(columns['algo_version']!['pk'], 2); + for (final name in ['rev_vec', 'state']) { + expect(columns[name]!['type'], 'BLOB'); + expect(columns[name]!['notnull'], 1); + } + expect(await db.query('day_checkpoint'), isEmpty); + }); + + test('E16 61 to 62 preserves decoded rows and revisions without backfill', () async { + var db = await LocalDb.instance; + await edRows(edStart, edStart + 3); + await db.insert('decoded_rr', {'device_id': '', 'ts_ms': edStart * 1000, + 'rec_ts': edStart, 'rr_ts_ms': edStart * 1000, 'beat_index': 0, 'rr_ms': 900}); + final raw = await db.query('decoded_onehz'); + final rr = await db.query('decoded_rr'); + final revs = await db.query('input_rev', orderBy: 'bucket'); + final rawShape = await db.rawQuery('PRAGMA table_info(decoded_onehz)'); + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + await db.setVersion(61); + await LocalDb.close(); + db = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull, + reason: 'a migration must preserve the original DB, not quarantine it'); + expect(await db.query('decoded_onehz'), raw); + expect(await db.query('decoded_rr'), rr); + expect(await db.query('input_rev', orderBy: 'bucket'), revs); + expect(await db.rawQuery('PRAGMA table_info(decoded_onehz)'), rawShape); + await _storage(); + expect(await db.getVersion(), 62); + expect(await db.query('day_checkpoint'), isEmpty, + reason: 'migration creates empty scratch storage without deriving old days'); + }); + + test('E16 same-version repair creates storage and is idempotent', () async { + final db = await LocalDb.instance; + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + await LocalDb.repairOpenSchemaForTest(db); + await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final before = await db.query('day_checkpoint'); + final storeId = await LocalDb.storeIdentity(); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.close(); + final reopened = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull); + expect(await reopened.query('day_checkpoint'), before); + expect(await LocalDb.storeIdentity(), storeId); + }); + + test('E16 upgrade and open repair create scratch without scanning raw input', () async { + var db = await LocalDb.instance; + await edRows(edStart, edStart + 3); + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + await db.setVersion(61); + await LocalDb.close(); + final sql = []; + // Observe the actual sqflite calls, including batches, without changing + // migration code or providing a synthetic checkpoint implementation. + // ignore: experimental_member_use + databaseFactoryOrNull = SqfliteDatabaseFactoryLogger(databaseFactoryFfi, + options: SqfliteLoggerOptions( + type: SqfliteDatabaseFactoryLoggerType.invoke, + log: (event) { + if (event is! SqfliteLoggerInvokeEvent) return; + void collect(Object? value) { + if (value is Map) { + if (value['sql'] case final String statement) sql.add(statement); + for (final v in value.values) { collect(v); } + } else if (value is List) { + for (final v in value) { collect(v); } + } + } + collect(event.arguments); + }, + )); + try { + db = await LocalDb.instance; + await LocalDb.repairOpenSchemaForTest(db); + final openingSql = List.of(sql); + expect(openingSql, isNotEmpty, reason: 'SQL observation must be active'); + final rawAccess = RegExp( + r'\b(?:FROM|JOIN|UPDATE|INTO|DELETE\s+FROM)\s+["`\[]?' + r'(?:decoded_onehz|decoded_rr|samples|raw_archive)\b', + caseSensitive: false); + expect(openingSql.where((statement) { + final ddl = statement.trimLeft().toUpperCase(); + if (ddl.startsWith('CREATE TRIGGER') || ddl.startsWith('CREATE VIEW')) return false; + return rawAccess.hasMatch(statement); + }), isEmpty, reason: 'rung 62 and repair must perform no raw scan/backfill/rewrite'); + await _storage(); + expect(await db.query('day_checkpoint'), isEmpty); + } finally { + await LocalDb.close(); + databaseFactory = databaseFactoryFfi; + } + }); + + test('E16 actual same-version reopen repairs missing checkpoint storage', () async { + final db = await LocalDb.instance; + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + final version = await db.getVersion(); + await LocalDb.close(); + final reopened = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull); + expect(await reopened.getVersion(), version); + await _storage(); + expect(await reopened.query('day_checkpoint'), isEmpty); + }); + + test('E16 rerunning rung 62 keeps checkpoint blobs byte for byte', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + await db.insert('day_checkpoint', _sampleRow(version: kAlgoVersion - 1)); + await db.insert('day_checkpoint', _sampleRow(day: '2020-01-03')); + final before = await db.query('day_checkpoint', orderBy: 'day_id,algo_version'); + final storeId = await LocalDb.storeIdentity(); + await db.setVersion(61); + await LocalDb.close(); + final reopened = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull); + expect(await reopened.query('day_checkpoint', orderBy: 'day_id,algo_version'), before); + expect(await LocalDb.storeIdentity(), storeId); + expect(await _rows(reopened), hasLength(1)); + expect((await _rows(reopened)).single['state'], _sampleRow()['state']); + }); + + test('E16 standalone payload carries night and cardio only, format 2', () async { + await _night(); + final engine = DerivationEngine(); + await _derive(engine); + final diagnostics = engine.debugCheckpoints; + expect(diagnostics.writes, greaterThan(0), + reason: 'the worker must emit a checkpoint from real derived inputs'); + expect(diagnostics.lastEncodedPayload, isNotNull); + expect(diagnostics.lastEncodedPayload, isNotEmpty); + final payload = diagnostics.lastDecodedPayload; + expect(payload, isNotNull, + reason: 'observe the worker decode, without decoding on the UI isolate'); + expect(payload!['v'], nightCheckpointFormat, + reason: 'RSA v3 and closed-bin states require the new format'); + expect(payload['night'], isNotNull, reason: 'fixture supplies a settled RR night'); + expect(payload['cardio'], isNotNull); + // Inspect the actual worker-emitted inner checkpoint. In #99 capped + // correctors carry their consumed input count at capped.n, not top-level n. + final night = payload['night'] as Map; + final tag = night['tag'] as Map; + final corrector = (night['corrector'] as Map).cast(); + expect(corrector['version'], 2); + expect((night['rsa'] as Map)['version'], 3); + expect((night['rsa_bins'] as Map)['v'], 2); + expect(corrector['type'], 'RrCorrector'); + expect(corrector['maxArtifactRun'], 32); + expect((corrector['capped'] as Map)['n'], tag['count'], + reason: 'the serialized capped corrector must describe exactly the tagged night prefix'); + expect(tag['count'], greaterThan(0)); + expect(ana.RrCorrector.fromJson(corrector).toJson(), corrector, + reason: '#99 must accept the real inner checkpoint without changing its state'); + for (final key in ['day', 'minutes', 'bills', 'awake', 'activity', 'curves']) { + expect(payload, isNot(contains(key)), reason: '$key is outside ED-2b adoption'); + } + }); + + test('E16 process restart resumes and equals cold derive byte for byte', () async { + await _night(); + // Close the entire selected night before the checkpoint boundary, so a + // restarted worker has no tail beats left to fold. + await edRows(edStart + 2100, edStart + 2700, hr: 60, az: 1); + final first = DerivationEngine(); + await _derive(first); + final before = await edStored(edDay); + final storeId = await LocalDb.storeIdentity(); + await LocalDb.close(); + await LocalDb.instance; + expect(await LocalDb.storeIdentity(), storeId); + final resumed = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(resumed); + final resumedBytes = await edStored(edDay); + expect(resumedBytes, before); + final cold = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(cold); + expect(await edStored(edDay), resumedBytes); + expect(resumed.debugCheckpoints.hits, greaterThan(0), + reason: 'a fresh engine after reopen must adopt the persisted night/cardio state'); + expect(resumed.debugNightStreams!.processedNightBeats, 0); + expect(cold.debugCheckpoints.hits, 0); + }); + + test('E16 resumed append equals cold derive on the identical extended input', () async { + await _night(); + await _derive(DerivationEngine()); + await edRows(edStart + 2100, edStart + 2400, hr: 61, az: 1); + final resumed = DerivationEngine(); + await _derive(resumed); + final actual = await edStored(edDay); + final cold = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(cold); + expect(await edStored(edDay), actual); + expect(resumed.debugCheckpoints.hits, greaterThan(0), + reason: 'new open-bucket tail must not discard the valid closed prefix'); + }); + + final damaged = Function(Map)>{ + 'truncated blob': (_) => {'state': Uint8List.fromList([1, 0, 255])}, + 'checksum': (row) { + final bytes = Uint8List.fromList(row['state'] as List); + bytes[bytes.length ~/ 2] ^= 1; + return {'state': bytes}; + }, + 'old format': (_) => {'fmt': 1}, + 'future format': (_) => {'fmt': 999}, + 'legacy RSA v2': (row) => _legacyRsa(row, bins: false), + 'legacy RSA-bin v2': (row) => _legacyRsa(row, bins: true), + 'old algorithm': (_) => {'algo_version': kAlgoVersion - 1}, + 'foreign context': (_) => {'ctx_sig': 'foreign-owner-profile-and-window'}, + 'misaligned boundary': (row) => {'cp_rec_ts': (row['cp_rec_ts'] as int) + 1}, + 'corrupt revision vector': (_) => {'rev_vec': Uint8List.fromList([1])}, + }; + for (final entry in damaged.entries) { + test('E16 ${entry.key} is refused whole and recomputed exactly', () async { + await _night(); + await _derive(DerivationEngine()); + final db = await _storage(); + final rows = await _rows(db); + expect(rows, hasLength(1), reason: 'real derive must persist a resume point'); + await db.update('day_checkpoint', entry.value(rows.single), + where: 'day_id = ? AND algo_version = ?', whereArgs: [edDay, kAlgoVersion]); + final engine = DerivationEngine(); + await _derive(engine); + final rejectedOutput = await edStored(edDay); + final cold = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(cold); + expect(await edStored(edDay), rejectedOutput); + expect(engine.debugCheckpoints.hits, 0, + reason: 'never salvage a night or cardio subset of an invalid checkpoint'); + if (entry.key == 'old algorithm') { + expect(engine.debugCheckpoints.misses, greaterThan(0), + reason: 'version-keyed lookup must miss the old algorithm row'); + } else { + expect(engine.debugCheckpoints.rejections, isNotEmpty, + reason: 'rejection must be observable after the real reader attempts it'); + } + expect(engine.debugCheckpoints.writes, greaterThan(0), reason: 'cold recompute replaces bad state'); + }); + } + + test('E16 changed closed input revision refuses an otherwise valid checkpoint', () async { + await _night(); + await _derive(DerivationEngine()); + await edRows(edStart, edStart + 1, hr: 60, az: 1); + final engine = DerivationEngine(); + await _derive(engine); + final revised = await edStored(edDay); + await _derive(DerivationEngine()..debugCheckpoints.disabled = true); + expect(await edStored(edDay), revised); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty, + reason: 'identical newest-wins replacement changes input_rev, so prefix must rebuild'); + }); + + for (final sameEngine in [true, false]) { + test('E16 reopen reuses checkpoints with unchanged revisions, ' + '${sameEngine ? 'retained' : 'fresh'} engine', () async { + await _night(); + await edRows(edStart + 2100, edStart + 2700, hr: 60, az: 1); + final storeId = await LocalDb.storeIdentity(); + final first = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(first); + final before = await LocalDb.inputDayFingerprint(edDay); + final output = await edStored(edDay); + final beats = first.debugNightStreams!.retainedNightBeats; + final work = first.debugNightStreams!.processedNightBeats; + final streamWork = {for (final key in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) + key: first.debugNightStreams!.processedBeats(key)}; + expect(beats, _manualBeatCount, + reason: 'one seeded RR beat per second, including the manual-window end'); + final saved = await _rows(await _storage()); + expect(saved, hasLength(1)); + await LocalDb.close(); + final db = await LocalDb.instance; + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(after.generation, greaterThan(before.generation)); + expect(await _rows(db), saved, + reason: 'ordinary reopen preserves the durable checkpoint'); + expect(await LocalDb.storeIdentity(), storeId); + final engine = sameEngine ? first : + (DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics()); + engine.debugCheckpoints.reset(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 1, + reason: 'the same store id permits disk adoption after RAM generation changes'); + expect(engine.debugCheckpoints.rejections, isEmpty); + expect(engine.debugNightStreams!.processedNightBeats, + sameEngine ? work : 0, + reason: 'resume adds no night work and preserves cumulative RAM counters'); + for (final entry in streamWork.entries) { + expect(engine.debugNightStreams!.processedBeats(entry.key), sameEngine ? entry.value : 0, + reason: '${entry.key} counters survive same-engine disk adoption'); + } + expect(await edStored(edDay), output); + final replacement = await _rows(db); + expect(replacement, hasLength(1)); + expect(replacement.single['ctx_sig'], saved.single['ctx_sig']); + final rebuilt = engine.debugNightStreams!.processedNightBeats; + await _derive(engine); + expect(engine.debugNightStreams!.processedNightBeats, rebuilt, + reason: 'the replacement RAM owner must be retained'); + expect(await edStored(edDay), output); + final resumed = DerivationEngine(); + await _derive(resumed); + expect(resumed.debugCheckpoints.hits, 1, + reason: 'the checkpoint remains reusable in the same store'); + expect(await edStored(edDay), output); + }); + } + + test('E16 committed import rejects stale disk state with unchanged raw revisions', () async { + await _night(); + final engine = DerivationEngine(); + await _derive(engine); + final before = await LocalDb.inputDayFingerprint(edDay); + final storeId = await LocalDb.storeIdentity(); + final saved = (await _rows(await _storage())).single; + final dir = await Directory.systemTemp.createTemp('ed2b_cp_generation_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE journal (date TEXT PRIMARY KEY, ' + "note TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL)"); + await source.insert('journal', {'date': edDay, 'note': 'generation-only', 'updated_at': 1}); + await source.close(); + await LocalDb.importFromDbFile(path); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(after.generation, greaterThan(before.generation), + reason: 'committed store replacement differs from a legitimate checkpoint restart'); + expect(await LocalDb.storeIdentity(), isNot(storeId)); + final db = await _storage(); + expect(await _rows(db), isEmpty); + // Defeat scratch clearing deliberately: identity must reject stale bytes + // even when the revision vector and raw inputs are unchanged. + await db.insert('day_checkpoint', saved); + await LocalDb.close(); + await LocalDb.instance; + engine.debugCheckpoints.reset(); + await _derive(engine); + final replaced = await edStored(edDay); + await _derive(DerivationEngine()..debugCheckpoints.disabled = true); + expect(await edStored(edDay), replaced); + expect(engine.debugCheckpoints.hits, 0, + reason: 'the original engine must not revive its pre-restore resume state'); + expect(engine.debugCheckpoints.misses, greaterThan(0), + reason: 'the new store identity must force a cold night/cardio lookup'); + expect(engine.debugCheckpoints.rejections, isNotEmpty); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + + test('E16 wipe and re-import reject stale checkpoint when revisions repeat', () async { + await _night(); + await _derive(DerivationEngine()); + final db = await _storage(); + final saved = (await _rows(db)).single; + final before = await LocalDb.inputDayFingerprint(edDay); + final storeId = await LocalDb.storeIdentity(); + final dir = await Directory.systemTemp.createTemp('ed2b_cp_wipe_'); + final path = p.join(dir.path, 'source.db'); + try { + await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'); + await File(db.path).copy(path); + await LocalDb.wipeAll(); + final wipedId = await LocalDb.storeIdentity(); + expect(wipedId, isNot(storeId)); + await LocalDb.importFromDbFile(path); + expect(await LocalDb.storeIdentity(), isNot(wipedId)); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum, + reason: 'input_rev is local, so reset and identical restore can repeat it'); + expect(after.revisionMax, before.revisionMax); + expect(await _rows(db), isEmpty); + await db.insert('day_checkpoint', saved); + await LocalDb.close(); + await LocalDb.instance; + final engine = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty); + expect(engine.debugNightStreams!.processedNightBeats, _manualBeatCount); + final actual = await edStored(edDay); + await _derive(DerivationEngine()..debugCheckpoints.disabled = true); + expect(await edStored(edDay), actual); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + + test('E16 replaced DB file rejects old checkpoint with identical inputs', () async { + await _night(); + await _derive(DerivationEngine()); + final original = await _storage(); + final saved = (await _rows(original)).single; + final before = await LocalDb.inputDayFingerprint(edDay); + final storeId = await LocalDb.storeIdentity(); + final liveName = LocalDb.dbName; + final livePath = original.path; + final replacementPath = p.join(p.dirname(livePath), 'ed2b_cp_replacement.db'); + await LocalDb.close(); + try { + await edFresh('ed2b_cp_replacement.db'); + await _night(); + final replacement = await _storage(); + final replacementId = await LocalDb.storeIdentity(); + expect(replacementId, isNot(storeId)); + await replacement.insert('day_checkpoint', saved); + await replacement.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'); + await LocalDb.close(); + await databaseFactory.deleteDatabase(livePath); + await File(replacementPath).copy(livePath); + LocalDb.dbName = liveName; + await LocalDb.instance; + expect(await LocalDb.storeIdentity(), replacementId); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(await _rows(await _storage()), [saved]); + final engine = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty, + reason: 'an intact old blob must not certify a different database file'); + expect(engine.debugNightStreams!.processedNightBeats, _manualBeatCount); + } finally { + await LocalDb.close(); + LocalDb.dbName = liveName; + await databaseFactory.deleteDatabase(replacementPath); + } + }); + + test('E16 rebuild with tolerant salvage rotates identity and rejects old state', () async { + await _night(); + await _derive(DerivationEngine()); + final db = await _storage(); + final saved = (await _rows(db)).single; + final storeId = await LocalDb.storeIdentity(); + final before = await LocalDb.inputDayFingerprint(edDay); + // Force the real ladder-rebuild path. The rename collision rolls back the + // failed open; all input tables remain available to tolerant salvage. + await db.execute('CREATE TABLE _raw_old (hex TEXT)'); + await db.execute('DROP TABLE baselines'); + await db.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await db.insert('baselines', {'key': 'unreadable-salvage-shape'}); + await db.setVersion(2); + await LocalDb.close(); + try { + final rebuilt = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNotNull); + final rebuild = LocalDb.lastRebuild!; + expect(rebuild.salvaged['baselines'], 0, + reason: 'tolerant salvage must continue past a failed table'); + expect(rebuild.salvaged['decoded_onehz'], _nightSeconds); + expect(rebuild.salvaged['decoded_rr'], _nightSeconds); + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect(await _rows(rebuilt), isEmpty); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + await rebuilt.insert('day_checkpoint', saved); + final engine = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty); + expect(engine.debugNightStreams!.processedNightBeats, _manualBeatCount); + } finally { + final rebuild = LocalDb.lastRebuild; + if (rebuild != null) await databaseFactory.deleteDatabase(rebuild.quarantinePath); + } + }); + + test('E16 wipe clears checkpoints with every other table', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final storeId = await LocalDb.storeIdentity(); + await LocalDb.wipeAll(); + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect(await db.query('day_checkpoint'), isEmpty); + expect(await _hasCheckpoint(db), isTrue, reason: 'wipe clears data and preserves schema'); + }); + + for (final fail in [false, true]) { + test('E16 ${fail ? 'partially failing' : 'successful'} restore clears local scratch', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final storeId = await LocalDb.storeIdentity(); + await db.execute("CREATE TRIGGER require_rotated_identity BEFORE INSERT ON journal " + "BEGIN SELECT CASE WHEN (SELECT id FROM store_identity WHERE singleton = 1) " + "= '$storeId' OR EXISTS(SELECT 1 FROM day_checkpoint) " + "THEN RAISE(ABORT, 'old checkpoint identity inside restore page') END; END"); + final dir = await Directory.systemTemp.createTemp('ed2b_cp_restore_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE journal (date TEXT PRIMARY KEY, ' + "note TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL)"); + await source.insert('journal', {'date': edDay, 'note': 'restored', 'updated_at': 1}); + // Even a source with matching schema/version must not import its + // checkpoint into a different store's revision history. + final ddl = (await db.rawQuery("SELECT sql FROM sqlite_master " + "WHERE type='table' AND name='day_checkpoint'")).single['sql'] as String; + await source.execute(ddl); + await source.insert('day_checkpoint', { + ..._sampleRow(), 'ctx_sig': 'foreign-store-checkpoint', + }); + if (fail) { + await source.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await source.insert('baselines', {'key': 'invalid-import'}); + } + await source.close(); + if (fail) { + await expectLater(LocalDb.importFromDbFile(path), throwsA(isA())); + } else { + await LocalDb.importFromDbFile(path); + } + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect((await db.query('journal')).single['note'], 'restored', + reason: 'at least one replacement page committed'); + expect(await db.query('day_checkpoint'), isEmpty, + reason: 'a committed restore invalidates local checkpoint inputs even if a later page fails'); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + } + + for (final fail in [false, true]) { + test('E16 ${fail ? 'rolled-back first page' : 'empty import'} preserves identity and scratch', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final saved = await _rows(db); + final storeId = await LocalDb.storeIdentity(); + final generation = LocalDb.storeGeneration; + final dir = await Directory.systemTemp.createTemp('ed2b_cp_no_commit_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + if (fail) await source.insert('baselines', {'key': 'invalid-first-page'}); + await source.close(); + if (fail) { + await expectLater(LocalDb.importFromDbFile(path), throwsA(isA())); + } else { + await LocalDb.importFromDbFile(path); + } + expect(await LocalDb.storeIdentity(), storeId); + expect(LocalDb.storeGeneration, generation); + expect(await _rows(db), saved); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + } + + test('E16 issue-report slices explicitly classify disposable checkpoints', () { + expect(kSliceTables, contains('day_checkpoint'), + reason: 'new storage must be classified, not silently use fallback'); + expect(kSliceTables['day_checkpoint']!.$1, SliceKeep.none); + expect(kSliceTables['store_identity']!.$1, SliceKeep.none); + }); + + test('GUARD restore allow-list excludes disposable checkpoints', () { + expect(LocalDb.restoreTablesForTest, isNot(contains('day_checkpoint'))); + expect(LocalDb.restoreTablesForTest, isNot(contains('store_identity'))); + expect(LocalDb.salvageTablesForTest, isNot(contains('store_identity'))); + }); + + test('GUARD disabled checkpoint observations are inert and output-neutral', () async { + await _night(); + final normal = DerivationEngine(); + await _derive(normal); + final expected = await edStored(edDay); + final disabled = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(disabled); + expect(await edStored(edDay), expected); + expect(disabled.debugCheckpoints.hits, 0); + expect(disabled.debugCheckpoints.writes, 0); + disabled.debugCheckpoints.reset(); + expect(disabled.debugCheckpoints.disabled, isTrue); + expect(disabled.debugCheckpoints.lastEncodedPayload, isNull); + expect(jsonEncode(disabled.debugCheckpoints.lastDecodedPayload), 'null'); + }); + for (final mode in ['success', 'later table failure', 'later archive-page failure', 'first failure']) { + test('E16 archive-only $mode rotates identity atomically with committed pages', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final before = await _rows(db); + final storeId = await LocalDb.storeIdentity(); + await db.execute('CREATE TRIGGER require_archive_identity BEFORE INSERT ON ' + '${LocalDb.rawArchiveTable} BEGIN SELECT CASE WHEN ' + "(SELECT id FROM store_identity WHERE singleton = 1) = '$storeId' " + "OR EXISTS(SELECT 1 FROM day_checkpoint) THEN RAISE(ABORT, 'old archive identity') END; END"); + if (mode == 'first failure' || mode == 'later archive-page failure') { + final refused = mode == 'first failure' ? 1 : 501; + await db.execute('CREATE TRIGGER refuse_archive_page BEFORE INSERT ON ' + '${LocalDb.rawArchiveTable} WHEN NEW.counter = $refused ' + "BEGIN SELECT RAISE(ABORT, 'archive page failure'); END"); + } + final dir = await Directory.systemTemp.createTemp('ed2b_archive_identity_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE raw_archive (hex TEXT PRIMARY KEY, ' + 'counter INTEGER, packet_type INTEGER, rec_ts INTEGER, captured_at INTEGER, reason TEXT)'); + final batch = source.batch(); + final count = mode == 'later archive-page failure' ? 501 : 1; + for (var i = 1; i <= count; i++) { + batch.insert('raw_archive', {'hex': i.toRadixString(16).padLeft(4, '0'), + 'counter': i, 'packet_type': 0, 'rec_ts': edStart, + 'captured_at': edStart * 1000, 'reason': 'identity_test'}); + } + await batch.commit(noResult: true); + if (mode == 'later table failure') { + await source.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await source.insert('baselines', {'key': 'missing-payload'}); + } + await source.close(); + if (mode == 'success') { + await LocalDb.importFromDbFile(path); + } else { + await expectLater(LocalDb.importFromDbFile(path), throwsA(isA())); + } + final copied = mode == 'first failure' ? 0 : + mode == 'later archive-page failure' ? 500 : 1; + expect(await db.query(LocalDb.rawArchiveTable), hasLength(copied)); + if (mode == 'first failure') { + expect(await LocalDb.storeIdentity(), storeId); + expect(await _rows(db), before); + } else { + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect(await db.query('day_checkpoint'), isEmpty); + } + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + } + + test('E16 deleting a day removes every disposable raw prefix and keeps unrelated input', () async { + await _night(); + final db = await _storage(); + final other = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(other, other + 2); + await db.insert('day_checkpoint', _sampleRow()); + await db.insert('day_checkpoint', _sampleRow(day: '2020-01-03')); + final identity = await LocalDb.storeIdentity(); + const learning = '{"nights":7,"folded_days":["2020-01-01"]}'; + await LocalDb.putBaseline('sleep_user_profile', learning); + await LocalDb.deleteDays({edDay}); + expect(await db.query('day_checkpoint'), isEmpty, + reason: 'a neighbouring checkpoint can contain the deleted evening too'); + expect(await LocalDb.storeIdentity(), isNot(identity)); + expect(await db.query('decoded_rr'), isEmpty); + final remaining = await db.query('decoded_onehz'); + expect(remaining.map((row) => row['rec_ts']), [other, other + 1]); + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], learning, + reason: 'learned state cannot be rebuilt once old input is pruned'); + }); + + test('E16 selected-day export omits scratch and uses a fresh identity', () async { + final db = await _storage(); + await edRows(edStart, edStart + 2); + final other = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(other, other + 2); + await db.insert('day_checkpoint', _sampleRow()); + final identity = await LocalDb.storeIdentity(); + final dir = await Directory.systemTemp.createTemp('ed2b_selected_export_'); + final provider = PathProviderPlatform.instance; + PathProviderPlatform.instance = _CheckpointPaths(dir.path); + try { + final path = await LocalDb.exportDaysDb({edDay}); + final out = await databaseFactory.openDatabase(path, options: OpenDatabaseOptions(readOnly: true)); + try { + expect(await _hasCheckpoint(out), isTrue); + expect(await out.query('day_checkpoint'), isEmpty); + expect((await out.query('store_identity')).single['id'], isNot(identity)); + expect((await out.query('decoded_onehz')).map((r) => r['rec_ts']), [edStart, edStart + 1]); + expect(await LocalDb.storeIdentity(), identity); + expect(await _rows(db), hasLength(1)); + } finally { + await out.close(); + } + } finally { + PathProviderPlatform.instance = provider; + await dir.delete(recursive: true); + } + }); + + test('E16 checkpoint and RAM reuse cannot finalize or fingerprint a held night', () async { + await _night(); + final later = DateTime(2020, 1, 5, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(later, later + 1); + DerivationEngine.debugHoldTonight = {edDay}; + await _derive(DerivationEngine()); + expect(await _rows(await _storage()), hasLength(1)); + final engine = DerivationEngine(); + for (var i = 0; i < 2; i++) { + await engine.run(edProfile, heavy: true); + expect(await LocalDb.finalizedDayIds(kAlgoVersion), isNot(contains(edDay))); + expect(await LocalDb.deriveFingerprint(edDay), isNull); + } + expect(engine.debugCheckpoints.hits, greaterThan(0), + reason: 'the hold must survive actual disk adoption'); + DerivationEngine.debugHoldTonight = {}; + await engine.run(edProfile, heavy: true); + expect(await LocalDb.finalizedDayIds(kAlgoVersion), contains(edDay)); + }); + +} diff --git a/test/ed2b_streams_test.dart b/test/ed2b_streams_test.dart index adb511f05..4de8c4931 100644 --- a/test/ed2b_streams_test.dart +++ b/test/ed2b_streams_test.dart @@ -334,7 +334,7 @@ void main() { if (n == 0) expect(d.processedNightBeats, 0); } }); - test('E10 closed RSA bins reapply the night artifact gate after append', () { + test('E10 closed RSA bins reapply the night artifact gate after append and JSON restart', () { final input = _input(beats: 6200); final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); final bins = NightRespBins(); @@ -348,6 +348,10 @@ void main() { expect(bins.snapshot(const [], const [], artifactFraction: 0), batch(0)); expect(batch(.5), isEmpty); expect(bins.snapshot(const [], const [], artifactFraction: .5), batch(.5)); + final saved = (jsonDecode(jsonEncode(bins.toJson())) as Map).cast(); + final restored = NightRespBins.fromJson(saved, c.nn.length); + expect(restored.snapshot(const [], const [], artifactFraction: .5), batch(.5)); + expect(restored.snapshot(const [], const [], artifactFraction: 0), batch(0)); }); } diff --git a/test/import_bulk_restore_test.dart b/test/import_bulk_restore_test.dart index 0204f908f..c40b65b96 100644 --- a/test/import_bulk_restore_test.dart +++ b/test/import_bulk_restore_test.dart @@ -62,11 +62,14 @@ int _callsTouching(String table) { const _t0 = 1790000000; // 2026-09-21, an arbitrary fixed second /// Stamped with DateTime.now(): differs between any two -/// runs, so the equivalence dump leaves it out. +/// runs, so the equivalence dump leaves it out. store_identity.id is a fresh +/// random id on every committed restore by design (it keys night +/// checkpoints); the singleton row itself is still compared. const _wallClockColumns = { 'compute_freshness': {'updated_at'}, 'activity_review_meta': {'activated_at', 'alert_after_ts'}, 'substrate_archive': {'updated_at'}, + 'store_identity': {'id'}, }; Map archiveSecond(int ts, int hr) => { From d765a2518be768d70a63d0f99cb1835a5f4c6464 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sun, 15 Nov 2026 15:43:01 -0700 Subject: [PATCH 07/11] deps: retain workout rest in incremental analytics pin --- lib/compute/derivation_engine.dart | 9 ++++++--- pubspec.lock | 4 ++-- pubspec.yaml | 4 ++-- test/ed2a_substrate_transfer_test.dart | 2 +- test/ed2b_algo_version_114_test.dart | 15 ++++++++++++--- test/fixtures/whoop_freeze/gen4_minus0400.json | 18 +++++++++--------- test/fixtures/whoop_freeze/gen4_plus0530.json | 18 +++++++++--------- test/fixtures/whoop_freeze/gen4_utc.json | 18 +++++++++--------- test/fixtures/whoop_freeze/gen5_minus0400.json | 18 +++++++++--------- test/fixtures/whoop_freeze/gen5_plus0530.json | 18 +++++++++--------- test/fixtures/whoop_freeze/gen5_utc.json | 18 +++++++++--------- 11 files changed, 77 insertions(+), 65 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 8032e74de..96724d7fc 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -1879,7 +1879,10 @@ import 'vendor_sleep.dart'; // excluding non-owner rows at the last second. D36 folds only the target night's // own observation, none when absent; forward-only, preserving existing profiles // and folded_days. The restricted E15 walk now also applies to folding passes. -const int kAlgoVersion = 122; +// 122 → 123 (analytics #99 + #98 @ 801c825): restore the observed 5-15 min +// workout rest bridge, including the duration-cap and saved-session overlap +// guards, while retaining #99's incremental, accumulator and checkpoint APIs. +const int kAlgoVersion = 123; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. /// @@ -2067,8 +2070,8 @@ const int kAlgoVersion = 122; // centre), for v112. // REPIN @ 3a7ca34: analytics main, #101 + #100 + #97 + #98, for v119. // REPIN @ f8e7d3c: rebased analytics #99, for v122 above. -// This head forks at 9db83ce and omits #98's workout rest bridge at 3a7ca34. -const String kAnalyticsPin = 'f8e7d3c478cc59707fde41f3edff7736026573fe'; +// REPIN @ 801c825: merge #98's workout rest bridge into #99, for v123. +const String kAnalyticsPin = '801c82536caf5ce70270aae5215f21cd423a8bd2'; // Repinned to analytics main's tip, which carries BOTH PR #72 (hrv_freq // Welch gap guard) and PR #73 (overreachingConjunction rhr quantum guard) — // the two independent kAlgoVersion bumps above (93 and 94). Verified both diff --git a/pubspec.lock b/pubspec.lock index 33ecd3c50..20857891f 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -937,8 +937,8 @@ packages: dependency: "direct main" description: path: "." - ref: f8e7d3c478cc59707fde41f3edff7736026573fe - resolved-ref: f8e7d3c478cc59707fde41f3edff7736026573fe + ref: "801c82536caf5ce70270aae5215f21cd423a8bd2" + resolved-ref: "801c82536caf5ce70270aae5215f21cd423a8bd2" url: "https://github.com/OpenStrap/analytics.git" source: git version: "1.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index 34fbc145a..ddb2cd798 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -523,8 +523,8 @@ dependencies: # (sleep staging and naps), #97 (stride-periodicity step counter) and #98 # (a mid-session rest stays inside one workout). # Rebased analytics #99: incremental kernels, RSA v3 and cosinor gate (v122). - # This head forks at 9db83ce and omits #98's workout rest bridge at 3a7ca34. - ref: f8e7d3c478cc59707fde41f3edff7736026573fe + # Merge 3a7ca34 into #99: retain the workout rest bridge (#98), for v123. + ref: 801c82536caf5ce70270aae5215f21cd423a8bd2 # BLE — flutter_blue_plus is the maintained cross-platform GATT client. flutter_blue_plus: ^1.36.8 diff --git a/test/ed2a_substrate_transfer_test.dart b/test/ed2a_substrate_transfer_test.dart index 0511227bc..c27b9e769 100644 --- a/test/ed2a_substrate_transfer_test.dart +++ b/test/ed2a_substrate_transfer_test.dart @@ -241,7 +241,7 @@ void main() { markTestSkipped('committed golden is UTC; run this test with TZ=UTC'); return; } - expect(kAlgoVersion, 122, reason: 'ED-2b repins analytics; transfer remains exact'); + expect(kAlgoVersion, 123, reason: 'ED-2b repins analytics; transfer remains exact'); final fixture = _fixture(); final baseline = substrateFromDecodedPage(fixture.frames, fixture.rr); final transferred = Substrate.fromTransfer(baseline.toJson()); diff --git a/test/ed2b_algo_version_114_test.dart b/test/ed2b_algo_version_114_test.dart index 30555cd37..4f92f547e 100644 --- a/test/ed2b_algo_version_114_test.dart +++ b/test/ed2b_algo_version_114_test.dart @@ -4,9 +4,10 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:openstrap_edge/compute/derivation_engine.dart'; void main() { - test('ED-2b takes one version bump and names the output changes', () { - expect(kAlgoVersion, 122); - expect(kAnalyticsPin, 'f8e7d3c478cc59707fde41f3edff7736026573fe'); + // WHY: finalized days need a new version when the analytics pin changes outputs. + test('ED-2b adoption and workout rest repin name their output changes', () { + expect(kAlgoVersion, 123); + expect(kAnalyticsPin, '801c82536caf5ce70270aae5215f21cd423a8bd2'); final src = File('lib/compute/derivation_engine.dart').readAsStringSync(); final at = src.indexOf('// 121 → 122'); expect(at, isNonNegative); @@ -17,5 +18,13 @@ void main() { 'forward-only', 'staging labels unchanged']) { expect(entry, contains(change)); } + final repin = src.indexOf('// 122 → 123'); + expect(repin, isNonNegative); + final repinEntry = src.substring( + repin, src.indexOf('const int kAlgoVersion', repin)); + for (final change in ['801c825', '#98', 'workout rest bridge', + 'duration-cap', 'saved-session overlap']) { + expect(repinEntry, contains(change)); + } }); } diff --git a/test/fixtures/whoop_freeze/gen4_minus0400.json b/test/fixtures/whoop_freeze/gen4_minus0400.json index f850fa4a6..c9369f003 100644 --- a/test/fixtures/whoop_freeze/gen4_minus0400.json +++ b/test/fixtures/whoop_freeze/gen4_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -24902,7 +24902,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49595,7 +49595,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen4_plus0530.json b/test/fixtures/whoop_freeze/gen4_plus0530.json index 2fe1d50a5..00da35798 100644 --- a/test/fixtures/whoop_freeze/gen4_plus0530.json +++ b/test/fixtures/whoop_freeze/gen4_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -24902,7 +24902,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49595,7 +49595,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen4_utc.json b/test/fixtures/whoop_freeze/gen4_utc.json index 4dd0e0b8a..9edb698b6 100644 --- a/test/fixtures/whoop_freeze/gen4_utc.json +++ b/test/fixtures/whoop_freeze/gen4_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -24902,7 +24902,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49595,7 +49595,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen5_minus0400.json b/test/fixtures/whoop_freeze/gen5_minus0400.json index 71bdb89f5..6b3f85ddf 100644 --- a/test/fixtures/whoop_freeze/gen5_minus0400.json +++ b/test/fixtures/whoop_freeze/gen5_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -25080,7 +25080,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49946,7 +49946,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen5_plus0530.json b/test/fixtures/whoop_freeze/gen5_plus0530.json index f42f13095..aea4380e8 100644 --- a/test/fixtures/whoop_freeze/gen5_plus0530.json +++ b/test/fixtures/whoop_freeze/gen5_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -25080,7 +25080,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49946,7 +49946,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen5_utc.json b/test/fixtures/whoop_freeze/gen5_utc.json index 215bdc3e0..88ecd59f1 100644 --- a/test/fixtures/whoop_freeze/gen5_utc.json +++ b/test/fixtures/whoop_freeze/gen5_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -25080,7 +25080,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49946,7 +49946,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 122, + "algo_version": 123, "payload_json": { "day_id": "2026-07-15", "active_min": 40, From 600d182ce0685da0438f363e13c056a6792243cc Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sun, 15 Nov 2026 16:43:31 -0700 Subject: [PATCH 08/11] deps: adopt merged analytics main for incremental derivation Pin analytics to 614a48e5ffc553b88de248fd9fc13b07d9c1ceba and advance the algorithm to 124. Preserve merged clock-refusal semantics in retained night derivation and skip checkpoints with regressing corrected clocks. Cover capped-reset derivation and legacy checkpoint codecs. Regenerate WHOOP freeze files through the documented process; only algorithm stamps change. --- lib/compute/derivation_engine.dart | 13 ++- lib/compute/night_prsa.dart | 2 +- lib/compute/night_rr_state.dart | 6 +- lib/compute/onehz_pipeline.dart | 10 +- pubspec.lock | 4 +- pubspec.yaml | 7 +- test/ed2a_substrate_transfer_test.dart | 2 +- test/ed2b_algo_version_114_test.dart | 13 ++- test/ed2b_merged_analytics_test.dart | 110 ++++++++++++++++++ .../fixtures/whoop_freeze/gen4_minus0400.json | 18 +-- test/fixtures/whoop_freeze/gen4_plus0530.json | 18 +-- test/fixtures/whoop_freeze/gen4_utc.json | 18 +-- .../fixtures/whoop_freeze/gen5_minus0400.json | 18 +-- test/fixtures/whoop_freeze/gen5_plus0530.json | 18 +-- test/fixtures/whoop_freeze/gen5_utc.json | 18 +-- 15 files changed, 204 insertions(+), 71 deletions(-) create mode 100644 test/ed2b_merged_analytics_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 96724d7fc..7e7cc6353 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -1882,7 +1882,15 @@ import 'vendor_sleep.dart'; // 122 → 123 (analytics #99 + #98 @ 801c825): restore the observed 5-15 min // workout rest bridge, including the duration-cap and saved-session overlap // guards, while retaining #99's incremental, accumulator and checkpoint APIs. -const int kAlgoVersion = 123; +// 123 → 124 (analytics main #102-#108 @ 614a48e): retain #98's workout +// rest bridge and adopt the merged clock policies: HRV/RSA/CVHR abstain on +// regressing clocks; frequency HRV omits re-anchored overlap up to 5%. +// Cosinor refuses unidentifiable phase columns and non-finite designs; +// irregular screening abstains on degenerate SDNN; missing stage scores widen +// uncertainty. The #102 batch output changes and D35/D36 remain as at v122. +// HrvFreqStream v2, CardioRowCache v3 and policy-bound PRSA states require cold +// rebuilds of old checkpoints, isolated by this algorithm version. +const int kAlgoVersion = 124; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. /// @@ -2071,7 +2079,8 @@ const int kAlgoVersion = 123; // REPIN @ 3a7ca34: analytics main, #101 + #100 + #97 + #98, for v119. // REPIN @ f8e7d3c: rebased analytics #99, for v122 above. // REPIN @ 801c825: merge #98's workout rest bridge into #99, for v123. -const String kAnalyticsPin = '801c82536caf5ce70270aae5215f21cd423a8bd2'; +// REPIN @ 614a48e: upstream main, merged #102-#109 and #98, for v124. +const String kAnalyticsPin = '614a48e5ffc553b88de248fd9fc13b07d9c1ceba'; // Repinned to analytics main's tip, which carries BOTH PR #72 (hrv_freq // Welch gap guard) and PR #73 (overreachingConjunction rhr quantum guard) — // the two independent kAlgoVersion bumps above (93 and 94). Verified both diff --git a/lib/compute/night_prsa.dart b/lib/compute/night_prsa.dart index 14f9e2502..beb64e9b2 100644 --- a/lib/compute/night_prsa.dart +++ b/lib/compute/night_prsa.dart @@ -41,7 +41,7 @@ class NightPrsa { return agg.finish(agg.addSlice(copy, tail, 0, tail.length)); } // Sequential addSlice preserves anchor order; never combine partial sums. - // #99 prsa_aggregator.dart declares profile/capacity abs+rel 1e-9, + // Merged analytics prsa_aggregator.dart declares profile/capacity abs+rel 1e-9, // confidence abs 1e-12 parity with the batch functions. return (finish(_dc, _dcState), finish(_ac, _acState)); } diff --git a/lib/compute/night_rr_state.dart b/lib/compute/night_rr_state.dart index 10e4d682c..eec65de08 100644 --- a/lib/compute/night_rr_state.dart +++ b/lib/compute/night_rr_state.dart @@ -100,6 +100,10 @@ class NightRrState { /// a live tail requires a separate fold; it must never rewind the live owner. Map? checkpoint(int boundary) { if (_corrector == null || _night == null) return null; + // The edge checkpoint requires ordered NN. A reset that regresses cannot + // restore its RSA/CVHR state; leave this disposable checkpoint cold. + // ↳ CLAIM(merged-clock-refusal) + if (ana.hasRegressingBeatTimes([..._nnTimes, ..._tailTimes])) return null; var n = 0; while (n < _times.length && _times[n] < boundary * 1000) { n++; } if (n == 0) return null; @@ -337,7 +341,7 @@ class NightRrState { ana.Metric nightCvhr(double artifactFraction) { final stream = _cvhr ??= ana.CvhrStream(); _feed('cvhr', (nn, times) { stream.fold(nn, times); return 1; }); - // #99 handles a fresh stream's entirely provisional night too. + // The merged stream handles a fresh, entirely provisional night too. return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, artifactFraction: artifactFraction); } diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 54a946d4f..66fb4cf13 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -621,6 +621,10 @@ Map _deriveDayBundle(Map inputJson, final nnTimes = corrected.nnTimesMs; final artifactFraction = (1.0 - corrected.cleanFraction).clamp(0.0, 1.0); final streams = nightState?.canStream == true ? nightState : null; + // CLAIM(merged-clock-refusal): capped resets preserve source time and can + // regress. Batch RSA/CVHR abstain; their streams require ordered clocks. + // Keep frequency's separate omission policy [ed2b_merged_analytics_test]. + final orderedStreams = hasRegressingBeatTimes(nnTimes) ? null : streams; final nnProof = nightState?.dependencyProof(nn, nnTimes); T nightMemo(String key, Object? extra, T Function() compute) => nnProof == null ? compute() : @@ -759,7 +763,7 @@ Map _deriveDayBundle(Map inputJson, // ── RESPIRATION (sleep-windowed) ─────────────────────────────────────────── final resp = nn.length >= 30 ? nightMemo('rsa', artifactFraction, () { - if (streams != null) return streams.nightResp(artifactFraction); + if (orderedStreams != null) return orderedStreams.nightResp(artifactFraction); streamDiagnostics?.observeBatch('rsa', nn.length); return rsaRespRate(nn, nnTimes, artifactFraction: artifactFraction); }) @@ -770,7 +774,7 @@ Map _deriveDayBundle(Map inputJson, ); final cvhr = nn.length >= 60 ? nightMemo('cvhr', artifactFraction, () { - if (streams != null) return streams.nightCvhr(artifactFraction); + if (orderedStreams != null) return orderedStreams.nightCvhr(artifactFraction); streamDiagnostics?.observeBatch('cvhr', nn.length); return cvhrApneaScreen(nn, nnTimes, artifactFraction: artifactFraction); }) @@ -799,7 +803,7 @@ Map _deriveDayBundle(Map inputJson, // Window the cleaned sleep NN into ~30-min bins, take each bin's RSA resp rate, // then BRV = dispersion + Theil-Sen trend of those per-window rates. final cachedRespWindows = nightMemo('rsa_bins', artifactFraction, () => - streams != null ? streams.nightRespWindows(artifactFraction) : + orderedStreams != null ? orderedStreams.nightRespWindows(artifactFraction) : respPerWindow(nn, nnTimes, artifactFraction: artifactFraction, diagnostics: streamDiagnostics)); final respWindows = nightState == null diff --git a/pubspec.lock b/pubspec.lock index 20857891f..a31620311 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -937,8 +937,8 @@ packages: dependency: "direct main" description: path: "." - ref: "801c82536caf5ce70270aae5215f21cd423a8bd2" - resolved-ref: "801c82536caf5ce70270aae5215f21cd423a8bd2" + ref: "614a48e5ffc553b88de248fd9fc13b07d9c1ceba" + resolved-ref: "614a48e5ffc553b88de248fd9fc13b07d9c1ceba" url: "https://github.com/OpenStrap/analytics.git" source: git version: "1.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index ddb2cd798..aad5e60c5 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -522,9 +522,10 @@ dependencies: # analytics main @ 3a7ca34: adds #101 (HRV/readiness/wellness fixes), #100 # (sleep staging and naps), #97 (stride-periodicity step counter) and #98 # (a mid-session rest stays inside one workout). - # Rebased analytics #99: incremental kernels, RSA v3 and cosinor gate (v122). - # Merge 3a7ca34 into #99: retain the workout rest bridge (#98), for v123. - ref: 801c82536caf5ce70270aae5215f21cd423a8bd2 + # analytics main @ 614a48e: merged #102-#108 incremental APIs and review + # fixes, retaining #98's workout rest bridge. Clock refusal, uncertainty + # and cosinor policies require kAlgoVersion 124. + ref: 614a48e5ffc553b88de248fd9fc13b07d9c1ceba # BLE — flutter_blue_plus is the maintained cross-platform GATT client. flutter_blue_plus: ^1.36.8 diff --git a/test/ed2a_substrate_transfer_test.dart b/test/ed2a_substrate_transfer_test.dart index c27b9e769..2c727d87e 100644 --- a/test/ed2a_substrate_transfer_test.dart +++ b/test/ed2a_substrate_transfer_test.dart @@ -241,7 +241,7 @@ void main() { markTestSkipped('committed golden is UTC; run this test with TZ=UTC'); return; } - expect(kAlgoVersion, 123, reason: 'ED-2b repins analytics; transfer remains exact'); + expect(kAlgoVersion, 124, reason: 'ED-2b repins analytics; transfer remains exact'); final fixture = _fixture(); final baseline = substrateFromDecodedPage(fixture.frames, fixture.rr); final transferred = Substrate.fromTransfer(baseline.toJson()); diff --git a/test/ed2b_algo_version_114_test.dart b/test/ed2b_algo_version_114_test.dart index 4f92f547e..7d7bea4d6 100644 --- a/test/ed2b_algo_version_114_test.dart +++ b/test/ed2b_algo_version_114_test.dart @@ -6,8 +6,8 @@ import 'package:openstrap_edge/compute/derivation_engine.dart'; void main() { // WHY: finalized days need a new version when the analytics pin changes outputs. test('ED-2b adoption and workout rest repin name their output changes', () { - expect(kAlgoVersion, 123); - expect(kAnalyticsPin, '801c82536caf5ce70270aae5215f21cd423a8bd2'); + expect(kAlgoVersion, 124); + expect(kAnalyticsPin, '614a48e5ffc553b88de248fd9fc13b07d9c1ceba'); final src = File('lib/compute/derivation_engine.dart').readAsStringSync(); final at = src.indexOf('// 121 → 122'); expect(at, isNonNegative); @@ -20,11 +20,16 @@ void main() { } final repin = src.indexOf('// 122 → 123'); expect(repin, isNonNegative); - final repinEntry = src.substring( - repin, src.indexOf('const int kAlgoVersion', repin)); + final repinEntry = src.substring(repin, src.indexOf('// 123 → 124', repin)); for (final change in ['801c825', '#98', 'workout rest bridge', 'duration-cap', 'saved-session overlap']) { expect(repinEntry, contains(change)); } + final mainRepin = src.substring(src.indexOf('// 123 → 124')); + for (final change in ['614a48e', '#102-#108', 'regressing clocks', + '5%', 'Cosinor', 'SDNN', 'uncertainty', 'HrvFreqStream v2', + 'CardioRowCache v3', 'PRSA']) { + expect(mainRepin, contains(change)); + } }); } diff --git a/test/ed2b_merged_analytics_test.dart b/test/ed2b_merged_analytics_test.dart new file mode 100644 index 000000000..6889a66c2 --- /dev/null +++ b/test/ed2b_merged_analytics_test.dart @@ -0,0 +1,110 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/cardio_rows_cache.dart'; +import 'package:openstrap_edge/compute/night_rr_state.dart'; + +void main() { + // WHY: a capped reset can put corrected NN behind its previous segment even + // with ordered raw timestamps. Merged batch RSA/CVHR abstain; their streams + // throw on that clock. One absent metric must not abort the whole day. + // Proves CLAIM(merged-clock-refusal). + // Catches: feeding a regressing corrected clock into retained RSA/CVHR. + test('merged clock refusals preserve a complete day and retained frequency', () { + const start = 1577930400; + double wobble(int i) => 1500.0 + (i % 7) * 12 - (i % 3) * 9; + // Analytics test/output_changes/review_inputs_test.dart's capped-reset + // fixture: 400 good beats, 33 artifacts, 400 good beats at 1.4 s cadence. + final rr = [for (var i = 0; i < 400; i++) wobble(i), + ...List.filled(33, 3000), + for (var i = 0; i < 400; i++) wobble(i)]; + final times = [for (var i = 0; i < rr.length; i++) start * 1000.0 + i * 1400]; + final diagnostics = Ed2bStreamDiagnostics(); + List? windows; + diagnostics.onRespWindows = (rates) => windows = rates; + for (final count in [400, 650, 833]) { + final input = DayBundleInput(date: '2020-01-02', dayTsSec: const [], + dayHr: const [], sleepTsSec: const [], sleepHr: const [], + sleepRrMs: rr.sublist(0, count), sleepRrTsMs: times.sublist(0, count), + sleepSkinTemp: const [], sleepJson: const {}, hypnoStages: const [], + sleepOnsetSec: start, sleepOffsetSec: start + 1200, + profile: const {}, deviceFamily: 'gen4').toJson(); + final cold = deriveDayBundle(input); + final retained = deriveDayBundle(input, streamDiagnostics: diagnostics); + final respiration = retained['respiration'] as Map; + expect(respiration['rsa'], (cold['respiration'] as Map)['rsa']); + expect(respiration['cvhr_apnea'], (cold['respiration'] as Map)['cvhr_apnea']); + expect((retained['clinical'] as Map)['hrv_freq'], + (cold['clinical'] as Map)['hrv_freq']); + if (count == 833) { + for (final key in ['rsa', 'cvhr_apnea']) { + expect((respiration[key] as Map)['value'], '—'); + expect((respiration[key] as Map)['note'], 'regressing beat time'); + } + final frequency = (retained['clinical'] as Map)['hrv_freq'] as Map; + expect(frequency['value'], isA()); + expect(frequency['note'], contains('left out')); + final c = ana.correctRr(rr, rrTsMs: times); + expect(windows, respPerWindow(c.nn, c.nnTimesMs, + artifactFraction: 1 - c.cleanFraction)); + // Cutting this prefix would otherwise try to fold the earlier valid + // RSA stream under the now-invalid clock. + expect(diagnostics.nightState.checkpoint(start + 900), isNull); + } + } + expect(diagnostics.processedNightBeats, rr.length); + expect(diagnostics.streamRestarts('frequency'), 1); + expect(diagnostics.retainedStreamBeats('frequency'), greaterThan(0)); + // Check that the fixture still exercises a real corrected-clock regression. + final corrected = ana.correctRr(rr, rrTsMs: times); + expect([for (var i = 1; i < corrected.nnTimesMs.length; i++) + corrected.nnTimesMs[i] < corrected.nnTimesMs[i - 1]], contains(true)); + }); + // WHY: analytics main changed codec policy. Old reusable work must cost a + // cold calculation, never load a result derived under the old pin. + for (final codec in ['frequency v1', 'PRSA without policy']) { + test('merged night checkpoint refuses $codec', () { + final state = NightRrState(); + final rr = List.filled(500, 950); + final times = [for (var i = 0; i < rr.length; i++) 1000000.0 + i * 950]; + state.correct(rr, times, day: '2020-01-02', onsetSec: 1000); + state.nightHrvFreq(0); + state.nightPrsa(); + final checkpoint = jsonDecode(jsonEncode(state.checkpoint(2000))) as Map; + expect(NightRrState.fromCheckpoint( + checkpoint.cast(), 2000).retainedNightBeats, 500); + if (codec == 'frequency v1') { + final frequency = checkpoint['frequency'] as Map; + frequency['version'] = 1; + frequency.remove('dropped'); + } else { + // PRSA's lookback inner now binds direction and anchor-ratio cap. + final dc = (checkpoint['prsa'] as Map)['dc'] as List; + final policy = (dc[1] as Map)['inner'] as Map; + policy.remove('deceleration'); + policy.remove('anchorRatioCap'); + } + expect(() => NightRrState.fromCheckpoint( + checkpoint.cast(), 2000), throwsFormatException); + }); + } + + // WHY: old word-wise fingerprints could alias after sign-bit changes. + // The merged codec invalidates those rows so staging rebuilds them. + test('merged cardio codec treats a v2 chunk as a cold miss', () { + final cache = ana.CardioRowCache()..insert( + const ana.CardioRowKey(1000, 30, 42), + const ana.CardioEpochRow(0, 60, 1, 20, 1, .5, 30)); + final json = cache.toJson(); + final current = Uint8List.fromList(utf8.encode(jsonEncode(json))); + expect(decodeCardioRows([current]).length, 1); + json['version'] = 2; + expect(() => ana.CardioRowCache.fromJson(json), throwsFormatException); + final bytes = Uint8List.fromList(utf8.encode(jsonEncode(json))); + expect(decodeCardioRows([bytes]).length, 0); + }); +} diff --git a/test/fixtures/whoop_freeze/gen4_minus0400.json b/test/fixtures/whoop_freeze/gen4_minus0400.json index c9369f003..8ba1db57b 100644 --- a/test/fixtures/whoop_freeze/gen4_minus0400.json +++ b/test/fixtures/whoop_freeze/gen4_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -24902,7 +24902,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49595,7 +49595,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen4_plus0530.json b/test/fixtures/whoop_freeze/gen4_plus0530.json index 00da35798..3355c1d46 100644 --- a/test/fixtures/whoop_freeze/gen4_plus0530.json +++ b/test/fixtures/whoop_freeze/gen4_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -24902,7 +24902,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49595,7 +49595,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen4_utc.json b/test/fixtures/whoop_freeze/gen4_utc.json index 9edb698b6..1073f8a70 100644 --- a/test/fixtures/whoop_freeze/gen4_utc.json +++ b/test/fixtures/whoop_freeze/gen4_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -24902,7 +24902,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49595,7 +49595,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen5_minus0400.json b/test/fixtures/whoop_freeze/gen5_minus0400.json index 6b3f85ddf..1fffab38b 100644 --- a/test/fixtures/whoop_freeze/gen5_minus0400.json +++ b/test/fixtures/whoop_freeze/gen5_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -25080,7 +25080,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49946,7 +49946,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen5_plus0530.json b/test/fixtures/whoop_freeze/gen5_plus0530.json index aea4380e8..18e58f2b8 100644 --- a/test/fixtures/whoop_freeze/gen5_plus0530.json +++ b/test/fixtures/whoop_freeze/gen5_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -25080,7 +25080,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49946,7 +49946,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, diff --git a/test/fixtures/whoop_freeze/gen5_utc.json b/test/fixtures/whoop_freeze/gen5_utc.json index 88ecd59f1..6bff9a346 100644 --- a/test/fixtures/whoop_freeze/gen5_utc.json +++ b/test/fixtures/whoop_freeze/gen5_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -25080,7 +25080,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -49946,7 +49946,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 123, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, From 172d278e3872d8a51bb177a212046d3dc8688e08 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:10:30 +0530 Subject: [PATCH 09/11] derive: read the ceiling provenance before the union load, clear a stale RAM night state so the disk checkpoint can restore it --- lib/compute/derivation_engine.dart | 61 ++++++++++++++++++++++-------- lib/compute/night_rr_state.dart | 7 ++++ 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 44723ac74..c9c6995db 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3344,6 +3344,23 @@ class DerivationEngine { } } + /// The revision a night's RAM state is keyed on: every touched day's input + /// revision summed over the sleep window, and the store generation. + Future<(int, int)> _nightInputRevision(int sleepOnsetSec, int sleepOffsetSec) async { + var revision = 0; + var generation = LocalDb.storeGeneration; + final first = DateTime.fromMillisecondsSinceEpoch(sleepOnsetSec * 1000); + final last = sleepOffsetSec > sleepOnsetSec ? sleepOffsetSec - 1 : sleepOnsetSec; + for (var date = DateTime(first.year, first.month, first.day); + date.millisecondsSinceEpoch ~/ 1000 <= last; + date = DateTime(date.year, date.month, date.day + 1)) { + final fp = await LocalDb.inputDayFingerprint(dayLabelOf(date)); + revision += fp.revisionSum; + generation = fp.generation; + } + return (revision, generation); + } + Future _sessionCeilingRevision() async { try { return await LocalDb.sessionCeilingRevision(); @@ -4400,6 +4417,11 @@ class DerivationEngine { // merged into the shared running max exactly once, below. final stats = _PrepareStats(); final union = debugDisableUnionLoads ? null : _PrepareUnion(); + // Provenance BEFORE any load: a decoded write that lands while the + // substrate is read then leaves the older revision attached to the newer + // rows, which the signature check rejects. Read after, it would attach the + // newer revision to rows that predate it and certify a stale ceiling. + final ceilingInput = await _ceilingInputRevision(dayId); final candidate = await _sleepCandidateForDay(dayId, stats: stats, union: union); final dayStart = _localDayLabelToSec(dayId); final dayEnd = _localNextDayLabelToSec(dayId); @@ -4429,7 +4451,6 @@ class DerivationEngine { ? (Map>.of(union.ownership), Map>.of(union.priority)) : await _resolveOwnership(unionFrom, unionTo); - final ceilingInput = await _ceilingInputRevision(dayId); if (union != null && (union.load == null || unionFrom < union.load!.from! || unionTo > union.load!.to!)) { // A banked candidate needs no search load. A live search already in @@ -4923,7 +4944,27 @@ class DerivationEngine { final rowBytes = [for (final c in sentChunks) c.bytes]; if (checkpointPlan != null && checkpointPlan.boundary > range.$1 && checkpointPlan.context != null) { - if (_nightStates[dayId]?.canStream != true && + // A RAM state clears itself when the input revision moved (see + // NightRrState.context), so one that no longer matches is not a live + // start: clear it here and let the compatible disk checkpoint restore + // the night, instead of skipping it and reprocessing the whole night. + final ram = _nightStates[dayId]; + var ramLive = ram?.canStream == true; + if (ram != null && ramLive) { + try { + final w = ram.window; + if (w == null) { + ramLive = false; + } else { + final (rev, gen) = await _nightInputRevision(w.$1, w.$2); + ramLive = ram.hasContext(rev, gen, kAlgoVersion); + } + } catch (_) { + ramLive = false; + } + if (!ramLive) ram.clear(); + } + if (!ramLive && _nightStates[dayId]?.acceptsCheckpointAlgorithm(kAlgoVersion) != false) { try { debugCheckpoints.reads++; @@ -6856,21 +6897,11 @@ class DerivationEngine { if ((retainNight && sleepSub.rrMs.isNotEmpty) || diagnostics != null) { nightState = _nightStates.remove(day.date) ?? NightRrState(); try { - var revision = 0; - var generation = LocalDb.storeGeneration; - final first = DateTime.fromMillisecondsSinceEpoch( - day.sleepOnsetSec * 1000); - final last = day.sleepOffsetSec > day.sleepOnsetSec - ? day.sleepOffsetSec - 1 : day.sleepOnsetSec; - for (var date = DateTime(first.year, first.month, first.day); - date.millisecondsSinceEpoch ~/ 1000 <= last; - date = DateTime(date.year, date.month, date.day + 1)) { - final fp = await LocalDb.inputDayFingerprint(dayLabelOf(date)); - revision += fp.revisionSum; - generation = fp.generation; - } + final (revision, generation) = + await _nightInputRevision(day.sleepOnsetSec, day.sleepOffsetSec); nightState.context(inputRevision: revision, generation: generation, algorithm: kAlgoVersion); + nightState.window = (day.sleepOnsetSec, day.sleepOffsetSec); nightRevision = revision; nightGeneration = generation; nightProvenance = true; diff --git a/lib/compute/night_rr_state.dart b/lib/compute/night_rr_state.dart index eec65de08..a7ec97aee 100644 --- a/lib/compute/night_rr_state.dart +++ b/lib/compute/night_rr_state.dart @@ -24,6 +24,10 @@ class NightRrState { ana.RrCorrector? _corrector; final List _rr = [], _times = [], _nn = [], _nnTimes = []; (int, int, int)? _context; + /// The sleep window (onset, offset, epoch seconds) this state was last keyed + /// on; set by the engine after [context]. Lets a later pass re-derive the + /// revision [context] would be given and ask [hasContext] before trusting RAM. + (int, int)? window; (String, int)? _night; final _memos = {}; final _digests = _PassListDigests(); @@ -221,6 +225,9 @@ class NightRrState { void beginPass() => _digests.clear(); void endPass() => _digests.clear(); + bool hasContext(int inputRevision, int generation, int algorithm) => + _context == (inputRevision, generation, algorithm); + void context({required int inputRevision, required int generation, required int algorithm}) { final next = (inputRevision, generation, algorithm); From 6c445b7bde0dfaab1f2ee4aa3cf1c43e9eaf93c7 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:01:12 +0530 Subject: [PATCH 10/11] derive: fold the observation of the chosen night, matched on its staged window (analytics #110) --- lib/compute/derivation_engine.dart | 44 +++++++++++++++++++++++++---- pubspec.lock | 4 +-- pubspec.yaml | 7 +++-- test/observation_of_night_test.dart | 34 ++++++++++++++++++++++ 4 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 test/observation_of_night_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index c9c6995db..d8af739ec 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -1882,12 +1882,15 @@ import 'vendor_sleep.dart'; // 122 → 123 (analytics #99 + #98 @ 801c825): restore the observed 5-15 min // workout rest bridge, including the duration-cap and saved-session overlap // guards, while retaining #99's incremental, accumulator and checkpoint APIs. -// 123 → 124 (analytics main #102-#108 @ 614a48e): retain #98's workout +// 123 → 124 (analytics main #102-#110 @ a438df4): retain #98's workout // rest bridge and adopt the merged clock policies: HRV/RSA/CVHR abstain on // regressing clocks; frequency HRV omits re-anchored overlap up to 5%. // Cosinor refuses unidentifiable phase columns and non-finite designs; // irregular screening abstains on degenerate SDNN; missing stage scores widen -// uncertainty. The #102 batch output changes and D35/D36 remain as at v122. +// uncertainty. The #102 batch output changes and D35/D36 remain as at v122; +// #110 tags each night observation with its staged window, so D36 folds the +// chosen night's own observation (the one whose window overlaps it), none when +// no window does, instead of the longest of the search window's sessions. // HrvFreqStream v2, CardioRowCache v3 and policy-bound PRSA states require cold // rebuilds of old checkpoints, isolated by this algorithm version. const int kAlgoVersion = 124; @@ -2080,7 +2083,10 @@ const int kAlgoVersion = 124; // REPIN @ f8e7d3c: rebased analytics #99, for v122 above. // REPIN @ 801c825: merge #98's workout rest bridge into #99, for v123. // REPIN @ 614a48e: upstream main, merged #102-#109 and #98, for v124. -const String kAnalyticsPin = '614a48e5ffc553b88de248fd9fc13b07d9c1ceba'; +// REPIN @ a438df4: analytics main, #110 (each night observation carries its +// staged window), still v124: that version had not shipped, so no stored day +// was derived at it with the previous pin. +const String kAnalyticsPin = 'a438df40b38a4586a52a68ab7e1c100fca111ccd'; // Repinned to analytics main's tip, which carries BOTH PR #72 (hrv_freq // Welch gap guard) and PR #73 (overreachingConjunction rhr quantum guard) — // the two independent kAlgoVersion bumps above (93 and 94). Verified both @@ -3149,6 +3155,29 @@ class CompactResult { /// The experimental HMM sleep staging switch (Settings), default off. const kSleepHmmStagesPref = 'sleep_hmm_stages'; +/// The observation of the night [onsetSec, offsetSec): the recorded one whose +/// staged window overlaps it most, or null when none does. An observation with +/// no window (an older analytics) never matches, and the night folds nothing +/// rather than a session of another day. +@visibleForTesting +ana.SleepNightObservation? observationOfNight( + List obs, int onsetSec, int offsetSec) { + ana.SleepNightObservation? best; + var bestOverlap = 0; + for (final o in obs) { + final s = o.startTsSec, e = o.endTsSec; + if (s == null || e == null) continue; + final int lo = s > onsetSec ? s : onsetSec; + final int hi = e < offsetSec ? e : offsetSec; + final int overlap = hi - lo; + if (overlap > bestOverlap) { + best = o; + bestOverlap = overlap; + } + } + return best; +} + class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; @@ -5058,9 +5087,12 @@ class DerivationEngine { // 0.0%. One fold per day_id, ever. if (mayFold && candidate.present) { final obs = ana.takeCardioObservations(); - if (obs.isNotEmpty) { - obs.sort((a, b) => b.epochs.compareTo(a.epochs)); - final main = obs.first; + // THIS night's own observation, matched on the window it was staged + // over. The search window also stages the day before's sessions; the + // longest of all of them is not necessarily the night chosen. + final main = observationOfNight( + obs, candidate.sleepOnsetSec, candidate.sleepOffsetSec); + if (main != null) { if (main.epochs >= 120) { // require ≥60 min — not a nap. // Return the raw OBSERVATION, not a folded profile. Folding here diff --git a/pubspec.lock b/pubspec.lock index a31620311..42c301a62 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -937,8 +937,8 @@ packages: dependency: "direct main" description: path: "." - ref: "614a48e5ffc553b88de248fd9fc13b07d9c1ceba" - resolved-ref: "614a48e5ffc553b88de248fd9fc13b07d9c1ceba" + ref: a438df40b38a4586a52a68ab7e1c100fca111ccd + resolved-ref: a438df40b38a4586a52a68ab7e1c100fca111ccd url: "https://github.com/OpenStrap/analytics.git" source: git version: "1.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index 0ac7bdb85..1cb573201 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -522,10 +522,11 @@ dependencies: # analytics main @ 3a7ca34: adds #101 (HRV/readiness/wellness fixes), #100 # (sleep staging and naps), #97 (stride-periodicity step counter) and #98 # (a mid-session rest stays inside one workout). - # analytics main @ 614a48e: merged #102-#108 incremental APIs and review - # fixes, retaining #98's workout rest bridge. Clock refusal, uncertainty + # analytics main @ a438df4: merged #102-#108 incremental APIs and review + # fixes, retaining #98's workout rest bridge, plus #110 (each recorded night + # observation carries the window it was staged over). Clock refusal, uncertainty # and cosinor policies require kAlgoVersion 124. - ref: 614a48e5ffc553b88de248fd9fc13b07d9c1ceba + ref: a438df40b38a4586a52a68ab7e1c100fca111ccd # BLE — flutter_blue_plus is the maintained cross-platform GATT client. flutter_blue_plus: ^1.36.8 diff --git a/test/observation_of_night_test.dart b/test/observation_of_night_test.dart new file mode 100644 index 000000000..998fbdbf5 --- /dev/null +++ b/test/observation_of_night_test.dart @@ -0,0 +1,34 @@ +// Which recorded observation belongs to the night a derive chose. The search +// window also stages the day before's sessions; the longest observation of all +// is not necessarily the chosen night's. +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; + +ana.SleepNightObservation _o(int epochs, int? start, int? end) => + ana.SleepNightObservation(epochs: epochs, startTsSec: start, endTsSec: end); + +void main() { + test('takes the observation staged over the night, not the longest one', () { + final dayBefore = _o(900, 1000, 28000); // longer, but another day + final night = _o(500, 40000, 58000); + expect(observationOfNight([dayBefore, night], 40500, 57500), same(night)); + expect(observationOfNight([night, dayBefore], 40500, 57500), same(night)); + }); + + test('of two overlapping windows, the one overlapping the night most wins', () { + final partial = _o(300, 30000, 41000); + final whole = _o(500, 40000, 58000); + expect(observationOfNight([partial, whole], 40500, 57500), same(whole)); + }); + + test('no observation overlapping the night folds nothing', () { + expect(observationOfNight([_o(900, 1000, 28000)], 40500, 57500), isNull); + expect(observationOfNight(const [], 40500, 57500), isNull); + }); + + test('an observation with no window never matches', () { + expect(observationOfNight([_o(900, null, null)], 40500, 57500), isNull); + expect(observationOfNight([_o(900, 40000, null)], 40500, 57500), isNull); + }); +} From 622355e4d85d8df63e63b77d11516472cdb5e6a2 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:22:43 +0530 Subject: [PATCH 11/11] test: the algo 124 guard follows the analytics repin --- test/ed2b_algo_version_114_test.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ed2b_algo_version_114_test.dart b/test/ed2b_algo_version_114_test.dart index 7d7bea4d6..b3de3bd82 100644 --- a/test/ed2b_algo_version_114_test.dart +++ b/test/ed2b_algo_version_114_test.dart @@ -7,7 +7,7 @@ void main() { // WHY: finalized days need a new version when the analytics pin changes outputs. test('ED-2b adoption and workout rest repin name their output changes', () { expect(kAlgoVersion, 124); - expect(kAnalyticsPin, '614a48e5ffc553b88de248fd9fc13b07d9c1ceba'); + expect(kAnalyticsPin, 'a438df40b38a4586a52a68ab7e1c100fca111ccd'); final src = File('lib/compute/derivation_engine.dart').readAsStringSync(); final at = src.indexOf('// 121 → 122'); expect(at, isNonNegative); @@ -26,7 +26,7 @@ void main() { expect(repinEntry, contains(change)); } final mainRepin = src.substring(src.indexOf('// 123 → 124')); - for (final change in ['614a48e', '#102-#108', 'regressing clocks', + for (final change in ['a438df4', '#102-#110', '#110', 'regressing clocks', '5%', 'Cosinor', 'SDNN', 'uncertainty', 'HrvFreqStream v2', 'CardioRowCache v3', 'PRSA']) { expect(mainRepin, contains(change));