diff --git a/lib/compute/cardio_rows_cache.dart b/lib/compute/cardio_rows_cache.dart new file mode 100644 index 000000000..db39b7844 --- /dev/null +++ b/lib/compute/cardio_rows_cache.dart @@ -0,0 +1,46 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/onehz.dart' as ana; + +/// Caller-owned bytes and provenance. Row objects exist only in the worker. +class CardioRowsChunk { + CardioRowsChunk(this.bytes, this.loSec, this.hiSec, this.revisions, + this.generation, this.algorithm); + final Uint8List bytes; + final int loSec, hiSec; + final Map revisions; + int generation, algorithm; + + bool overlaps(int lo, int hi) => hiSec >= lo && loSec <= hi; + + bool matches(Map current, int gen, int algo) => + generation == gen && algorithm == algo && + current.entries.every((e) => + !revisions.containsKey(e.key) || revisions[e.key] == e.value); +} + +/// Worker only. Damage or an unknown codec version costs a cold staging. +ana.CardioRowCache decodeCardioRows(List chunks) { + final rows = ana.CardioRowCache(); + for (final bytes in chunks) { + try { + rows.merge(ana.CardioRowCache.fromJson( + (jsonDecode(utf8.decode(bytes)) as Map).cast())); + } on FormatException { + // Refuse the whole chunk. + } on TypeError { + // A structurally invalid JSON value is also a miss. + } + } + return rows; +} + +/// Keep at most one search window plus slack and 8,000 row variants. A limit +/// only evicts reusable work; analytics always recomputes a missing row. +Uint8List encodeCardioRows(ana.CardioRowCache rows, int lo, int hi) { + var kept = 0; + rows.retainWhere((k) => + k.startSec >= lo && k.startSec <= hi && kept++ < 8000); + return Uint8List.fromList(utf8.encode(jsonEncode(rows.toJson()))); +} diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 34c91b8f2..d8af739ec 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -36,6 +36,7 @@ import 'readiness_calibration_backfill.dart' show backfillReadinessCalibration, seedRecentReadinessZ; import 'package:crypto/crypto.dart' show sha1; +import 'package:sqflite/sqflite.dart' show ConflictAlgorithm; import 'package:flutter/foundation.dart'; import 'findings.dart'; import 'nap_overlap_cleanup.dart'; @@ -72,8 +73,14 @@ import 'hr_max.dart' show estimatedMaxHr, kHrFloorBpm, smoothedMaxHr, smoothedMinHr; import 'movement_floor_policy.dart' as mfp; import 'sleep_profile_policy.dart'; +import 'ed2b_checkpoint_diagnostics.dart'; import 'staged_sleep_cache.dart'; +import 'ed2b_stream_diagnostics.dart'; +import 'cardio_rows_cache.dart'; +import 'night_rr_state.dart'; +import 'night_checkpoint.dart'; import 'substrate_retention.dart'; +import 'substrate_union.dart'; import 'derive_prepare.dart'; import 'onehz_pipeline.dart'; import 'step_cadence.dart'; @@ -1861,7 +1868,32 @@ import 'vendor_sleep.dart'; // applyNapEdits — so a pre-cap manual nap longer than 3 h no longer counts // into naps/nap_min. Days derived at v120 keep their stale naps without this // bump. -const int kAlgoVersion = 121; +// 121 → 122 (analytics #99 @ f8e7d3c): RR artifact-run cap 32; +// toleranced batch aggregators hrvTime, banisterTrimp, ctlAtlTsb, cosinor, +// timeInZone and circadianNonparametric move last digits; Lomb–Scargle origin, +// singular-cosinor abstention, non-finite guards and IrregularScreenState's +// irregular-screen gaps change outputs. Retains the cosinor significance gate, +// nullable phase/amplitudeP, RSA floor guard and checkpoint v3, and SRI rules. +// Pooled-run batch staging labels unchanged; cardio caching respects run bounds. +// D35 queries coverage and resolves half-open ownership through inclusive to + 1, +// excluding non-owner rows at the last second. D36 folds only the target night's +// own observation, none when absent; forward-only, preserving existing profiles +// and folded_days. The restricted E15 walk now also applies to folding passes. +// 122 → 123 (analytics #99 + #98 @ 801c825): restore the observed 5-15 min +// workout rest bridge, including the duration-cap and saved-session overlap +// guards, while retaining #99's incremental, accumulator and checkpoint APIs. +// 123 → 124 (analytics main #102-#110 @ a438df4): retain #98's workout +// rest bridge and adopt the merged clock policies: HRV/RSA/CVHR abstain on +// regressing clocks; frequency HRV omits re-anchored overlap up to 5%. +// Cosinor refuses unidentifiable phase columns and non-finite designs; +// irregular screening abstains on degenerate SDNN; missing stage scores widen +// uncertainty. The #102 batch output changes and D35/D36 remain as at v122; +// #110 tags each night observation with its staged window, so D36 folds the +// chosen night's own observation (the one whose window overlaps it), none when +// no window does, instead of the longest of the search window's sessions. +// HrvFreqStream v2, CardioRowCache v3 and policy-bound PRSA states require cold +// rebuilds of old checkpoints, isolated by this algorithm version. +const int kAlgoVersion = 124; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. /// @@ -2048,7 +2080,13 @@ const int kAlgoVersion = 121; // merge + spread calibration + capped re-centring + robust lnRMSSD // centre), for v112. // REPIN @ 3a7ca34: analytics main, #101 + #100 + #97 + #98, for v119. -const String kAnalyticsPin = '3a7ca34d30233ff11e6c2877083f319a907ccb31'; +// REPIN @ f8e7d3c: rebased analytics #99, for v122 above. +// REPIN @ 801c825: merge #98's workout rest bridge into #99, for v123. +// REPIN @ 614a48e: upstream main, merged #102-#109 and #98, for v124. +// REPIN @ a438df4: analytics main, #110 (each night observation carries its +// staged window), still v124: that version had not shipped, so no stored day +// was derived at it with the previous pin. +const String kAnalyticsPin = 'a438df40b38a4586a52a68ab7e1c100fca111ccd'; // Repinned to analytics main's tip, which carries BOTH PR #72 (hrv_freq // Welch gap guard) and PR #73 (overreachingConjunction rhr quantum guard) — // the two independent kAlgoVersion bumps above (93 and 94). Verified both @@ -2977,6 +3015,13 @@ class _PrepareStats { int rows = 0; } +class _PrepareUnion { + RetainedSubstrate? load; + Substrate? search; + Map> ownership = const {}; + Map> priority = const {}; +} + /// Run [worker] over [items] with at most [concurrency] running at once. Each /// of up to [concurrency] "lanes" pulls the next unclaimed item as soon as /// it's free — a mix of fast (empty/mostly-empty day) and slow (heavy @@ -3110,10 +3155,87 @@ class CompactResult { /// The experimental HMM sleep staging switch (Settings), default off. const kSleepHmmStagesPref = 'sleep_hmm_stages'; +/// The observation of the night [onsetSec, offsetSec): the recorded one whose +/// staged window overlaps it most, or null when none does. An observation with +/// no window (an older analytics) never matches, and the night folds nothing +/// rather than a session of another day. +@visibleForTesting +ana.SleepNightObservation? observationOfNight( + List obs, int onsetSec, int offsetSec) { + ana.SleepNightObservation? best; + var bestOverlap = 0; + for (final o in obs) { + final s = o.startTsSec, e = o.endTsSec; + if (s == null || e == null) continue; + final int lo = s > onsetSec ? s : onsetSec; + final int hi = e < offsetSec ? e : offsetSec; + final int overlap = hi - lo; + if (overlap > bestOverlap) { + best = o; + bestOverlap = overlap; + } + } + return best; +} + class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; + // ED-2b observations of the worker-owned caches and checkpoint path. + final debugCheckpoints = Ed2bCheckpointDiagnostics(); + @visibleForTesting + void Function(String day)? debugOnProfileFold; + @visibleForTesting + Future debugFoldObservation(String day, String observation) => + _foldObservationIntoProfile(day, observation); + @visibleForTesting + // Optional observations follow the real day-bundle worker and its RAM owner. + Ed2bStreamDiagnostics? debugNightStreams; + @visibleForTesting + void Function(Map input, Map output)? + debugOnNightBundle; + @visibleForTesting + Uint8List? get debugCardioRowBytes => + _cardioRowChunks.isEmpty ? null : _cardioRowChunks.last.bytes; + @visibleForTesting + bool debugDisableCardioRows = false; + @visibleForTesting + int debugCardioRowHits = 0; + @visibleForTesting + int debugCardioRowMisses = 0; + @visibleForTesting + List get debugCardioRowWorkerOperations => _cardioRowOperations.toList(); + @visibleForTesting + void debugRekeyCardioRows({required int inputRevision, + required int generation, required int algorithm}) { + for (final c in _cardioRowChunks) { + c.generation = generation; + c.algorithm = algorithm; + c.revisions.updateAll((_, fp) => (inputRevision, fp.$2, generation)); + } + } + @visibleForTesting + bool debugDisableUnionLoads = false; + @visibleForTesting + int get debugUnionWorkerCuts => _unionWorkerCuts; + @visibleForTesting + void debugRekeyUnionAlgorithm(int algorithm) { + for (final union in _retained.unions) { union.version = algorithm; } + } + @visibleForTesting + void Function(int from, int to, String load)? debugOnSubstrateLoad; + @visibleForTesting + Future<(Map>, Map>)> + debugResolveOwnership(int from, int to) => _resolveOwnership(from, to); + @visibleForTesting + Future debugLoadOwnedSubstrate(int from, int to, + {required String day}) async { + final (ownership, _) = await _resolveOwnership(from, to); + return _loadSubstrateRange(from, to, dayId: day, ownership: ownership); + } + + // ED-2a RED seams. Empty cache views and no-op controls describe the current // uncached engine. Counters below observe real worker work, never results. // E12/E13 back staging and retention; E18 backs finished-session reuse. @@ -3159,6 +3281,15 @@ class DerivationEngine { } final _retained = SubstrateRetention(); + final _unionLock = _AsyncLock(); + int _unionWorkerCuts = 0; + final _cardioRowChunks = []; + final _cardioRowOperations = {}; + final _nightStates = {}; + final _checkpointPlans = {}; + int? _checkpointGeneration; + static const _cardioRowsSlackSec = 3600; + static const _cardioRowsMaxBytes = 8 * 1024 * 1024; int _retentionEpoch = 0; int? _retentionPassEpoch; bool _retentionInBackground = false; @@ -3175,6 +3306,12 @@ class DerivationEngine { void _clearRetainedSubstrate() { _retentionEpoch++; _retained.clear(); + _cardioRowChunks.clear(); + _nightStates.clear(); + _checkpointPlans.clear(); + debugCheckpoints.lastEncodedPayload = null; + debugCheckpoints.lastDecodedPayload = null; + debugNightStreams?.nightState.clear(); } void _beginRetentionPass() { @@ -3185,12 +3322,20 @@ class DerivationEngine { void _endRetentionPass() { _retained.retainDays(_retentionPreparedDays); + _nightStates.removeWhere((day, state) { + if (_retentionPreparedDays.contains(day)) return false; + state.clear(); + return true; + }); _retentionPreparedDays.clear(); _retentionPassEpoch = null; + _checkpointPlans.clear(); } void _dropRetainedDay(String day) { _retained.remove(day); + _nightStates.remove(day)?.clear(); + _checkpointPlans.remove(day); _retentionPreparedDays.remove(day); } @visibleForTesting @@ -3228,6 +3373,23 @@ class DerivationEngine { } } + /// The revision a night's RAM state is keyed on: every touched day's input + /// revision summed over the sleep window, and the store generation. + Future<(int, int)> _nightInputRevision(int sleepOnsetSec, int sleepOffsetSec) async { + var revision = 0; + var generation = LocalDb.storeGeneration; + final first = DateTime.fromMillisecondsSinceEpoch(sleepOnsetSec * 1000); + final last = sleepOffsetSec > sleepOnsetSec ? sleepOffsetSec - 1 : sleepOnsetSec; + for (var date = DateTime(first.year, first.month, first.day); + date.millisecondsSinceEpoch ~/ 1000 <= last; + date = DateTime(date.year, date.month, date.day + 1)) { + final fp = await LocalDb.inputDayFingerprint(dayLabelOf(date)); + revision += fp.revisionSum; + generation = fp.generation; + } + return (revision, generation); + } + Future _sessionCeilingRevision() async { try { return await LocalDb.sessionCeilingRevision(); @@ -4257,6 +4419,7 @@ class DerivationEngine { static const int _rawDecodeBatchSize = 2000; static const int _maxDayRawRows = 500000; static const int _maxDayRawPages = 300; + static const int _maxUnionBeats = 4 * _maxDayRawRows; Future _prepareTargetDay(String dayId) async { try { @@ -4282,7 +4445,13 @@ class DerivationEngine { // is telemetry-only). Each day now gets its own local accumulator, // merged into the shared running max exactly once, below. final stats = _PrepareStats(); - final candidate = await _sleepCandidateForDay(dayId, stats: stats); + final union = debugDisableUnionLoads ? null : _PrepareUnion(); + // Provenance BEFORE any load: a decoded write that lands while the + // substrate is read then leaves the older revision attached to the newer + // rows, which the signature check rejects. Read after, it would attach the + // newer revision to rows that predate it and certify a stale ceiling. + final ceilingInput = await _ceilingInputRevision(dayId); + final candidate = await _sleepCandidateForDay(dayId, stats: stats, union: union); final dayStart = _localDayLabelToSec(dayId); final dayEnd = _localNextDayLabelToSec(dayId); @@ -4306,8 +4475,28 @@ class DerivationEngine { if (candidate.present) candidate.sleepOffsetSec, ].reduce(math.max); - final (ownership, priority) = await _resolveOwnership(unionFrom, unionTo); - final ceilingInput = await _ceilingInputRevision(dayId); + final (ownership, priority) = union?.load != null && + union!.load!.from == unionFrom && union.load!.to == unionTo + ? (Map>.of(union.ownership), + Map>.of(union.priority)) + : await _resolveOwnership(unionFrom, unionTo); + if (union != null && (union.load == null || + unionFrom < union.load!.from! || unionTo > union.load!.to!)) { + // A banked candidate needs no search load. A live search already in + // the holder can extend to a wider selected night by borrowing itself. + final from = union.load == null + ? math.min(dayStart, candidate.present ? candidate.sleepOnsetSec : dayStart) + : unionFrom; + final to = union.load == null + ? math.max(dayEnd - 1 + napBoundaryBufferSec, + candidate.present ? candidate.sleepOffsetSec - 1 : 0) + : unionTo; + union.load = await _loadUnion(dayId, from, to, ownership, stats: stats); + } + final windows = await _cutUnionWindows(union?.load, [ + (dayStart, dayEnd - 1 + napBoundaryBufferSec), + (candidate.sleepOnsetSec, candidate.sleepOffsetSec - 1), + ], ownership, dayStart: dayStart, dayEnd: dayEnd); // Load the day PLUS the nap boundary buffer in ONE pass (each // _loadSubstrateRange spawns its own isolate, so a second load would @@ -4315,7 +4504,7 @@ class DerivationEngine { // this, the live decoded path — run()/runDays()/rescanRecent(), i.e. every // non-import day — fell back to napSub == daySub and went on bisecting // naps at midnight. - var napSub = await _loadSubstrateRange( + var napSub = windows.$1[0] ?? await _loadSubstrateRange( dayStart, dayEnd - 1 + napBoundaryBufferSec, dayId: dayId, @@ -4329,22 +4518,23 @@ class DerivationEngine { // "Saw" is the calendar day, the same test the scope and the re-derive // use (`decodedRecTsMaxByDay`): band rows only in the next morning's nap // buffer do not keep the wearable off a day it alone covers. - final wearable = - napSub.slice(dayStart, dayEnd).isEmpty ? await activeWearable() : null; + final wearable = (windows.$2 ?? napSub.slice(dayStart, dayEnd)).isEmpty + ? await activeWearable() : null; if (wearable != null) { await _resolveSparseOwnership( unionFrom, unionTo, wearable.$1, ownership, priority); napSub = await wearableSubstrate( wearable, dayStart, dayEnd - 1 + napBoundaryBufferSec); } - final daySub = napSub.slice(dayStart, dayEnd); + final daySub = wearable == null && windows.$2 != null + ? windows.$2! : napSub.slice(dayStart, dayEnd); Substrate sleepSub = Substrate.empty; if (candidate.present && candidate.sleepOffsetSec > candidate.sleepOnsetSec) { sleepSub = wearable != null ? await wearableSubstrate(wearable, candidate.sleepOnsetSec, candidate.sleepOffsetSec - 1) - : await _loadSubstrateRange( + : windows.$1[1] ?? await _loadSubstrateRange( candidate.sleepOnsetSec, candidate.sleepOffsetSec - 1, dayId: dayId, @@ -4424,6 +4614,8 @@ class DerivationEngine { /// candidate — so the spans differ, but the rule producing them does not. Future<(Map>, Map>)> _resolveOwnership(int from, int to) async { + // Loads include `to`; coverage queries and ownership spans are half-open. + final end = to + 1; final ownership = >{}; // The order each signal ACTUALLY resolved under, carried on the prepared // day so `priority_hash` names it by construction instead of re-reading @@ -4452,7 +4644,7 @@ class DerivationEngine { // gives it a rank — the conservative default, and the one that keeps // today's single-device installs byte-identical. final rawPriority = await LocalDb.signalPriority(sig); - final coverage = await LocalDb.coverageIntervals(sig, from, to); + final coverage = await LocalDb.coverageIntervals(sig, from, end); var resolved = rawPriority.isEmpty ? const [LocalDb.kPrimaryDeviceId] // A device with a coverage row here is definitionally declaring @@ -4483,7 +4675,7 @@ class DerivationEngine { coverage: coverage, priority: resolved, from: from, - to: to, + to: end, signal: sig, ); } @@ -4532,6 +4724,7 @@ class DerivationEngine { Future _sleepCandidateForDay( String dayId, { _PrepareStats? stats, + _PrepareUnion? union, }) async { // A user sleep override is the source of truth — never serve the cached auto // candidate, and don't cache the override result (so a later edit / clear is @@ -4580,6 +4773,32 @@ class DerivationEngine { // a night that was never scored. final (searchOwnership, searchPriority) = await _resolveOwnership(range.$1, range.$2); + final rowsEpoch = _retentionEpoch; + _checkpointPlans.remove(dayId); + NightCheckpointPlan? checkpointPlan; + if (!debugCheckpoints.disabled && _canRetainSubstrate) { + try { + final db = await LocalDb.instance; + final generation = LocalDb.storeGeneration; + if (_checkpointGeneration != null && _checkpointGeneration != generation) { + // Invalidate cached state while preserving the owners' work counters. + for (final state in _nightStates.values) { state.clear(); } + _cardioRowChunks.clear(); + _checkpointPlans.clear(); + debugCheckpoints.lastEncodedPayload = null; + debugCheckpoints.lastDecodedPayload = null; + } + _checkpointGeneration = generation; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [range.$1 ~/ 900, range.$2 ~/ 900]); + checkpointPlan = NightCheckpointPlan(range.$1, range.$2, + {for (final r in rows) r['bucket'] as int: r['rev'] as int}, + generation, rowsEpoch, await LocalDb.storeIdentity(executor: db)); + } catch (_) { + // Missing provenance costs a cold pass, never a partial adoption. + } + } Future>?> readRevisions() async { try { final out = >[]; @@ -4596,9 +4815,28 @@ class DerivationEngine { return null; // Unavailable provenance costs a staging, never a hit. } } - - var revisions = debugDisableStagedSleepCache ? null : await readRevisions(); - final searchSub = await _loadSubstrateRange( + var revisions = (!debugDisableStagedSleepCache || + (!debugDisableCardioRows && _canRetainSubstrate)) + ? await readRevisions() + : null; + Substrate? cutSearch; + if (union != null) { + final from = range.$1; + final to = _localNextDayLabelToSec(dayId) - 1 + napBoundaryBufferSec; + final (owned, ranked) = await _resolveOwnership(from, to); + union.ownership = owned; + union.priority = ranked; + union.load = await _loadUnion(dayId, from, to, owned, stats: stats, + search: union, window: (from: range.$1, to: range.$2, + ownership: searchOwnership)); + cutSearch = union.search; + union.search = null; + if (cutSearch == null) { + final cuts = await _cutUnionWindows(union.load, [range], searchOwnership); + cutSearch = cuts.$1.single; + } + } + final searchSub = cutSearch ?? await _loadSubstrateRange( range.$1, range.$2, dayId: dayId, @@ -4672,17 +4910,18 @@ class DerivationEngine { // ([_prepareTargetDay]): band rows from the previous afternoon, before // the ring went on, must not leave the ring's night unstaged. final ownNight = override == null && - searchSub.slice(_localDayLabelToSec(dayId), range.$2 + 1).isEmpty + (searchSub.lastTs ?? -1) < _localDayLabelToSec(dayId) ? await wearableNight(range.$1, range.$2) : null; if (ownNight != null) vendorNights = [...vendorNights, ownNight]; String? stagingKey; - if (revisions != null) { + if ((revisions != null && !debugDisableStagedSleepCache) || checkpointPlan != null) { try { - stagingKey = stagingInputKey( + final stagingWearable = override == null ? await activeWearable() : null; + String inputKey(bool prefixContext) => stagingInputKey( fromSec: range.$1, toSec: range.$2, - revisions: revisions, + revisions: revisions ?? const [], sub: searchSub, ownership: searchOwnership, priority: searchPriority, @@ -4690,22 +4929,110 @@ class DerivationEngine { profileJson: profileJson, priorSleep: priorSleep, vendorNights: vendorNights, - activeWearable: override == null ? await activeWearable() : null, + activeWearable: stagingWearable, + prefixContext: prefixContext, ); + if (revisions != null && !debugDisableStagedSleepCache) stagingKey = inputKey(false); + if (checkpointPlan != null) { + // Prefix reuse omits revision aggregates. The durable store id + // survives restart and changes when wipe/restore replaces inputs. + checkpointPlan.context = sha1.convert(utf8.encode(jsonEncode([ + checkpointPlan.storeId, inputKey(true), + ]))).toString(); + checkpointPlan.boundary = ((searchSub.lastTs ?? 0) + 1) ~/ 900 * 900; + } } catch (_) { // A context that cannot be keyed still takes the ordinary worker path. + checkpointPlan = null; } } // Cancellable + TIMED OUT. This site previously used a bare `Isolate.run` // with no timeout at all, so a hung staging pass never completed its future // — `_running` stayed true and `DeriveScheduler._drain` never returned, i.e. // all derivation was dead until app restart. - final skipUnusedDays = !mayFold && !debugDisableTargetDaySkip; + final skipUnusedDays = !debugDisableTargetDaySkip; // Read here: a `_perDayTimeout` inside `stage` would capture `this` into // the context the worker closure ships, and the engine's callbacks (the // app's own fields) are not sendable. final stageTimeout = _perDayTimeout; - Future<(String, String?, int)> stage() => _runIsolateCancellable(() { + final cacheRows = !debugDisableCardioRows && _canRetainSubstrate && + rowsEpoch == _retentionEpoch && revisions != null; + final rowRevisions = { + for (final r in revisions ?? const >[]) + r[0] as String: (r[1] as int, r[2] as int, r[3] as int), + }; + final rowGeneration = LocalDb.storeGeneration; + final rowsLo = range.$1 - _cardioRowsSlackSec; + final rowsHi = range.$2 + _cardioRowsSlackSec; + _cardioRowChunks.removeWhere((c) => + c.generation != rowGeneration || c.algorithm != kAlgoVersion || + (c.overlaps(rowsLo, rowsHi) && + !c.matches(rowRevisions, rowGeneration, kAlgoVersion))); + final sentChunks = cacheRows ? [for (final c in _cardioRowChunks) + if (c.overlaps(rowsLo, rowsHi)) c] : []; + final rowBytes = [for (final c in sentChunks) c.bytes]; + if (checkpointPlan != null && checkpointPlan.boundary > range.$1 && + checkpointPlan.context != null) { + // A RAM state clears itself when the input revision moved (see + // NightRrState.context), so one that no longer matches is not a live + // start: clear it here and let the compatible disk checkpoint restore + // the night, instead of skipping it and reprocessing the whole night. + final ram = _nightStates[dayId]; + var ramLive = ram?.canStream == true; + if (ram != null && ramLive) { + try { + final w = ram.window; + if (w == null) { + ramLive = false; + } else { + final (rev, gen) = await _nightInputRevision(w.$1, w.$2); + ramLive = ram.hasContext(rev, gen, kAlgoVersion); + } + } catch (_) { + ramLive = false; + } + if (!ramLive) ram.clear(); + } + if (!ramLive && + _nightStates[dayId]?.acceptsCheckpointAlgorithm(kAlgoVersion) != false) { + try { + debugCheckpoints.reads++; + final row = await LocalDb.dayCheckpoint(dayId, kAlgoVersion, + maxBytes: maxNightCheckpointBytes); + if (row == null) { + debugCheckpoints.misses++; + } else { + final context = checkpointPlan.context!; + final from = checkpointPlan.from, through = checkpointPlan.through; + final inputRevisions = checkpointPlan.revisions; + final searchRr = searchSub.rrMs, searchTimes = searchSub.rrTsMs; + final restored = await _runIsolateCancellable( + () => decodeNightCheckpoint(row, dayId, kAlgoVersion, context, + from, through, inputRevisions, + searchRr: searchRr, searchTimes: searchTimes), _perDayTimeout, + label: 'checkpoint-read $dayId'); + if (restored.rejection != null) { + debugCheckpoints.rejections.add(restored.rejection!); + debugCheckpoints.misses++; + } else if (LocalDb.storeGeneration == checkpointPlan.generation && + rowsEpoch == _retentionEpoch) { + checkpointPlan.restored = restored.night; + debugCheckpoints.lastDecodedPayload = restored.payload; + if (cacheRows && restored.cardio != null) rowBytes.add(restored.cardio!); + } + } + } catch (_) { + debugCheckpoints.misses++; + } + } + checkpointPlan.cardio = rowBytes; + _checkpointPlans[dayId] = checkpointPlan; + while (_checkpointPlans.length > 3) { + _checkpointPlans.remove(_checkpointPlans.keys.first); + } + } + Future<(String, String?, int, Uint8List?, int, int, List)> stage() => + _runIsolateCancellable(() { try { final p = profileJson == null ? null @@ -4727,15 +5054,25 @@ class DerivationEngine { ana.cardioRecordObservations = true; ana.resetCardioObservations(); debugSegmentedDayCount = 0; + final rowOperations = []; + final ana.CardioRowCache? rowCache; + if (cacheRows) { + rowCache = decodeCardioRows(rowBytes); + rowOperations.addAll(['decode:worker', 'merge:worker']); + } else { + rowCache = null; + } final candidate = prepareSleepSessionCandidate( searchSub, targetDay: dayId, skipUnusedDays: skipUnusedDays, + recordTargetOnly: true, override: override, priorSleep: priorSleep, vendorNights: vendorNights, + rowCache: rowCache, ); - // Fold the MAIN sleep (most epochs) of a freshly-staged night into the + // Fold the TARGET sleep (most epochs of its own observations) into the // rolling profile — done here in the worker because the observations live // in THIS isolate's globals. Skipped for overrides. EWMA self-seeds. String? observationJson; @@ -4748,11 +5085,14 @@ class DerivationEngine { // onto whichever day was re-derived last. Measured effect of that // corrupt profile on the same nights: wake 4.3% -> 36.4%, deep 1.9% -> // 0.0%. One fold per day_id, ever. - if (mayFold) { + if (mayFold && candidate.present) { final obs = ana.takeCardioObservations(); - if (obs.isNotEmpty) { - obs.sort((a, b) => b.epochs.compareTo(a.epochs)); - final main = obs.first; + // THIS night's own observation, matched on the window it was staged + // over. The search window also stages the day before's sessions; the + // longest of all of them is not necessarily the night chosen. + final main = observationOfNight( + obs, candidate.sleepOnsetSec, candidate.sleepOffsetSec); + if (main != null) { if (main.epochs >= 120) { // require ≥60 min — not a nap. // Return the raw OBSERVATION, not a folded profile. Folding here @@ -4775,7 +5115,12 @@ class DerivationEngine { } } } - return (jsonEncode(candidate.toJson()), observationJson, debugSegmentedDayCount); + final rowsOut = rowCache == null ? null : + encodeCardioRows(rowCache, rowsLo, rowsHi); + if (rowCache != null) rowOperations.add('prune:worker'); + return (jsonEncode(candidate.toJson()), observationJson, + debugSegmentedDayCount, rowsOut, rowCache?.hits ?? 0, + rowCache?.misses ?? 0, rowOperations); }, stageTimeout, label: 'sleep-staging $dayId'); final hit = stagingKey == null ? null : _stagedSleepCache.lookup( dayId, kAlgoVersion, stagingKey, mayFold: mayFold); @@ -4788,8 +5133,31 @@ class DerivationEngine { } else { debugStagingComputations++; final result = await stage(); + if (checkpointPlan != null && result.$4 != null) { + checkpointPlan.cardio = [result.$4!]; + } candidateJson = result.$1; observationJson = result.$2; + debugCardioRowHits += result.$5; + debugCardioRowMisses += result.$6; + _cardioRowOperations.addAll(result.$7); + if (_canRetainSubstrate && rowsEpoch == _retentionEpoch && + LocalDb.storeGeneration == rowGeneration && !debugDisableCardioRows) { + _cardioRowChunks.removeWhere(sentChunks.contains); + if (result.$4 != null && result.$4!.length <= _cardioRowsMaxBytes) { + _cardioRowChunks.add(CardioRowsChunk(result.$4!, rowsLo, rowsHi, + rowRevisions, rowGeneration, kAlgoVersion)); + } + // Bound both disjoint windows and repeated fingerprint variants. + final newest = _cardioRowChunks.fold(rowsHi, + (hi, c) => math.max(hi, c.hiSec)); + _cardioRowChunks.removeWhere((c) => c.hiSec < newest - 4 * 24 * 3600); + while (_cardioRowChunks.length > 4 || + _cardioRowChunks.fold(0, (n, c) => n + c.bytes.length) > + _cardioRowsMaxBytes) { + _cardioRowChunks.removeAt(0); + } + } debugOnStagingWalk?.call(mayFold, result.$3, observationJson); if (stagingKey != null) { _stagedSleepCache.store(dayId, StagedSleepEntry( @@ -4929,6 +5297,7 @@ class DerivationEngine { // profile, fold, and clobber the other, losing both the fold and its // day_id from folded_days. SQLite's write lock is cross-connection and // therefore cross-isolate. + debugOnProfileFold?.call(dayId); await LocalDb.updateBaseline('sleep_user_profile', (current) { // Re-derive freshness INSIDE the transaction: the value this day read // before staging is stale by definition, another lane may have folded @@ -5028,6 +5397,116 @@ class DerivationEngine { return (startSec: startSec, endSec: endSec); } + /// The E13 owner holds one union per day. Only the loading turn is serial: + /// a neighbouring day can borrow its completed overlap while staging runs. + Future _loadUnion(String dayId, int from, int to, + Map> ownership, {_PrepareStats? stats, + _PrepareUnion? search, SubstrateUnionWindow? window}) { + final epoch = _retentionEpoch; + return _unionLock.run(() async { + int generation; + Map revisions; + try { + final db = await LocalDb.instance; + generation = LocalDb.storeGeneration; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [from ~/ 900, to ~/ 900]); + revisions = {for (final r in rows) + (r['bucket'] as num).toInt(): (r['rev'] as num).toInt()}; + } catch (_) { + _dropRetainedDay(dayId); + return null; + } + final context = substrateLoadContext(from, to, generation, ownership); + final bases = _canRetainSubstrate ? [for (final base in _retained.unions) + if (base.version == kAlgoVersion && base.generation == generation && + base.context == substrateLoadContext(base.from!, base.to!, + generation, base.ownership)) base] : []; + final stretches = reusableUnionStretches(bases, from, to, revisions, ownership); + final timeout = _perDayTimeout; + final whole = stretches.length == 1 && + stretches.single.$1 == stretches.single.$3.from && + stretches.single.$2 == stretches.single.$3.to; + final borrowed = whole ? [stretches.single.$3] : + stretches.isEmpty ? [] : + await _runIsolateCancellable(borrowUnionJob(stretches, ownership), + timeout, label: 'union-borrow $dayId'); + if (!whole) _unionWorkerCuts += borrowed.whereType().length; + final parts = []; + Future read(int a, int b) async { + debugOnSubstrateLoad?.call(a, b, 'union'); + final (sub, extras) = await _readSubstrateRange(a, b, dayId: dayId, + stats: stats, ownership: ownership, retain: true, + unionWindow: a == from && b == to ? window : null); + if (extras.isEmpty) return null; + final cut = extras.remove('union_window') as Substrate?; + if (cut != null) { + search?.search = cut; + _unionWorkerCuts++; + } + return RetainedSubstrate(version: kAlgoVersion, context: context, + revisions: revisions, sub: sub, extras: extras, cut: a, + from: a, to: b, generation: generation, ownership: ownership); + } + var pos = from; + for (var i = 0; i < stretches.length; i++) { + final part = borrowed[i]; + if (part == null) continue; + if (part.from! > pos) { + final gap = await read(pos, part.from! - 1); + if (gap == null) return null; + parts.add(gap); + } + parts.add(part); + pos = part.to! + 1; + } + if (pos <= to) { + final gap = await read(pos, to); + if (gap == null) return null; + parts.add(gap); + } + var loaded = parts.length == 1 ? parts.single : + await _runIsolateCancellable( + joinUnionJob(parts, context, revisions, ownership), timeout, + label: 'union-join $dayId'); + loaded ??= await read(from, to); + if (loaded == null) return null; + final rows = (loaded.extras['raw_rows_by_bucket'] as Map) + .values.fold(0, (int n, v) => n + v); + _enforcePrepareBudget(dayId: dayId, fromRecTs: from, toRecTs: to, + rangePages: (rows / _rawDecodeBatchSize).ceil(), rangeRows: rows); + if (_canRetainSubstrate && epoch == _retentionEpoch && + LocalDb.storeGeneration == generation && + loaded.sub.length <= _maxDayRawRows && + loaded.sub.rrMs.length <= _maxUnionBeats) { + // A sole borrowed part still carries its original, wider context. + loaded = RetainedSubstrate(version: kAlgoVersion, context: context, + revisions: revisions, sub: loaded.sub, extras: loaded.extras, + cut: from, from: from, to: to, generation: generation, + ownership: ownership); + _retained.store(dayId, 'union', loaded); + } + return loaded; + }); + } + + /// The calendar day deliberately keeps the buffered load's provenance and + /// beat-time slice semantics, just as the separate-window path does. + Future<(List, Substrate?)> _cutUnionWindows( + RetainedSubstrate? load, List<(int, int)> ranges, + Map> ownership, + {int? dayStart, int? dayEnd}) async { + if (load == null) return ([for (final _ in ranges) null], null); + final timeout = _perDayTimeout; + final cuts = await _runIsolateCancellable( + cutUnionJob(load, ranges, ownership, dayStart, dayEnd), + timeout, label: 'union-windows'); + _unionWorkerCuts += cuts.$1.whereType().length; + if (cuts.$2 != null) _unionWorkerCuts++; + return cuts; + } + Future _loadSubstrateRange( int fromRecTs, int toRecTs, { @@ -5036,6 +5515,7 @@ class DerivationEngine { Map> ownership = const {}, String load = 'direct', }) async { + debugOnSubstrateLoad?.call(fromRecTs, toRecTs, load); if (toRecTs < fromRecTs) { _dropRetainedDay(dayId); return Substrate.empty; @@ -5097,6 +5577,7 @@ class DerivationEngine { Map> ownership = const {}, RetainedSubstrate? seed, bool retain = false, + SubstrateUnionWindow? unionWindow, }) async { if (toRecTs < fromRecTs) return (Substrate.empty, const {}); final port = ReceivePort(); @@ -5138,6 +5619,11 @@ class DerivationEngine { final payload = ((message['payload'] as Map?) ?? const {}) .cast(); extras = ((message['retain'] as Map?) ?? const {}).cast(); + final window = message['union_window']; + if (window is Map) { + extras = {...extras, + 'union_window': Substrate.fromTransfer(window.cast())}; + } if (!result.isCompleted) result.complete(Substrate.fromTransfer(payload)); } return; @@ -5169,7 +5655,10 @@ class DerivationEngine { } try { final worker = await ready.future; - worker.send({'type': 'config', 'mode': 'substrate', 'retain': retain}); + worker.send({'type': 'config', 'mode': 'substrate', 'retain': retain, + if (unionWindow != null) ...{'union_window': unionWindow, + 'load_from': fromRecTs, 'load_to': toRecTs, 'ownership': ownership}, + }); if (seed != null) worker.send(seed.seed(fromRecTs)); int? afterRecTs; int? afterCursor; @@ -6422,11 +6911,96 @@ class DerivationEngine { // Cancellable: on timeout the isolate is KILLED, not merely abandoned to // keep burning a core behind the worker pool's back. - final bundle = await _runIsolateCancellable( - () => deriveDayBundle(withHistory), + final nightEpoch = _retentionEpoch; + final retainNight = _canRetainSubstrate && !suppliedSubstrate; + final diagnostics = debugNightStreams; + final checkpointPlan = _checkpointPlans.remove(day.date); + final checkpointEnabled = !debugCheckpoints.disabled && retainNight && + checkpointPlan?.context != null && checkpointPlan!.epoch == nightEpoch && + checkpointPlan.generation == LocalDb.storeGeneration && + day.sleepOnsetSec >= checkpointPlan.from && + day.sleepOffsetSec <= checkpointPlan.through + 1; + var nightRevision = 0, nightGeneration = LocalDb.storeGeneration; + var nightProvenance = false; + // Keep DB reads and engine fields outside the shipped closure. A changed + // revision rejects even a byte-identical replacement; raw prefix proofs in + // the worker additionally detect moved windows and in-memory mutations. + NightRrState? nightState; + if ((retainNight && sleepSub.rrMs.isNotEmpty) || diagnostics != null) { + nightState = _nightStates.remove(day.date) ?? NightRrState(); + try { + final (revision, generation) = + await _nightInputRevision(day.sleepOnsetSec, day.sleepOffsetSec); + nightState.context(inputRevision: revision, generation: generation, + algorithm: kAlgoVersion); + nightState.window = (day.sleepOnsetSec, day.sleepOffsetSec); + nightRevision = revision; + nightGeneration = generation; + nightProvenance = true; + } catch (_) { + nightState.clear(); // Unavailable provenance never certifies reuse. + } + diagnostics?.nightState = nightState; + } + final canCheckpoint = checkpointEnabled && nightProvenance && + nightEpoch == _retentionEpoch && + LocalDb.storeGeneration == checkpointPlan.generation; + final restoredNight = canCheckpoint ? checkpointPlan.restored : null; + final checkpointContext = checkpointEnabled ? checkpointPlan.context! : ''; + final checkpointBoundary = checkpointEnabled ? checkpointPlan.boundary : 0; + final checkpointFrom = checkpointEnabled ? checkpointPlan.from : 0; + final checkpointVector = checkpointEnabled ? checkpointRevisions( + checkpointPlan.revisions, checkpointFrom, checkpointBoundary) : Uint8List(0); + final checkpointRows = checkpointEnabled ? checkpointPlan.cardio : []; + final checkpointDay = day.date; + final workerResult = await _runIsolateCancellable(() { + var state = nightState; + var resumed = false; + if (restoredNight != null && restoredNight.acceptsPrefix( + input.sleepRrMs, input.sleepRrTsMs, input.date, + input.sleepOnsetSec)) { + restoredNight.bindRestoredContext(nightRevision, nightGeneration, kAlgoVersion, + previous: state); + state = restoredNight; + resumed = true; + if (diagnostics != null) diagnostics.nightState = state; + } + final result = deriveDayBundle(withHistory, streamDiagnostics: diagnostics, + nightState: state); + NightCheckpointWrite? checkpoint; + if (canCheckpoint) { + try { + checkpoint = encodeNightCheckpoint(state, checkpointRows, checkpointDay, + kAlgoVersion, checkpointContext, checkpointFrom, checkpointBoundary, + checkpointVector); + } catch (_) { + // Scratch serialization cannot turn a successful derive into an error. + } + } + return (result, state, diagnostics, resumed, + restoredNight != null && !resumed, checkpoint); + }, _perDayTimeout, label: 'day-bundle ${day.date}', ); + final bundle = workerResult.$1; + if (workerResult.$4) debugCheckpoints.hits++; + if (workerResult.$5) { + debugCheckpoints.rejections.add('checkpoint night prefix/window'); + debugCheckpoints.misses++; + } + if (diagnostics != null) debugNightStreams = workerResult.$3; + final returnedNight = workerResult.$2; + if (returnedNight != null && retainNight && _canRetainSubstrate && + nightEpoch == _retentionEpoch) { + _nightStates[day.date] = returnedNight; + while (_nightStates.length > math.min(2, _retained.maxDays)) { + _nightStates.remove(_nightStates.keys.first)?.clear(); + } + } else { + returnedNight?.clear(); + } + debugOnNightBundle?.call(withHistory, bundle); // Readiness came back absent for TODAY specifically (not a historical // backfill day, which would just be noise) — log why. This ran inside // Isolate.run so it couldn't call Firebase itself; it just returned the @@ -7268,6 +7842,39 @@ class DerivationEngine { }, ); if (effectiveFinalized || effectivePartial) _dropRetainedDay(day.date); + if (!debugCheckpoints.disabled && !suppliedSubstrate) { + try { + final db = await LocalDb.instance; + if (effectiveFinalized || effectivePartial) { + await db.delete('day_checkpoint', where: 'day_id = ?', whereArgs: [day.date]); + } else if (checkpointEnabled && workerResult.$6 != null && + _canRetainSubstrate && nightEpoch == _retentionEpoch && + LocalDb.storeGeneration == checkpointPlan.generation) { + final checkpoint = workerResult.$6!; + await db.transaction((txn) async { + if (LocalDb.storeGeneration != checkpointPlan.generation || + await LocalDb.storeIdentity(executor: txn) != checkpointPlan.storeId) { + return; + } + await txn.insert('day_checkpoint', { + 'day_id': day.date, 'algo_version': kAlgoVersion, + 'fmt': nightCheckpointFormat, 'ctx_sig': checkpointContext, + 'cp_rec_ts': checkpointBoundary, 'rev_vec': checkpointVector, + 'state': checkpoint.bytes, 'night_ref': null, + 'computed_at': DateTime.now().millisecondsSinceEpoch, + }, conflictAlgorithm: ConflictAlgorithm.replace); + await txn.rawDelete('DELETE FROM day_checkpoint WHERE day_id NOT IN ' + '(SELECT day_id FROM day_checkpoint GROUP BY day_id ' + 'ORDER BY day_id DESC LIMIT 3) OR algo_version != ?', [kAlgoVersion]); + }); + debugCheckpoints.writes++; + debugCheckpoints.lastEncodedPayload = checkpoint.bytes; + debugCheckpoints.lastDecodedPayload = checkpoint.payload; + } + } catch (_) { + // Disposable scratch is never required to commit a day result. + } + } // AFTER the row: retired only once the row that proves it is durable. if (quietSettled) await LocalDb.deleteComputeFreshness(quietKey); // NOTE: the sweep's `history` snapshot is deliberately NOT updated here. diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index b008c9473..12845ca04 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -6,6 +6,8 @@ import 'package:openstrap_analytics/onehz.dart' as ana; import '../ble/adapters/signals.dart'; import '../data/coverage_resolver.dart'; import 'substrate.dart'; +import 'substrate_retention.dart'; +import 'substrate_union.dart'; import 'vendor_sleep.dart'; class PreparedDerivationDay { @@ -317,6 +319,9 @@ void derivationPrepareWorker(SendPort mainSendPort) { final state = _PrepareAccumulator(); String? targetDay; var mode = 'prepared_day'; + SubstrateUnionWindow? unionWindow; + var loadFrom = 0, loadTo = -1; + Map> loadOwnership = const {}; mainSendPort.send(port.sendPort); void handle(Object? message) { @@ -348,6 +353,10 @@ void derivationPrepareWorker(SendPort mainSendPort) { final cfgMode = message['mode']?.toString(); if (cfgMode != null && cfgMode.isNotEmpty) mode = cfgMode; state.retain = message['retain'] == true; + unionWindow = message['union_window'] as SubstrateUnionWindow?; + loadFrom = message['load_from'] as int? ?? 0; + loadTo = message['load_to'] as int? ?? -1; + loadOwnership = message['ownership'] as Map>? ?? const {}; return; } if (type == 'seed') { @@ -358,11 +367,20 @@ void derivationPrepareWorker(SendPort mainSendPort) { try { final substrate = state.buildSubstrate(); if (mode == 'substrate') { + final extras = state.retainable ? state.retainExtras() : null; + final request = unionWindow; + final window = request == null || extras == null ? null : + cutSubstrateUnion(RetainedSubstrate(version: 0, context: '', + revisions: const {}, sub: substrate, extras: extras, cut: loadFrom, + from: loadFrom, to: loadTo, ownership: loadOwnership), + request.from, request.to, request.ownership, + retainMetadata: false)?.sub; mainSendPort.send({ 'type': 'result', 'kind': 'substrate', 'payload': substrate.toJson(), - if (state.retainable) 'retain': state.retainExtras(), + 'retain': ?extras, + if (window != null) 'union_window': window.toJson(), }); } else { final payload = prepareDerivationPayload( @@ -415,6 +433,8 @@ PreparedDerivationPayload prepareDerivationPayload( List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], bool skipUnusedDays = false, + ana.CardioRowCache? rowCache, + bool recordTargetOnly = false, }) { if (sub.isEmpty || sub.lastTs == null) { return const PreparedDerivationPayload(dataNowSec: 0, days: []); @@ -422,6 +442,8 @@ PreparedDerivationPayload prepareDerivationPayload( final days = []; for (final day in calendarDays( sub, + rowCache: rowCache, + recordTargetOnly: recordTargetOnly, override: override, targetDay: targetDay, skipUnusedDays: skipUnusedDays, @@ -480,6 +502,8 @@ SleepSessionCandidate prepareSleepSessionCandidate( List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], bool skipUnusedDays = false, + ana.CardioRowCache? rowCache, + bool recordTargetOnly = false, }) { // An UNCLAIMED device night (our rows saw under half of it) ending today IS // the night: above our detection, below the user's word. It never reaches @@ -523,6 +547,8 @@ SleepSessionCandidate prepareSleepSessionCandidate( claimed.add(n); } final payload = prepareDerivationPayload(sub, + rowCache: rowCache, + recordTargetOnly: recordTargetOnly, targetDay: targetDay, override: override, skipUnusedDays: skipUnusedDays, @@ -653,6 +679,7 @@ class _PrepareAccumulator { final List _rrStart = []; final Map _buckets = {}; Float64List? _rrTsRaw; + final List> _provenance = []; void seed(Map message) { List ints(String k) => message[k] as List; @@ -681,6 +708,11 @@ class _PrepareAccumulator { rrTsMs.addAll(dbls('rr_ts_raw').getRange(0, beats)); rrMs.addAll(dbls('rr_ms').getRange(0, beats)); _rrStart.addAll(starts.getRange(0, n)); + for (final run in (message['provenance'] as List?) ?? const []) { + final r = (run as List).cast(); + if ((r[0] as int) >= n) break; + _provenance.add(List.from(r)); + } for (final wire in message['meta'] as List) { final m = wire as List; final bucket = m[0] as int; @@ -698,6 +730,7 @@ class _PrepareAccumulator { Map retainExtras() => { 'rr_start': Int32List.fromList(_rrStart), 'rr_ts_raw': _rrTsRaw!, + 'provenance': _provenance, 'meta': [for (final e in _buckets.entries) [e.key, e.value.families.toList(), e.value.deviceIds.toList(), e.value.skinUnit]], @@ -779,6 +812,16 @@ class _PrepareAccumulator { if (retain) { _rrStart.add(rrMs.length); _buckets.putIfAbsent(recTs ~/ 900, _BucketAcc.new).note(row); + final family = row?['device_family']; + final id = row?['device_id']; + final unit = row?['skin_temp_raw'] is num ? 'raw' : + (row?['skin_temp_c'] is num ? 'c' : null); + final f = family is String && family.isNotEmpty ? family : null; + final d = id is String ? id : null; + final last = _provenance.isEmpty ? null : _provenance.last; + if (last == null || last[1] != f || last[2] != d || last[3] != unit) { + _provenance.add([tsSec.length, f, d, unit]); + } } tsSec.add(recTs); // NULL (absent), a beat-only second (no row at all) and an impossible diff --git a/lib/compute/ed2b_checkpoint_diagnostics.dart b/lib/compute/ed2b_checkpoint_diagnostics.dart new file mode 100644 index 000000000..a39c0ce58 --- /dev/null +++ b/lib/compute/ed2b_checkpoint_diagnostics.dart @@ -0,0 +1,24 @@ +import 'dart:typed_data'; + +/// Observations of the real checkpoint path. No storage, codec or resume policy +/// lives here. Counters remain zero until the GREEN implementation reports work. +class Ed2bCheckpointDiagnostics { + bool disabled = false; + int reads = 0; + int writes = 0; + int hits = 0; + int misses = 0; + final List rejections = []; + Uint8List? lastEncodedPayload; + Map? lastDecodedPayload; + + void reset() { + reads = 0; + writes = 0; + hits = 0; + misses = 0; + rejections.clear(); + lastEncodedPayload = null; + lastDecodedPayload = null; + } +} diff --git a/lib/compute/ed2b_stream_diagnostics.dart b/lib/compute/ed2b_stream_diagnostics.dart new file mode 100644 index 000000000..76ff21597 --- /dev/null +++ b/lib/compute/ed2b_stream_diagnostics.dart @@ -0,0 +1,45 @@ +import 'night_rr_state.dart'; + +// Work is counted beside the real owner and the remaining batch metric calls. +class Ed2bStreamDiagnostics { + int passes = 0; + NightRrState nightState = NightRrState(); + int get processedNightBeats => nightState.processedNightBeats; + int get nightRebuilds => nightState.nightRebuilds; + final Map _batchBeats = {}; + final Map _batchCalls = {}; + bool disabled = false; + void Function(List nn, List times, double cleanFraction)? + onCorrected; + void Function()? onBeforePassReturn; + void Function(List)? onRespWindows; + + void beginPass() { passes++; } + void endPass() { onBeforePassReturn?.call(); } + void context({required int inputRevision, required int generation, + required int algorithm}) => nightState.context(inputRevision: inputRevision, + generation: generation, algorithm: algorithm); + // Total input work: GREEN counts only newly appended/settled beats here, + // regardless of whether the operation uses a batch call or an addSlice. + int get retainedNightBeats => nightState.retainedNightBeats; + int get settledNightBeats => nightState.settledNightBeats; + int get identityMemoEntries => nightState.identityMemoEntries; + int get fullListDigests => nightState.fullListDigests; + int get maximumListDigestsPerIdentity => nightState.maximumListDigestsPerIdentity; + int retainedStreamBeats(String stream) => nightState.retainedStreamBeats(stream); + int processedBeats(String stream) => nightState.processedBeats(stream) + + (_batchBeats[stream] ?? 0); + int slicesProcessed(String stream) => nightState.slicesProcessed(stream) + + (_batchCalls[stream] ?? 0); + int streamRestarts(String stream) => nightState.streamRestarts(stream); + + void observeNightCorrection(int rawBeats, List nn, + List times, double cleanFraction) { + onCorrected?.call(nn, times, cleanFraction); + } + + void observeBatch(String stream, int beats) { + _batchCalls[stream] = (_batchCalls[stream] ?? 0) + 1; + _batchBeats[stream] = (_batchBeats[stream] ?? 0) + beats; + } +} diff --git a/lib/compute/night_checkpoint.dart b/lib/compute/night_checkpoint.dart new file mode 100644 index 000000000..d30e9491a --- /dev/null +++ b/lib/compute/night_checkpoint.dart @@ -0,0 +1,115 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; + +import 'cardio_rows_cache.dart'; +import 'night_rr_state.dart'; + +// Format 2 carries RSA v3 and closed-bin spectral states, not saved rates. +// Older closed windows cannot be certified under the current RSA floor gate. +const nightCheckpointFormat = 2; +const maxNightCheckpointBytes = 8 * 1024 * 1024; + +/// One request's provenance, read before loading substrate. No decoded rows. +class NightCheckpointPlan { + NightCheckpointPlan(this.from, this.through, this.revisions, this.generation, + this.epoch, this.storeId); + final int from, through, generation, epoch; + final String storeId; + final Map revisions; + String? context; + int boundary = 0; + List cardio = []; + NightRrState? restored; +} + +Uint8List checkpointRevisions(Map revisions, int from, int boundary) => + Uint8List.fromList(utf8.encode(jsonEncode([ + for (var b = from ~/ 900; b < boundary ~/ 900; b++) + [b, revisions[b] ?? 0], + ]))); + +typedef NightCheckpointRead = ({NightRrState? night, Uint8List? cardio, + Map? payload, String? rejection}); +typedef NightCheckpointWrite = ({Uint8List bytes, Map payload}); + +/// Worker only. SHA-256 covers both sections and all SQL provenance. No section +/// is returned unless every section and the complete revision vector validate. +NightCheckpointRead decodeNightCheckpoint(Map row, + String day, int algorithm, String context, int from, int through, + Map revisions, {List? searchRr, List? searchTimes}) { + try { + final boundary = row['cp_rec_ts']; + if (row['day_id'] != day || row['algo_version'] != algorithm || + row['fmt'] != nightCheckpointFormat || row['ctx_sig'] != context || + row['night_ref'] != null || boundary is! int || boundary % 900 != 0 || + boundary <= from || boundary > through + 1) { + throw const FormatException('checkpoint provenance'); + } + final expectedRevisions = checkpointRevisions(revisions, from, boundary); + if (base64Encode(row['rev_vec'] as List) != base64Encode(expectedRevisions)) { + throw const FormatException('checkpoint revisions'); + } + final bytes = row['state'] as List; + if (bytes.length <= 32 || bytes.length > maxNightCheckpointBytes) { + throw const FormatException('checkpoint length'); + } + final body = bytes.sublist(32); + if (base64Encode(sha256.convert(body).bytes) != base64Encode(bytes.sublist(0, 32))) { + throw const FormatException('checkpoint checksum'); + } + final payload = (jsonDecode(utf8.decode(body)) as Map).cast(); + if (payload.length != 4 || payload['v'] != nightCheckpointFormat || + jsonEncode(payload['provenance']) != jsonEncode([ + day, algorithm, context, boundary, base64Encode(expectedRevisions), + ])) { + throw const FormatException('checkpoint envelope'); + } + final night = NightRrState.fromCheckpoint( + (payload['night'] as Map).cast(), boundary); + if (((payload['night'] as Map)['tag'] as Map)['day'] != day || + (searchRr != null && (searchTimes == null || + !night.acceptsSearchPrefix(searchRr, searchTimes, day)))) { + throw const FormatException('checkpoint raw prefix'); + } + final cardioJson = (payload['cardio'] as Map).cast(); + final cardio = ana.CardioRowCache.fromJson(cardioJson); + if (cardio.length > 8000) throw const FormatException('checkpoint cardio size'); + var outside = false; + cardio.retainWhere((k) { + if (k.startSec < from - 3600 || k.startSec + k.epochSec > boundary) outside = true; + return true; + }); + if (outside) throw const FormatException('checkpoint cardio window'); + return (night: night, + cardio: Uint8List.fromList(utf8.encode(jsonEncode(cardioJson))), + payload: payload, rejection: null); + } catch (e) { + return (night: null, cardio: null, payload: null, + rejection: 'checkpoint refused: ${e.runtimeType}'); + } +} + +/// Worker only. Oversized or absent nights stay on the ordinary cold path. +NightCheckpointWrite? encodeNightCheckpoint(NightRrState? night, + List chunks, String day, int algorithm, String context, + int from, int boundary, Uint8List revisions) { + final nightJson = night?.checkpoint(boundary); + if (nightJson == null) return null; + final cardio = decodeCardioRows(chunks); + var kept = 0; + cardio.retainWhere((k) => k.startSec >= from - 3600 && + k.startSec + k.epochSec <= boundary && kept++ < 8000); + final payload = { + 'v': nightCheckpointFormat, + 'provenance': [day, algorithm, context, boundary, base64Encode(revisions)], + 'night': nightJson, 'cardio': cardio.toJson(), + }; + final body = utf8.encode(jsonEncode(payload)); + if (body.length + 32 > maxNightCheckpointBytes) return null; + final bytes = Uint8List.fromList([...sha256.convert(body).bytes, ...body]); + return (bytes: bytes, + payload: (jsonDecode(utf8.decode(body)) as Map).cast()); +} diff --git a/lib/compute/night_prsa.dart b/lib/compute/night_prsa.dart new file mode 100644 index 000000000..beb64e9b2 --- /dev/null +++ b/lib/compute/night_prsa.dart @@ -0,0 +1,48 @@ +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart'; + +final _dc = prsaAggregator(deceleration: true); +final _ac = prsaAggregator(deceleration: false); + +/// Settled night NN in reference order. Provisional tails only touch copies. +class NightPrsa { + NightPrsa(); + Object? _dcState = _dc.empty(), _acState = _ac.empty(); + + Map toJson() => { + 'v': 1, 'dc': _dc.encodeState(_dcState), 'ac': _ac.encodeState(_acState), + }; + + factory NightPrsa.fromJson(Map json, int fed) { + if (json['v'] != 1) throw const FormatException('PRSA version'); + final dc = json['dc'], ac = json['ac']; + if (dc is! List || ac is! List || dc.first != fed || ac.first != fed) { + throw const FormatException('PRSA feed count'); + } + return NightPrsa() + .._dcState = _dc.decodeState(dc) + .._acState = _ac.decodeState(ac); + } + + void fold(List nn) { + final column = DoubleColumn(Float64List.fromList(nn)); + _dcState = _dc.addSlice(_dcState, column, 0, column.length); + _acState = _ac.addSlice(_acState, column, 0, column.length); + } + + (Metric, Metric) snapshot(List tailNn) { + final tail = DoubleColumn(Float64List.fromList(tailNn)); + Metric finish( + Aggregator> agg, + Object? state) { + final copy = agg.decodeState(agg.encodeState(state)); + return agg.finish(agg.addSlice(copy, tail, 0, tail.length)); + } + // Sequential addSlice preserves anchor order; never combine partial sums. + // Merged analytics prsa_aggregator.dart declares profile/capacity abs+rel 1e-9, + // confidence abs 1e-12 parity with the batch functions. + return (finish(_dc, _dcState), finish(_ac, _acState)); + } +} diff --git a/lib/compute/night_resp_bins.dart b/lib/compute/night_resp_bins.dart new file mode 100644 index 000000000..728769538 --- /dev/null +++ b/lib/compute/night_resp_bins.dart @@ -0,0 +1,109 @@ +import 'package:openstrap_analytics/onehz.dart'; + +/// Thirty-minute bins anchored at the first corrected beat, like the batch. +/// Keep closed spectral states so every snapshot applies the current night's +/// artifact gate through analytics, including to bins closed on a prior pass. +class NightRespBins { + NightRespBins(); + double? _origin; + int _openIndex = 0, _openCount = 0; + RsaRespStream _open = RsaRespStream(); + final List<({int count, RsaRespStream stream})> _closed = []; + + Map toJson() => { + 'v': 2, 'origin': _origin, 'index': _openIndex, 'count': _openCount, + 'open': _open.toJson(), + 'closed': [for (final bin in _closed) + {'count': bin.count, 'state': bin.stream.toJson()}], + }; + + factory NightRespBins.fromJson(Map json, int fed) { + final origin = json['origin'], index = json['index'], count = json['count']; + final closed = json['closed'] as List; + if (json['v'] != 2 || index is! int || index < 0 || count is! int || + count < 0 || count > fed || closed.length > fed ~/ 60 || + (fed == 0 ? origin != null : origin is! num || !origin.isFinite)) { + throw const FormatException('RSA bins'); + } + final open = (json['open'] as Map).cast(); + if (open['n'] != count) throw const FormatException('RSA bin count'); + final result = NightRespBins() + .._origin = (origin as num?)?.toDouble() + .._openIndex = index + .._openCount = count + .._open = RsaRespStream.fromJson(open); + var total = count; + for (final bin in closed) { + final n = (bin as Map)['count']; + final state = (bin['state'] as Map).cast(); + if (n is! int || n < 60 || n > fed || state['n'] != n) { + throw const FormatException('closed RSA bin count'); + } + total += n; + if (total > fed) throw const FormatException('RSA bins feed count'); + result._closed.add((count: n, stream: RsaRespStream.fromJson(state))); + } + return result; + } + + int _bin(double time, double origin) => ((time - origin) / 1800000.0).floor(); + + void _appendRate(List out, Metric metric) { + final rate = metric.present ? metric.value!.brpm : null; + if (rate != null) out.add(rate); + } + + /// Returns the number of nonempty bin slices actually folded. + int fold(List nn, List times) { + var from = 0, slices = 0; + void feed(int end) { + if (end == from) return; + _open.fold(nn.sublist(from, end), times.sublist(from, end)); + _openCount += end - from; + slices++; + } + for (var i = 0; i < times.length; i++) { + final index = _bin(times[i], _origin ??= times[i]); + if (index == _openIndex) continue; + feed(i); + if (_openCount >= 60) _closed.add((count: _openCount, stream: _open)); + _open = RsaRespStream(); + _openCount = 0; + _openIndex = index; + from = i; + } + feed(nn.length); + return slices; + } + + List snapshot(List tailNn, List tailTimes, + {required double artifactFraction}) { + if (_origin == null && tailNn.isEmpty) return []; + final origin = _origin ?? tailTimes.first; + final out = []; + for (final bin in _closed) { + _appendRate(out, bin.stream.snapshot(artifactFraction: artifactFraction)); + } + var i = 0; + while (i < tailNn.length && _bin(tailTimes[i], origin) == _openIndex) { + i++; + } + if (_openCount + i >= 60) { + _appendRate(out, _open.snapshot(tailNnMs: tailNn.sublist(0, i), + tailNnTimesMs: tailTimes.sublist(0, i), artifactFraction: artifactFraction)); + } + while (i < tailNn.length) { + final index = _bin(tailTimes[i], origin); + var end = i + 1; + while (end < tailNn.length && _bin(tailTimes[end], origin) == index) { + end++; + } + if (end - i >= 60) { + _appendRate(out, rsaRespRate(tailNn.sublist(i, end), + tailTimes.sublist(i, end), artifactFraction: artifactFraction)); + } + i = end; + } + return out; + } +} diff --git a/lib/compute/night_rr_state.dart b/lib/compute/night_rr_state.dart new file mode 100644 index 000000000..a7ec97aee --- /dev/null +++ b/lib/compute/night_rr_state.dart @@ -0,0 +1,400 @@ +import 'dart:collection'; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/onehz.dart' as ana; + +import 'night_prsa.dart'; +import 'night_resp_bins.dart'; + +typedef NightRr = ({List nn, List nnTimesMs, + double cleanFraction, int droppedCount, int correctedCount}); +typedef NightListProof = ({int beats, int times, (int, int) nn, (int, int) ts}); + +NightRr nightRrBatch(List rr, List times) { + final c = ana.correctRr(rr, rrTsMs: times); + return (nn: c.nn, nnTimesMs: c.nnTimesMs, cleanFraction: c.cleanFraction, + droppedCount: c.droppedCount, correctedCount: c.correctedCount); +} + +/// One RAM night, copied into a worker and returned with its result. Raw +/// prefix copies prove append-only input; settled output is never re-corrected. +class NightRrState { + NightRrState(); + static const maxRetainedBeats = 200000; + ana.RrCorrector? _corrector; + final List _rr = [], _times = [], _nn = [], _nnTimes = []; + (int, int, int)? _context; + /// The sleep window (onset, offset, epoch seconds) this state was last keyed + /// on; set by the engine after [context]. Lets a later pass re-derive the + /// revision [context] would be given and ask [hasContext] before trusting RAM. + (int, int)? window; + (String, int)? _night; + final _memos = {}; + final _digests = _PassListDigests(); + List _tailNn = const [], _tailTimes = const []; + ana.HrvFreqStream? _frequency; + ana.RsaRespStream? _rsa; + ana.CvhrStream? _cvhr; + NightPrsa? _prsa; + NightRespBins? _rsaBins; + final Map _fed = {}, _processed = {}, _slices = {}, _restarts = {}; + int processedNightBeats = 0, nightRebuilds = 0; + + bool get canStream => _corrector != null; + // Settled beats represented by each stream, including compacted summaries. + int retainedStreamBeats(String key) => _fed[key] ?? 0; + int processedBeats(String key) => _processed[key] ?? 0; + int slicesProcessed(String key) => _slices[key] ?? 0; + int streamRestarts(String key) => _restarts[key] ?? 0; + int get retainedNightBeats => _rr.length; + int get settledNightBeats => _nn.length; + int get identityMemoEntries => _digests.entries; + int get fullListDigests => _digests.computed; + int get maximumListDigestsPerIdentity => _digests.maximumPerIdentity; + + // A cleared owner still records its algorithm. A transition must rebuild; + // only a fresh owner or a same-algorithm restart may adopt disk state. + bool acceptsCheckpointAlgorithm(int algorithm) => + _context == null || _context!.$3 == algorithm; + + bool acceptsPrefix(List rr, List times, String day, int onset) { + if (_night != (day, onset) || rr.length < _rr.length || + rr.length != times.length) { + return false; + } + for (var i = 0; i < _rr.length; i++) { + if (!identical(rr[i], _rr[i]) || !identical(times[i], _times[i])) { + return false; + } + } + return true; + } + + bool acceptsSearchPrefix(List rr, List times, String day) { + if (_night?.$1 != day || rr.length != times.length || _times.isEmpty) { + return false; + } + var start = 0; + while (start < times.length && times[start] < _times.first) { start++; } + if (start + _rr.length > rr.length) return false; + for (var i = 0; i < _rr.length; i++) { + if (!identical(rr[start + i], _rr[i]) || !identical(times[start + i], _times[i])) { + return false; + } + } + return true; + } + + void bindRestoredContext(int revision, int generation, int algorithm, + {NightRrState? previous}) { + _context = (revision, generation, algorithm); + // Disk state carries no work counters. Adopting it after clearing a RAM + // owner must preserve that owner's cumulative observations. + if (previous == null) return; + processedNightBeats = previous.processedNightBeats; + nightRebuilds = previous.nightRebuilds; + _processed.addAll(previous._processed); + _slices.addAll(previous._slices); + _restarts.addAll(previous._restarts); + _digests.computed = previous._digests.computed; + _digests.maximumPerIdentity = previous._digests.maximumPerIdentity; + } + + /// Worker only. A checkpoint stops before the closed input boundary. Cutting + /// a live tail requires a separate fold; it must never rewind the live owner. + Map? checkpoint(int boundary) { + if (_corrector == null || _night == null) return null; + // The edge checkpoint requires ordered NN. A reset that regresses cannot + // restore its RSA/CVHR state; leave this disposable checkpoint cold. + // ↳ CLAIM(merged-clock-refusal) + if (ana.hasRegressingBeatTimes([..._nnTimes, ..._tailTimes])) return null; + var n = 0; + while (n < _times.length && _times[n] < boundary * 1000) { n++; } + if (n == 0) return null; + final NightRrState state; + if (n == _rr.length) { + state = this; + } else { + state = NightRrState(); + state.correct(_rr.sublist(0, n), _times.sublist(0, n), + day: _night!.$1, onsetSec: _night!.$2); + if (_frequency != null) { + final stream = state._frequency = ana.HrvFreqStream(); + state._feed('frequency', (nn, ts) { stream.fold(nn, ts); return 1; }); + } + if (_rsa != null) { + final stream = state._rsa = ana.RsaRespStream(); + state._feed('rsa', (nn, ts) { stream.fold(nn, ts); return 1; }); + } + if (_cvhr != null) { + final stream = state._cvhr = ana.CvhrStream(); + state._feed('cvhr', (nn, ts) { stream.fold(nn, ts); return 1; }); + } + if (_prsa != null) { + final stream = state._prsa = NightPrsa(); + state._feed('prsa', (nn, ts) { stream.fold(nn); return 1; }); + } + if (_rsaBins != null) { + final stream = state._rsaBins = NightRespBins(); + state._feed('rsa_bins', stream.fold); + } + } + return { + 'v': 1, 'tag': {'day': state._night!.$1, 'onset': state._night!.$2, + 'count': state._rr.length}, + 'rr': state._rr, 'times': state._times, + 'corrector': state._corrector!.toJson(), + 'nn': state._nn, 'nnTimes': state._nnTimes, 'fed': state._fed, + 'frequency': state._frequency?.toJson(), 'rsa': state._rsa?.toJson(), + 'cvhr': state._cvhr?.toJson(), 'prsa': state._prsa?.toJson(), + 'rsa_bins': state._rsaBins?.toJson(), + }; + } + + factory NightRrState.fromCheckpoint(Map json, int boundary) { + List doubles(Object? v) { + final list = (v as List).map((x) => (x as num).toDouble()).toList(); + if (list.length > maxRetainedBeats || list.any((x) => !x.isFinite)) { + throw const FormatException('night column'); + } + return list; + } + final tag = json['tag'] as Map; + final rr = doubles(json['rr']), times = doubles(json['times']); + final nn = doubles(json['nn']), nnTimes = doubles(json['nnTimes']); + final correctorJson = (json['corrector'] as Map).cast(); + final count = correctorJson['maxArtifactRun'] == null + ? correctorJson['n'] : (correctorJson['capped'] as Map)['n']; + if (json['v'] != 1 || tag['day'] is! String || tag['onset'] is! int || + rr.isEmpty || rr.length != times.length || rr.length != tag['count'] || + count != rr.length || nn.length != nnTimes.length || nn.length > rr.length || + times.any((t) => t >= boundary * 1000) || + correctorJson['maxArtifactRun'] != 32 || correctorJson['alpha'] != 5.2 || + correctorJson['windowBeats'] != 91 || correctorJson['minThresholdMs'] != 100 || + correctorJson['reanchorGapMs'] != 1000) { + throw const FormatException('night shape/count'); + } + for (var i = 1; i < times.length; i++) { + if (times[i] < times[i - 1]) throw const FormatException('night times'); + } + for (var i = 1; i < nnTimes.length; i++) { + if (nnTimes[i] < nnTimes[i - 1]) throw const FormatException('NN times'); + } + final state = NightRrState() + .._night = (tag['day'] as String, tag['onset'] as int) + .._corrector = ana.RrCorrector.fromJson(correctorJson) + .._rr.addAll(rr).._times.addAll(times).._nn.addAll(nn).._nnTimes.addAll(nnTimes); + final fed = (json['fed'] as Map).cast(); + const keys = ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']; + if (fed.keys.any((k) => !keys.contains(k))) { + throw const FormatException('night stream key'); + } + for (final key in keys) { + final data = json[key]; + if (data == null) { + if (fed.containsKey(key)) throw const FormatException('missing stream'); + continue; + } + if (fed[key] != nn.length) throw const FormatException('stream feed count'); + final map = (data as Map).cast(); + if (key == 'prsa') { + state._prsa = NightPrsa.fromJson(map, fed[key]!); + } else if (key == 'rsa_bins') { + state._rsaBins = NightRespBins.fromJson(map, fed[key]!); + } else { + if (map['n'] != fed[key]) throw const FormatException('stream count'); + if (key == 'frequency' && map['oversample'] != 4.0) { + throw const FormatException('frequency config'); + } + if (key == 'cvhr') { + final defaults = ana.CvhrStream().toJson(); + for (final name in ['minWidthSec', 'maxWidthSec', 'depthWidthRatioMin', + 'envWindowSec', 'maxGapSec']) { + if (map[name] != defaults[name]) throw const FormatException('CVHR config'); + } + } + if (key == 'frequency') state._frequency = ana.HrvFreqStream.fromJson(map); + if (key == 'rsa') state._rsa = ana.RsaRespStream.fromJson(map); + if (key == 'cvhr') state._cvhr = ana.CvhrStream.fromJson(map); + } + } + state._fed.addAll(fed); + return state; + } + + void beginPass() => _digests.clear(); + void endPass() => _digests.clear(); + + bool hasContext(int inputRevision, int generation, int algorithm) => + _context == (inputRevision, generation, algorithm); + + void context({required int inputRevision, required int generation, + required int algorithm}) { + final next = (inputRevision, generation, algorithm); + if (_context != next) clear(); + _context = next; + } + + void clear() { + _resetNight(); + _digests.clear(); + } + + void _resetNight() { + _corrector = null; + _rr.clear(); _times.clear(); _nn.clear(); _nnTimes.clear(); + _night = null; + _memos.clear(); + _tailNn = const []; _tailTimes = const []; + _frequency = null; _rsa = null; _cvhr = null; + _prsa = null; _rsaBins = null; + _fed.clear(); + } + + /// Shared by the night metric memos. No digest outlives this pass's + /// identities; only its integer words may be kept as a dependency proof. + NightListProof dependencyProof(List nn, List times) => + (beats: nn.length, times: times.length, + nn: _digests.digest(nn), ts: _digests.digest(times)); + + NightRr correct(List rr, List times, + {required String day, required int onsetSec}) { + final night = (day, onsetSec); + var held = _rr.length; + var extendsPrefix = _night == night && rr.length >= held && + times.length == rr.length; + for (var i = 0; extendsPrefix && i < held; i++) { + extendsPrefix = identical(rr[i], _rr[i]) && identical(times[i], _times[i]); + } + if (!extendsPrefix) { + _resetNight(); + held = 0; + } + // Keep malformed/oversized input on the unchanged batch path. Eviction + // affects work only, never absence rules or numeric output. + if (rr.isEmpty || times.length != rr.length || + rr.length > maxRetainedBeats) { + _resetNight(); + processedNightBeats += rr.length; + return nightRrBatch(rr, times); + } + _night = night; + if (_corrector == null) nightRebuilds++; + final corrector = _corrector ??= ana.RrCorrector(); + if (rr.length > held) { + final addRr = held == 0 ? rr : rr.sublist(held); + final addTimes = held == 0 ? times : times.sublist(held); + final ana.RrSettled settled; + try { + settled = corrector.fold(addRr, tsMs: addTimes); + } on ArgumentError { + _resetNight(); + processedNightBeats += rr.length; + return nightRrBatch(rr, times); + } + processedNightBeats += addRr.length; + _rr.addAll(addRr); _times.addAll(addTimes); + _nn.addAll(settled.nn); _nnTimes.addAll(settled.nnTimes); + } + final tail = corrector.snapshot(); + _tailNn = tail.tailNn; _tailTimes = tail.tailNnTimes; + return (nn: [..._nn, ...tail.tailNn], + nnTimesMs: [..._nnTimes, ...tail.tailNnTimes], + cleanFraction: tail.cleanFraction, droppedCount: tail.droppedCount, + correctedCount: tail.correctedCount); + } + + /// Exact result reuse. Growing lists snapshot incremental metric streams; + /// unchanged lists also avoid re-evaluating their provisional tails. + T memo(String key, Object dependencies, T Function() compute) { + if (_corrector == null) return compute(); + final old = _memos[key]; + if (old != null && old.dependencies == dependencies) return old.value as T; + final result = compute(); + if (!_memos.containsKey(key) && _memos.length >= 8) { + _memos.remove(_memos.keys.first); + } + _memos[key] = (dependencies: dependencies, value: result); + return result; + } + + void _feed(String key, int Function(List, List) fold) { + final held = _fed[key] ?? 0; + if (!_fed.containsKey(key)) { + _restarts[key] = streamRestarts(key) + 1; + _fed[key] = 0; + } + if (held == _nn.length) return; + final slices = fold(_nn.sublist(held), _nnTimes.sublist(held)); + _processed[key] = processedBeats(key) + _nn.length - held; + _slices[key] = slicesProcessed(key) + slices; + _fed[key] = _nn.length; + } + + ana.Metric nightHrvFreq(double artifactFraction) { + final stream = _frequency ??= ana.HrvFreqStream(); + _feed('frequency', (nn, times) { stream.fold(nn, times); return 1; }); + return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, + artifactFraction: artifactFraction); + } + + ana.Metric nightResp(double artifactFraction) { + final stream = _rsa ??= ana.RsaRespStream(); + _feed('rsa', (nn, times) { stream.fold(nn, times); return 1; }); + return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, + artifactFraction: artifactFraction); + } + + ana.Metric nightCvhr(double artifactFraction) { + final stream = _cvhr ??= ana.CvhrStream(); + _feed('cvhr', (nn, times) { stream.fold(nn, times); return 1; }); + // The merged stream handles a fresh, entirely provisional night too. + return stream.snapshot(tailNnMs: _tailNn, tailNnTimesMs: _tailTimes, + artifactFraction: artifactFraction); + } + + (ana.Metric, ana.Metric) nightPrsa() { + final stream = _prsa ??= NightPrsa(); + _feed('prsa', (nn, times) { stream.fold(nn); return 1; }); + return stream.snapshot(_tailNn); + } + + List nightRespWindows(double artifactFraction) { + final stream = _rsaBins ??= NightRespBins(); + _feed('rsa_bins', stream.fold); + return stream.snapshot(_tailNn, _tailTimes, artifactFraction: artifactFraction); + } +} + +/// Short window lists are walked directly. Only long shared lists occupy +/// identity keys, and the pipeline releases all keys in finally. +class _PassListDigests { + final HashMap, (int, int, int, int)> _byList = HashMap.identity(); + final _bits = ByteData(8); + int computed = 0, maximumPerIdentity = 0; + int get entries => _byList.length; + void clear() => _byList.clear(); + + (int, int) digest(List list) { + final hit = _byList[list]; + if (hit != null && hit.$1 == list.length) return (hit.$2, hit.$3); + var a = 0x243f6a8885a308d3, b = 0x13198a2e03707344; + for (final value in list) { + _bits.setFloat64(0, value); + final word = _bits.getInt64(0); + a = _mix(a + word + 0x9e3779b97f4a7c15); + b = _mix((b ^ word) * 0xd6e8feb86659fd93 + 0x632be59bd9b4e019); + } + computed++; + final count = (hit?.$4 ?? 0) + 1; + if (count > maximumPerIdentity) maximumPerIdentity = count; + if (list.length >= 2048) _byList[list] = (list.length, a, b, count); + return (a, b); + } + + static int _mix(int z) { + z = (z ^ (z >>> 30)) * 0xbf58476d1ce4e5b9; + z = (z ^ (z >>> 27)) * 0x94d049bb133111eb; + return z ^ (z >>> 31); + } +} diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 989a0ae93..71d8545b7 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -33,6 +33,8 @@ import 'package:openstrap_analytics/onehz.dart'; // instead of two that can drift. import 'inputs/garmin_inputs.dart' show isWearableFamily, kHrCadenceSec, kOneHzRunGapSec; +import 'ed2b_stream_diagnostics.dart'; +import 'night_rr_state.dart'; import 'hr_max.dart' show estimatedMaxHr, @@ -646,7 +648,25 @@ class DayBundleInput { /// Pure: takes the serialized [DayBundleInput] map, returns a plain JSON map (the /// full derived bundle). Call directly + synchronously in tests, or via /// `Isolate.run(() => deriveDayBundle(input))` in production. -Map deriveDayBundle(Map inputJson) { +Map deriveDayBundle(Map inputJson, { + Ed2bStreamDiagnostics? streamDiagnostics, + NightRrState? nightState, +}) { + final state = streamDiagnostics?.disabled == true + ? null : nightState ?? streamDiagnostics?.nightState; + streamDiagnostics?.beginPass(); + state?.beginPass(); + try { + final bundle = _deriveDayBundle(inputJson, state, streamDiagnostics); + streamDiagnostics?.endPass(); + return bundle; + } finally { + state?.endPass(); + } +} + +Map _deriveDayBundle(Map inputJson, + NightRrState? nightState, Ed2bStreamDiagnostics? streamDiagnostics) { var d = DayBundleInput.fromJson(inputJson); // EXPERIMENTAL HMM re-staging (default OFF — PR #628 review): when the @@ -704,10 +724,26 @@ Map deriveDayBundle(Map inputJson) { // HRV/RHR are rest/sleep-only per the catalog. Running correctRr+hrvTime over // the SLEEP RR (not the whole day) is what brings RMSSD back to physiological // tens-of-ms instead of the whole-day ~166 ms inflated value. - final corrected = correctRr(d.sleepRrMs, rrTsMs: d.sleepRrTsMs); + final corrected = nightState == null + ? nightRrBatch(d.sleepRrMs, d.sleepRrTsMs) + : nightState.correct(d.sleepRrMs, d.sleepRrTsMs, + day: d.date, onsetSec: d.sleepOnsetSec); + streamDiagnostics?.observeNightCorrection(d.sleepRrMs.length, corrected.nn, + corrected.nnTimesMs, corrected.cleanFraction); final nn = corrected.nn; final nnTimes = corrected.nnTimesMs; final artifactFraction = (1.0 - corrected.cleanFraction).clamp(0.0, 1.0); + final streams = nightState?.canStream == true ? nightState : null; + // CLAIM(merged-clock-refusal): capped resets preserve source time and can + // regress. Batch RSA/CVHR abstain; their streams require ordered clocks. + // Keep frequency's separate omission policy [ed2b_merged_analytics_test]. + final orderedStreams = hasRegressingBeatTimes(nnTimes) ? null : streams; + final nnProof = nightState?.dependencyProof(nn, nnTimes); + T nightMemo(String key, Object? extra, T Function() compute) => + nnProof == null ? compute() : + nightState!.memo(key, (nnProof, extra), compute); + Map nightJson(Map json) => + nightState == null ? json : _copyNightJson(json); // Σ RR ÷ wall span of the raw sleep RR: above 1.10 the stream holds more // beat-time than elapsed (duplicated or interleaved beats), and every RMSSD // below refuses it. Null under 10 min of span. @@ -766,7 +802,11 @@ Map deriveDayBundle(Map inputJson) { final sleepSessionRmssd = sleepSessionRmssdMetric.present ? sessionDetail!.rmssd : null; final hrvF = nn.length >= 20 - ? hrvFreq(nn, nnTimes, artifactFraction: artifactFraction) + ? nightMemo('frequency', artifactFraction, () { + if (streams != null) return streams.nightHrvFreq(artifactFraction); + streamDiagnostics?.observeBatch('frequency', nn.length); + return hrvFreq(nn, nnTimes, artifactFraction: artifactFraction); + }) : Metric.absent( tier: Tier.high, inputs_used: const ['rr_cleaned'], @@ -819,8 +859,13 @@ Map deriveDayBundle(Map inputJson) { // HR dip: day-side = waking HR outside the sleep window; night-side = sleep HR. final dayOnly = _dayHrOutsideSleep(d); final dip = hrDip(dayOnly, sleepHr); - final dc = decelerationCapacity(nn); - final ac = accelerationCapacity(nn); + final prsa = nightMemo('prsa', null, () { + if (streams != null) return streams.nightPrsa(); + streamDiagnostics?.observeBatch('prsa', nn.length); + return (decelerationCapacity(nn), accelerationCapacity(nn)); + }); + final dc = prsa.$1; + final ac = prsa.$2; // Baevsky Stress Index over the sleep NN — resting autonomic tension (a // transparent RR-histogram metric; no ML). Daily resting-stress indicator. // nnTimesMs lets it segment at a charging/off-wrist gap instead of letting @@ -830,14 +875,22 @@ Map deriveDayBundle(Map inputJson) { // ── RESPIRATION (sleep-windowed) ─────────────────────────────────────────── final resp = nn.length >= 30 - ? rsaRespRate(nn, nnTimes, artifactFraction: artifactFraction) + ? nightMemo('rsa', artifactFraction, () { + if (orderedStreams != null) return orderedStreams.nightResp(artifactFraction); + streamDiagnostics?.observeBatch('rsa', nn.length); + return rsaRespRate(nn, nnTimes, artifactFraction: artifactFraction); + }) : Metric.absent( tier: Tier.estimate, inputs_used: const ['rr_cleaned'], note: needInputNote('nn_beats', have: nn.length, need: 30), ); final cvhr = nn.length >= 60 - ? cvhrApneaScreen(nn, nnTimes, artifactFraction: artifactFraction) + ? nightMemo('cvhr', artifactFraction, () { + if (orderedStreams != null) return orderedStreams.nightCvhr(artifactFraction); + streamDiagnostics?.observeBatch('cvhr', nn.length); + return cvhrApneaScreen(nn, nnTimes, artifactFraction: artifactFraction); + }) : Metric.absent( tier: Tier.estimate, inputs_used: const ['rr_cleaned'], @@ -862,8 +915,13 @@ Map deriveDayBundle(Map inputJson) { // ── BREATHING-RATE VARIABILITY (per-window RSA over the sleep NN) ────────── // Window the cleaned sleep NN into ~30-min bins, take each bin's RSA resp rate, // then BRV = dispersion + Theil-Sen trend of those per-window rates. - final respWindows = - respPerWindow(nn, nnTimes, artifactFraction: artifactFraction); + final cachedRespWindows = nightMemo('rsa_bins', artifactFraction, () => + orderedStreams != null ? orderedStreams.nightRespWindows(artifactFraction) : + respPerWindow(nn, nnTimes, artifactFraction: artifactFraction, + diagnostics: streamDiagnostics)); + final respWindows = nightState == null + ? cachedRespWindows : cachedRespWindows.toList(); + streamDiagnostics?.onRespWindows?.call(respWindows); final brv = respWindows.length >= 3 ? breathingRateVariability(respWindows) : Metric.absent( @@ -1431,11 +1489,11 @@ Map deriveDayBundle(Map inputJson) { 'min_diffs_per_window': sessionDetail?.minDiffsPerWindow, }, 'rmssd_nocturnal': robustRmssd.toJson(), - 'hrv_freq': hrvF.toJson((v) => v.toJson()), + 'hrv_freq': nightJson(hrvF.toJson((v) => v.toJson())), 'resting_hr': rhr.toJson((v) => v.toJson()), 'hr_dip': dip.toJson((v) => v.toJson()), - 'prsa_dc': dc.toJson((v) => v.toJson()), - 'prsa_ac': ac.toJson((v) => v.toJson()), + 'prsa_dc': nightJson(dc.toJson((v) => v.toJson())), + 'prsa_ac': nightJson(ac.toJson((v) => v.toJson())), 'readiness_lnrmssd': lnReadiness.toJson((v) => v.toJson()), 'readiness_composite': { ...composite.toJson((v) => v.toJson()), @@ -1543,8 +1601,8 @@ Map deriveDayBundle(Map inputJson) { }; final respiration = { - 'rsa': resp.toJson((v) => v.toJson()), - 'cvhr_apnea': cvhr.toJson((v) => v.toJson()), + 'rsa': nightJson(resp.toJson((v) => v.toJson())), + 'cvhr_apnea': nightJson(cvhr.toJson((v) => v.toJson())), 'odi': odi.toJson((v) => v.toJson()), }; @@ -2074,6 +2132,7 @@ List respPerWindow( List nn, List nnTimes, { required double artifactFraction, + Ed2bStreamDiagnostics? diagnostics, double windowMs = 1800000.0, int minBeats = 60, }) { @@ -2093,12 +2152,24 @@ List respPerWindow( if (segNn.length < minBeats) continue; final r = rsaRespRate(segNn, binsTs[idx]!, artifactFraction: artifactFraction); + diagnostics?.observeBatch('rsa_bins', segNn.length); final b = r.present ? r.value!.brpm : null; if (b != null) out.add(b); } return out; } +// Metric serializers can expose lists owned by the cached value or envelope. +// Detach these small JSON blocks before returning them to a caller. +Map _copyNightJson(Map json) { + Object? copy(Object? value) { + if (value is Map) return value.map((k, v) => MapEntry(k, copy(v))); + if (value is List) return [for (final v in value) copy(v)]; + return value; + } + return json.map((k, v) => MapEntry(k, copy(v))); +} + /// Wrap a value sub-map in the {value,confidence,tier,inputs_used} envelope the /// serve seam reads via `.value`. Null inner → honest "—". Map _envelope( diff --git a/lib/compute/staged_sleep_cache.dart b/lib/compute/staged_sleep_cache.dart index 30fe3d638..67174684f 100644 --- a/lib/compute/staged_sleep_cache.dart +++ b/lib/compute/staged_sleep_cache.dart @@ -76,6 +76,7 @@ String stagingInputKey({ required List<({int startSec, int endSec, String dayKey})> priorSleep, required List vendorNights, required (String, String)? activeWearable, + bool prefixContext = false, }) { // Count local midnights, including DST days, rather than dividing by 86400. var walked = 0; @@ -114,12 +115,13 @@ String stagingInputKey({ } final now = DateTime.now(); return jsonEncode([ - fromSec, toSec, revisions, + fromSec, toSec, prefixContext ? const [] : revisions, [for (final sig in InputSignal.values) [sig.name, [for (final o in ownership[sig] ?? const []) [o.start, o.end, o.deviceId]], priority[sig]]], sub.deviceFamily, (sub.deviceIds.toList()..sort()), - sub.length, sub.firstTs, sub.lastTs, sub.rrMs.length, + if (prefixContext) ...[null, null, null, null] + else ...[sub.length, sub.firstTs, sub.lastTs, sub.rrMs.length], now.timeZoneName, now.timeZoneOffset.inSeconds, zoneOffsets, override == null ? null : [override.dayId, override.onsetSec, override.offsetSec, override.source], diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index 08b534fe9..8e1403f06 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -1220,8 +1220,8 @@ class SleepWindowOverride { String localDateLabel(int epochSec) => dayLabelOf(DateTime.fromMillisecondsSinceEpoch(epochSec * 1000)); -// RED controls: the walk still segments every day. The counter observes its -// actual body; GREEN must preserve the full walk for folding callers. +// Walk controls: the counter observes the actual segmentation body; the full +// walk remains available to check the restricted walk's target output. int debugSegmentedDayCount = 0; bool debugStageEveryWalkedDay = false; @@ -1257,8 +1257,8 @@ bool debugStageEveryWalkedDay = false; /// /// With [skipUnusedDays], only [targetDay] is needed. Earlier days can be /// skipped when even one new history key per day cannot reach the prior's -/// 14-night threshold. Callers needing profile observations must keep the -/// full walk: its longest observation can belong to an earlier day. +/// 14-night threshold. Callers needing profile observations can use +/// [recordTargetOnly] to record the target night even during a full walk. List calendarDays( Substrate sub, { SleepWindowOverride? override, @@ -1267,6 +1267,10 @@ List calendarDays( List vendorNights = const [], String? targetDay, bool skipUnusedDays = false, + ana.CardioRowCache? rowCache, + // D36: other walked nights must not train under the target's ID. + // Restore the caller's recording flag on every exit. + bool recordTargetOnly = false, }) { final tzOffset = tzOffsetAt ?? tzOffsetSecondsAt; if (sub.isEmpty) return const []; @@ -1294,6 +1298,9 @@ List calendarDays( } skipBefore = {for (final h in sleepHistory) h.dayKey}.length + earlier < 14; } + final recording = ana.cardioRecordObservations; + final onlyTarget = recordTargetOnly && targetDay != null; + try { var guard = 0; while (dayStart < dataEnd && guard++ < 400) { final dayEnd = _nextLocalMidnight(dayStart); @@ -1314,6 +1321,7 @@ List calendarDays( } } debugSegmentedDayCount++; + if (onlyTarget) ana.cardioRecordObservations = recording && dayLabel == targetDay; // The main sleep that ENDS in this calendar day: search from the previous // local noon through this local midnight, then let the sleep selector pick // the overnight main block from any naps / split fragments it sees. The old @@ -1386,6 +1394,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, forcedWindow: (onsetSec: ov.onsetSec, offsetSec: ov.offsetSec), @@ -1409,6 +1418,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, habitualMidsleepSec: habitualMidsleepSec, @@ -1442,6 +1452,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, forcedWindow: @@ -1476,6 +1487,7 @@ List calendarDays( accelSlice, hrSlice, hrBaseline: hrBaseline, + rowCache: rowCache, rrMs: rrMsSeg, rrTsMs: rrTsSeg, forcedWindow: (onsetSec: vn.onsetSec, offsetSec: vn.offsetSec), @@ -1581,6 +1593,9 @@ List calendarDays( )); dayStart = dayEnd; } + } finally { + ana.cardioRecordObservations = recording; + } return days; } diff --git a/lib/compute/substrate_retention.dart b/lib/compute/substrate_retention.dart index af77a2b6a..2cfdd50fe 100644 --- a/lib/compute/substrate_retention.dart +++ b/lib/compute/substrate_retention.dart @@ -8,7 +8,8 @@ import 'substrate.dart'; class RetainedSubstrate { RetainedSubstrate({required this.version, required this.context, required this.revisions, required this.sub, required this.extras, - required this.cut}); + required this.cut, this.from, this.to, this.generation, + this.ownership = const {}}); int version; final String context; @@ -16,6 +17,8 @@ class RetainedSubstrate { final Substrate sub; final Map extras; final int cut; + final int? from, to, generation; + final Map> ownership; /// A changed closed bucket requires a cold load. Otherwise rebuild the tail /// from [cut], or return the whole load when every bucket is unchanged. @@ -35,7 +38,7 @@ class RetainedSubstrate { }; } -/// At most three loads per day: search, buffered day and selected sleep. +/// At most three loads per day: separate windows, or their shared union. /// Both days and loads are ordered least recently used first. class SubstrateRetention { final _days = >{}; @@ -49,6 +52,12 @@ class SubstrateRetention { } } Set get days => _days.keys.toSet(); + Iterable get unions sync* { + for (final loads in _days.values) { + final union = loads['union']; + if (union != null) yield union; + } + } RetainedSubstrate? lookup(String day, String load) { final loads = _days.remove(day); diff --git a/lib/compute/substrate_union.dart b/lib/compute/substrate_union.dart new file mode 100644 index 000000000..c52be1148 --- /dev/null +++ b/lib/compute/substrate_union.dart @@ -0,0 +1,271 @@ +import 'dart:typed_data'; + +import '../ble/adapters/signals.dart'; +import '../data/coverage_resolver.dart'; +import 'substrate.dart'; +import 'substrate_retention.dart'; + +typedef SubstrateUnionWindow = ({int from, int to, + Map> ownership}); + +/// Compare the loader's decisions, including null-owner (unfiltered) spans. +/// Wider resolver windows can seed hysteresis differently. +bool sameSubstrateOwnership(Map> a, + Map> b, int from, int to) { + List clip(List spans) { + final out = []; + for (final s in spans) { + if (s.deviceId == null || s.end <= from || s.start > to) continue; + final lo = s.start < from ? from : s.start; + final hi = s.end > to + 1 ? to + 1 : s.end; + if (out.isNotEmpty && out.last.end == lo && + out.last.deviceId == s.deviceId) { + out[out.length - 1] = + (start: out.last.start, end: hi, deviceId: s.deviceId); + } else { + out.add((start: lo, end: hi, deviceId: s.deviceId)); + } + } + return out; + } + for (final signal in InputSignal.values) { + final x = clip(a[signal] ?? const []), y = clip(b[signal] ?? const []); + if (x.length != y.length) return false; + for (var i = 0; i < x.length; i++) { + if (x[i] != y[i]) return false; + } + } + return true; +} + +int _lowerBound(List ts, int value) { + var lo = 0, hi = ts.length; + while (lo < hi) { + final mid = (lo + hi) >> 1; + if (ts[mid] < value) { lo = mid + 1; } else { hi = mid; } + } + return lo; +} + +List> _runs(RetainedSubstrate load) => + (load.extras['provenance'] as List).cast>(); +String? _unit(List> runs) { + for (final r in runs) { + if (r[3] != null) return r[3] as String; + } + return null; +} +(Set, Set) _sources(List> runs) { + final families = {}, ids = {}; + for (final r in runs) { + if (r[1] != null) families.add(r[1] as String); + if (r[2] != null) ids.add(r[2] as String); + } + return (families, ids); +} + +/// Worker-only: reproduce a decoded load by record second, restarting the +/// beat running maximum from the original placements. Substrate.slice instead +/// selects beats by their repaired time and keeps the parent's provenance. +RetainedSubstrate? cutSubstrateUnion(RetainedSubstrate load, int from, int to, + Map> ownership, {bool retainMetadata = true}) { + if (load.from == null || from < load.from! || to > load.to! || to < from || + load.extras['provenance'] == null || + !sameSubstrateOwnership(load.ownership, ownership, from, to)) { + return null; + } + final s = load.sub, n = s.length; + final starts = load.extras['rr_start'] as List; + final raw = load.extras['rr_ts_raw'] as List; + if (starts.length != n || raw.length != s.rrMs.length) return null; + final lo = _lowerBound(s.tsSec, from), hi = _lowerBound(s.tsSec, to + 1); + final r0 = lo < n ? starts[lo] : s.rrMs.length; + final r1 = hi < n ? starts[hi] : s.rrMs.length; + final runs = _runs(load), selected = >[]; + for (var i = 0; i < runs.length; i++) { + final start = runs[i][0] as int; + final end = i + 1 < runs.length ? runs[i + 1][0] as int : n; + if (end <= lo || start >= hi) continue; + selected.add([start < lo ? 0 : start - lo, ...runs[i].skip(1)]); + } + // The union has already suppressed other temperature units. A later + // window that would choose another unit must take the ordinary loader. + final unit = _unit(selected); + if (unit != null && unit != _unit(runs)) return null; + final (families, ids) = _sources(selected); + final axis = Float64List(r1 - r0)..setRange(0, r1 - r0, raw, r0); + monotonizeBeatAxis(axis); + final columns = s.toJson(); + final out = { + for (final key in _intColumns) + key: (columns[key] as List).sublist(lo, hi), + for (final key in _motionColumns) + key: (columns[key] as List).sublist(lo, hi), + 'rr_ts_ms': axis, 'rr_ms': s.rrMs.sublist(r0, r1), + 'device_family': families.length == 1 ? families.first : null, + 'device_ids': ids.toList(), + }; + return RetainedSubstrate(version: load.version, context: load.context, + revisions: load.revisions, from: from, to: to, + generation: load.generation, ownership: ownership, + sub: Substrate.fromTransfer(out), cut: from, + extras: !retainMetadata ? const {} : { + 'rr_start': Int32List.fromList([ + for (var i = lo; i < hi; i++) starts[i] - r0]), + 'rr_ts_raw': Float64List(r1 - r0)..setRange(0, r1 - r0, raw, r0), + 'provenance': selected, + 'meta': [for (final m in load.extras['meta'] as List) + if ((m as List)[0] as int >= from ~/ 900 && + m[0] as int <= to ~/ 900) m], + 'raw_rows_by_bucket': { + for (final e in ((load.extras['raw_rows_by_bucket'] as Map?) ?? + const {}).entries) + if (e.key >= from ~/ 900 && e.key <= to ~/ 900) e.key: e.value}, + }); +} + +const _intColumns = ['ts_sec', 'hr', 'spo2_red', 'spo2_ir', 'skin_temp', + 'skin_contact', 'step_count', 'hr_valid', 'band_sleep_state']; +const _motionColumns = ['ax', 'ay', 'az']; + +/// Worker-only: join disjoint loads and repair their raw beat axis once across +/// the seams. Refuse independently selected temperature units that cannot be +/// reconstructed from already suppressed columns. +RetainedSubstrate? joinSubstrateUnions(List parts, + String context, Map revisions, + Map> ownership) { + if (parts.isEmpty) return null; + final units = {for (final p in parts) _unit(_runs(p))}..remove(null); + if (units.length > 1) return null; + var n = 0, beats = 0; + for (var i = 0; i < parts.length; i++) { + if (i > 0 && parts[i].from != parts[i - 1].to! + 1) return null; + n += parts[i].sub.length; + beats += parts[i].sub.rrMs.length; + } + final columns = { + for (final key in _intColumns) key: Int64List(n), + for (final key in _motionColumns) key: Float64List(n), + 'rr_ms': Float64List(beats), + }; + final raw = Float64List(beats), starts = Int32List(n); + final runs = >[], meta = >{}; + final rawRows = {}; + var at = 0, rat = 0; + for (final p in parts) { + final source = p.sub.toJson(), pn = p.sub.length, rn = p.sub.rrMs.length; + for (final key in [..._intColumns, ..._motionColumns]) { + (columns[key] as List).setRange(at, at + pn, source[key] as List); + } + (columns['rr_ms'] as List).setRange(rat, rat + rn, p.sub.rrMs); + raw.setRange(rat, rat + rn, p.extras['rr_ts_raw'] as List); + final rs = p.extras['rr_start'] as List; + for (var i = 0; i < pn; i++) { starts[at + i] = rs[i] + rat; } + for (final r in _runs(p)) { + final last = runs.isEmpty ? null : runs.last; + if (last == null || last[1] != r[1] || last[2] != r[2] || last[3] != r[3]) { + runs.add([(r[0] as int) + at, ...r.skip(1)]); + } + } + for (final m in p.extras['meta'] as List) { + final wire = m as List; + final target = meta.putIfAbsent(wire[0] as int, + () => [wire[0], {}, {}, null]); + (target[1] as Set).addAll((wire[1] as List).cast()); + (target[2] as Set).addAll((wire[2] as List).cast()); + target[3] ??= wire[3]; + } + for (final e in (p.extras['raw_rows_by_bucket'] as Map).entries) { + rawRows[e.key] = (rawRows[e.key] ?? 0) + e.value; + } + at += pn; rat += rn; + } + final axis = Float64List.fromList(raw); + monotonizeBeatAxis(axis); + final (families, ids) = _sources(runs); + columns.addAll({'rr_ts_ms': axis, + 'device_family': families.length == 1 ? families.first : null, + 'device_ids': ids.toList()}); + return RetainedSubstrate(version: parts.first.version, context: context, + revisions: revisions, from: parts.first.from, to: parts.last.to, + generation: parts.first.generation, ownership: ownership, + sub: Substrate.fromTransfer(columns), cut: parts.first.from!, + extras: {'rr_start': starts, 'rr_ts_raw': raw, 'provenance': runs, + 'raw_rows_by_bucket': rawRows, + 'meta': [for (final m in meta.values) + [m[0], (m[1] as Set).toList(), (m[2] as Set).toList(), m[3]]], + }); +} + +/// Revision/ownership proofs are small; column cuts and joins run in workers. +List<(int, int, RetainedSubstrate)> reusableUnionStretches( + Iterable bases, int from, int to, + Map revisions, Map> ownership) { + final candidates = <(int, int, RetainedSubstrate)>[]; + for (final base in bases) { + final lo = from > base.from! ? from : base.from!; + final hi = to < base.to! ? to : base.to!; + int? start; + void close(int end) { + if (start == null) return; + final a = start! < lo ? lo : start!; + final b = end > hi ? hi : end; + start = null; + if (b < a || (a != base.from && a % 900 != 0) || + (b != base.to && (b + 1) % 900 != 0)) { return; } + if (sameSubstrateOwnership(base.ownership, ownership, a, b)) { + candidates.add((a, b, base)); + } + } + for (var b = lo ~/ 900; b <= hi ~/ 900; b++) { + if ((revisions[b] ?? 0) == (base.revisions[b] ?? 0)) { + start ??= b * 900; + } else { close(b * 900 - 1); } + } + close(hi); + } + final out = <(int, int, RetainedSubstrate)>[]; + var pos = from; + while (pos <= to) { + (int, int, RetainedSubstrate)? best; + int? next; + for (final c in candidates) { + if (c.$1 <= pos && c.$2 >= pos && + (best == null || c.$2 > best.$2)) { best = c; } + if (c.$1 > pos && (next == null || c.$1 < next)) next = c.$1; + } + if (best == null) { + if (next == null) break; + pos = next; + } else { + if (pos != best.$3.from && pos % 900 != 0) { + pos = (pos ~/ 900 + 1) * 900; + continue; + } + out.add((pos, best.$2, best.$3)); + pos = best.$2 + 1; + } + } + return out; +} + +// Factories keep dispatch closures in a scope containing only plain inputs. +List Function() borrowUnionJob( + List<(int, int, RetainedSubstrate)> stretches, + Map> ownership) => () => [ + for (final s in stretches) cutSubstrateUnion(s.$3, s.$1, s.$2, ownership)]; + +RetainedSubstrate? Function() joinUnionJob(List parts, + String context, Map revisions, + Map> ownership) => () => + joinSubstrateUnions(parts, context, revisions, ownership); + +(List, Substrate?) Function() cutUnionJob(RetainedSubstrate load, + List<(int, int)> ranges, Map> ownership, + int? dayStart, int? dayEnd) => () { + final windows = [for (final r in ranges) + cutSubstrateUnion(load, r.$1, r.$2, ownership, retainMetadata: false)?.sub]; + final day = dayStart == null || dayEnd == null ? null : + windows.first?.slice(dayStart, dayEnd); + return (windows, day); +}; diff --git a/lib/data/db.dart b/lib/data/db.dart index c225345cc..dbf819ae4 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -688,7 +688,7 @@ class LocalDb { /// pass it: sqflite throws `ArgumentError('onCreate must be null if no /// version is specified')` BEFORE opening anything when `onCreate` is given /// without `version` (sqflite_common database_mixin.dart). - static const int schemaVersion = 61; + static const int schemaVersion = 62; /// In-memory cache identity, advanced after open, wipe or committed import. /// It is local to this isolate; no cache survives a process restart. @@ -1540,6 +1540,10 @@ class LocalDb { await _createInputRev(db); await _createSessionCeilingRev(db); } + if (oldV < 62) { + await _createStoreIdentity(db); + await _createDayCheckpoint(db); + } }, onOpen: (db) async { await _repairOpenSchema(db); @@ -1662,12 +1666,68 @@ class LocalDb { // A decoded-table rebuild above drops its triggers; attach them last. await _createInputRev(db); await _createSessionCeilingRev(db); + await _createStoreIdentity(db); + await _createDayCheckpoint(db); // Views LAST — they depend on metric_series / day_result / baselines / sessions // / notifications all existing. DROP+CREATE so a shape change takes effect. await _ensureCoachViews(db); await _dropRawStore(db); } + // Stable across opens, unique to this file's current input history. Neither + // restore nor salvage imports it: input_rev is local and can repeat on reset. + static Future _createStoreIdentity(DatabaseExecutor db) async { + await db.execute('CREATE TABLE IF NOT EXISTS store_identity (' + 'singleton INTEGER PRIMARY KEY CHECK(singleton = 1), id TEXT NOT NULL)'); + await db.execute('INSERT OR IGNORE INTO store_identity(singleton, id) ' + 'VALUES(1, lower(hex(randomblob(16))))'); + } + + static Future storeIdentity({DatabaseExecutor? executor}) async => + (await (executor ?? await instance).rawQuery( + 'SELECT id FROM store_identity WHERE singleton = 1')).single['id'] as String; + + static Future _rotateStoreIdentity(DatabaseExecutor db) async { + await db.execute('INSERT OR REPLACE INTO store_identity(singleton, id) ' + 'VALUES(1, lower(hex(randomblob(16))))'); + await db.delete('day_checkpoint'); + } + + static Future _createDayCheckpoint(Database db) async { + await db.execute('CREATE TABLE IF NOT EXISTS day_checkpoint (' + 'day_id TEXT NOT NULL, algo_version INTEGER NOT NULL, ' + 'fmt INTEGER NOT NULL, ctx_sig TEXT NOT NULL, cp_rec_ts INTEGER NOT NULL, ' + 'rev_vec BLOB NOT NULL, state BLOB NOT NULL, night_ref TEXT, ' + 'computed_at INTEGER NOT NULL, PRIMARY KEY(day_id, algo_version))'); + } + + /// Chunk the disposable blob like substrate_archive: a whole night can + /// exceed Android's cursor window. A racing replacement fails codec checks. + static Future?> dayCheckpoint( + String day, int algorithm, {required int maxBytes}) async { + final db = await instance; + const where = 'day_id = ? AND algo_version = ?'; + final rows = await db.rawQuery('SELECT day_id, algo_version, fmt, ctx_sig, ' + 'cp_rec_ts, rev_vec, night_ref, computed_at, length(state) AS state_length ' + 'FROM day_checkpoint WHERE $where', [day, algorithm]); + if (rows.isEmpty) return null; + final row = Map.of(rows.single); + final length = row.remove('state_length') as int; + final bytes = Uint8List(length <= maxBytes ? length : 0); + row['state'] = bytes; + if (length > maxBytes) return row; + for (var off = 0; off < length; off += 128 * 1024) { + final parts = await db.rawQuery( + 'SELECT substr(state, ?, ?) AS part FROM day_checkpoint WHERE $where', + [off + 1, 128 * 1024, day, algorithm]); + if (parts.isEmpty) return null; + final part = parts.single['part'] as Uint8List; + if (part.length > length - off) return null; + bytes.setRange(off, off + part.length, part); + } + return row; + } + /// One revision counter per absolute 15-minute input bucket, shared by the /// canonical 1 Hz rows and sparse beats. Triggers cover every write path and /// roll back with the input transaction. No raw rows are read at creation. @@ -11517,6 +11577,9 @@ class LocalDb { } await db.transaction((txn) async { + // Scratch prefixes can include an evening owned by another day label. + // Clear every copy atomically with deletion so deleted raw stays deleted. + await _rotateStoreIdentity(txn); // `raw_archive` frames are kept, but a later archive re-drive must not // decode them back into a day the user deleted: record the windows. final prevRow = await txn.query('compute_freshness', @@ -11821,6 +11884,7 @@ class LocalDb { final t = r['name']; if (t is String) deleted += await txn.delete(t); } + await _rotateStoreIdentity(txn); }); _markStoreReplaced(); return deleted; @@ -12277,6 +12341,7 @@ class LocalDb { /// frame already here, or in the backup twice, lands once. Paged on rowid, /// like every other table the merge reads. Returns rows offered. static Future _mergeRawArchive(Database src, Database db, { + required Future Function(Transaction) beforePageCommit, required void Function() onPageCommitted, }) async { var copied = 0; @@ -12297,26 +12362,29 @@ class LocalDb { ); if (page.isEmpty) return; after = (page.last['_r'] as num).toInt(); - final batch = db.batch(); - for (final r in page) { - final hex = r['hex']; - if (hex is! String) continue; - batch.insert( - rawArchiveTable, - rawArchiveRow( - deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, - hex: hex, - counter: (r['counter'] as num?)?.toInt(), - packetType: (r['packet_type'] as num?)?.toInt() ?? 0, - recTs: (r['rec_ts'] as num?)?.toInt(), - capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, - reason: r['reason'] as String? ?? 'unknown', - ), - conflictAlgorithm: ConflictAlgorithm.ignore, - ); - copied++; - } - await batch.commit(noResult: true); + await db.transaction((txn) async { + await beforePageCommit(txn); + final batch = txn.batch(); + for (final r in page) { + final hex = r['hex']; + if (hex is! String) continue; + batch.insert( + rawArchiveTable, + rawArchiveRow( + deviceId: r['device_id'] as String? ?? kPrimaryDeviceId, + hex: hex, + counter: (r['counter'] as num?)?.toInt(), + packetType: (r['packet_type'] as num?)?.toInt() ?? 0, + recTs: (r['rec_ts'] as num?)?.toInt(), + capturedAt: (r['captured_at'] as num?)?.toInt() ?? 0, + reason: r['reason'] as String? ?? 'unknown', + ), + conflictAlgorithm: ConflictAlgorithm.ignore, + ); + copied++; + } + await batch.commit(noResult: true); + }); onPageCommitted(); } } @@ -12368,13 +12436,19 @@ class LocalDb { only: only, tolerant: tolerant, skip: skip, + beforePageCommit: (txn) async { + if (!pageCommitted) await _rotateStoreIdentity(txn); + }, onPageCommitted: () => pageCommitted = true, ); } finally { // A later page, source close or follow-up write can fail after a durable // replacement. Invalidate on every such exit, but never on a rolled-back // first page. Keep the flag local so concurrent merges cannot share it. - if (pageCommitted) _markStoreReplaced(); + if (pageCommitted) { + _markStoreReplaced(); + await (await instance).delete('day_checkpoint'); + } } } @@ -12382,6 +12456,7 @@ class LocalDb { String path, { List? only, bool tolerant = false, + required Future Function(Transaction) beforePageCommit, required void Function() onPageCommitted, Set skip = const {}, }) async { @@ -12790,6 +12865,7 @@ class LocalDb { for (final upper in ends) { final first = lower == _kBeforeEveryRowid; await db.transaction((txn) async { + await beforePageCommit(txn); if (t == 'day_result' && first) { // Inside the first chunk's transaction, so a day the engine // finalizes while the restore runs is still protected. @@ -12913,6 +12989,7 @@ class LocalDb { if (t == 'raw_archive') { try { counts[t] = await _mergeRawArchive(src, db, + beforePageCommit: beforePageCommit, onPageCommitted: onPageCommitted); } catch (_) { if (!tolerant) rethrow; @@ -13117,6 +13194,9 @@ class LocalDb { } } await db.transaction((txn) async { + // Rotate atomically with the first committed page. A killed + // process must not leave old scratch valid after partial restore. + await beforePageCommit(txn); if (t == SubstrateArchive.table) { final r = await SubstrateArchive.restoreRows( txn, diff --git a/lib/data/recent_slice.dart b/lib/data/recent_slice.dart index 2840705d4..5da2df053 100644 --- a/lib/data/recent_slice.dart +++ b/lib/data/recent_slice.dart @@ -84,6 +84,8 @@ const Map kSliceTables = { 'derive_fingerprint': (SliceKeep.all, null, 'which days changed since they were last scored'), 'input_rev': (SliceKeep.all, null, 'per-bucket input revisions'), 'session_ceiling_rev': (SliceKeep.all, null, 'session ceiling revisions'), + 'store_identity': (SliceKeep.none, null, 'local checkpoint store identity'), + 'day_checkpoint': (SliceKeep.none, null, 'disposable night/cardio checkpoints'), 'sleep_nap': (SliceKeep.all, null, 'naps'), 'sleep_override': (SliceKeep.all, null, 'sleep times the user edited'), // Workouts and activity: one row each. diff --git a/lib/data/restore_classification.dart b/lib/data/restore_classification.dart index ee17dee63..3d9c04f4b 100644 --- a/lib/data/restore_classification.dart +++ b/lib/data/restore_classification.dart @@ -39,6 +39,8 @@ const kRestoreClassification = { 'compute_freshness': RestoreClass.localOnly, 'derive_fingerprint': RestoreClass.rebuildable, 'input_rev': RestoreClass.rebuildable, + 'day_checkpoint': RestoreClass.rebuildable, + 'store_identity': RestoreClass.localOnly, 'session_ceiling_rev': RestoreClass.rebuildable, 'sleep_nap': RestoreClass.merge, 'sleep_override': RestoreClass.merge, diff --git a/pubspec.lock b/pubspec.lock index 6e95eb2e8..42c301a62 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -937,8 +937,8 @@ packages: dependency: "direct main" description: path: "." - ref: "3a7ca34d30233ff11e6c2877083f319a907ccb31" - resolved-ref: "3a7ca34d30233ff11e6c2877083f319a907ccb31" + ref: a438df40b38a4586a52a68ab7e1c100fca111ccd + resolved-ref: a438df40b38a4586a52a68ab7e1c100fca111ccd url: "https://github.com/OpenStrap/analytics.git" source: git version: "1.0.0" diff --git a/pubspec.yaml b/pubspec.yaml index b72fcf7b8..1cb573201 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -522,7 +522,11 @@ dependencies: # analytics main @ 3a7ca34: adds #101 (HRV/readiness/wellness fixes), #100 # (sleep staging and naps), #97 (stride-periodicity step counter) and #98 # (a mid-session rest stays inside one workout). - ref: 3a7ca34d30233ff11e6c2877083f319a907ccb31 + # analytics main @ a438df4: merged #102-#108 incremental APIs and review + # fixes, retaining #98's workout rest bridge, plus #110 (each recorded night + # observation carries the window it was staged over). Clock refusal, uncertainty + # and cosinor policies require kAlgoVersion 124. + ref: a438df40b38a4586a52a68ab7e1c100fca111ccd # BLE — flutter_blue_plus is the maintained cross-platform GATT client. flutter_blue_plus: ^1.36.8 diff --git a/test/data_ownership_test.dart b/test/data_ownership_test.dart index 576523a81..aca6e952c 100644 --- a/test/data_ownership_test.dart +++ b/test/data_ownership_test.dart @@ -4,7 +4,8 @@ // The reset test is the load-bearing one. The old reset walked the derived // days and deleted those, which left about twenty tables standing behind a // dialog that said otherwise — so the assertion here is deliberately -// "EVERY table in sqlite_master is empty", not a list of table names. A list +// "EVERY user-data table in sqlite_master is empty", not a list of table names. +// The store_identity row is rotated on wipe to invalidate old checkpoints. A list // would have to be updated when a table is added, which is exactly the failure // mode being tested for. @@ -60,7 +61,7 @@ void main() { }); group('wipeAll', () { - test('leaves EVERY table empty, including the ones a list would forget', + test('leaves EVERY user-data table empty and rotates local store identity', () async { await _seedEverything(); final db = await LocalDb.instance; @@ -83,11 +84,14 @@ void main() { expect(n, greaterThan(0), reason: '$t was never seeded'); } + final storeId = await LocalDb.storeIdentity(); await LocalDb.wipeAll(); + expect(await LocalDb.storeIdentity(), isNot(storeId)); for (final t in tables) { final n = _count(await db.rawQuery('SELECT COUNT(*) AS n FROM $t')); - expect(n, 0, reason: '$t survived "Delete everything"'); + expect(n, t == 'store_identity' ? 1 : 0, + reason: '$t survived "Delete everything"'); } }); diff --git a/test/ed2a_e15_restricted_target_walk_test.dart b/test/ed2a_e15_restricted_target_walk_test.dart index 331d4a43a..61135f276 100644 --- a/test/ed2a_e15_restricted_target_walk_test.dart +++ b/test/ed2a_e15_restricted_target_walk_test.dart @@ -113,7 +113,7 @@ void main() { reason: 'nonfolding production staging must skip the irrelevant earlier day'); }); - test('mayFold always walks earlier days and folds once (guard; D36 excluded)', () async { + test('mayFold skips unused earlier days and folds the target once (D36)', () async { await edFresh('ed2a_e15_fold.db'); await edRows(sub.tsSec.first, sub.tsSec.last + 1, hr: 55, az: 1); final edge = DateTime(2020, 1, 3, 12).millisecondsSinceEpoch ~/ 1000; @@ -123,12 +123,12 @@ void main() { ana.cardioUserProfile = null; ana.cardioRecordObservations = true; ana.resetCardioObservations(); - final full = prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: false); - final observations = ana.takeCardioObservations() - ..sort((a, b) => b.epochs.compareTo(a.epochs)); - expect(observations, isNotEmpty); - final o = observations.first; - final fullObservation = jsonEncode({'epochs': o.epochs, + final full = prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: false, + recordTargetOnly: true); + final observations = ana.takeCardioObservations(); + expect(observations, hasLength(1)); + final o = observations.single; + final targetObservation = jsonEncode({'epochs': o.epochs, 'hr_floor_p5': o.hrFloorP5, 'hr_floor_p25': o.hrFloorP25, 'hr_sleep_median': o.hrSleepMedian, 'hr_arousal': o.hrArousal, 'rmssd_med': o.rmssdMed, 'rmssd_mad': o.rmssdMad, @@ -139,10 +139,10 @@ void main() { (fold, count, obs) => walked.add((fold, count, obs)); final candidate = await e.debugSleepCandidate(edDay); expect(jsonEncode(candidate.toJson()), jsonEncode(full.toJson())); - expect(walked.single.$3, fullObservation, reason: "preserve the full walk's actual fold input"); + expect(walked.single.$3, targetObservation, reason: "D36 folds the target's own observation"); expect(walked.single.$1, isTrue, reason: 'fixture reaches a folding pass'); - expect(walked.single.$2, 2, - reason: 'the longest observation can still belong to the earlier day upstream'); + expect(walked.single.$2, 1, + reason: 'D36 permits the output-neutral E15 skip on folding passes'); expect(walked.single.$3, isNotNull, reason: 'fixture must produce a real observation'); final profile = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; final p = jsonDecode(profile as String) as Map; diff --git a/test/ed2a_e18_session_reuse_test.dart b/test/ed2a_e18_session_reuse_test.dart index b6dfa2702..aa4b1502b 100644 --- a/test/ed2a_e18_session_reuse_test.dart +++ b/test/ed2a_e18_session_reuse_test.dart @@ -47,7 +47,7 @@ void main() { await db.setVersion(60); await LocalDb.close(); db = await LocalDb.instance; - expect(await db.getVersion(), 61); + expect(await db.getVersion(), LocalDb.schemaVersion); expect(await db.query('sessions'), sessions); expect(await db.query('session_ceiling_rev'), isEmpty, reason: 'upgrade never scans or backfills existing sessions'); @@ -75,9 +75,18 @@ void main() { expect(await db.query('session_ceiling_rev'), hasLength(1)); expect(await db.query('input_rev'), isNotEmpty); final generation = LocalDb.storeGeneration; + final identity = await LocalDb.storeIdentity(); await LocalDb.wipeAll(); + expect(await LocalDb.storeIdentity(), isNot(identity)); for (final table in await LocalDb.tableNames()) { - expect(await db.query(table, limit: 1), isEmpty, reason: table); + if (table == 'store_identity') { + final rows = await db.query(table); + expect(rows, hasLength(1)); + expect(rows.single['singleton'], 1); + expect(rows.single['id'], matches(RegExp(r'^[0-9a-f]{32}$'))); + } else { + expect(await db.query(table, limit: 1), isEmpty, reason: table); + } } expect(LocalDb.storeGeneration, greaterThan(generation)); await edSession(); diff --git a/test/ed2a_input_revision_test.dart b/test/ed2a_input_revision_test.dart index 78823fb04..00fc1db97 100644 --- a/test/ed2a_input_revision_test.dart +++ b/test/ed2a_input_revision_test.dart @@ -155,10 +155,10 @@ void main() { expect(LocalDb.restoreTablesForTest, isNot(contains('input_rev'))); }); - test('rung 61 is reserved for additive input revisions', () async { - expect(LocalDb.schemaVersion, 61, reason: 'upstream is schema 60'); + test('rung 61 input revisions remain present with rung 62 checkpoints', () async { + expect(LocalDb.schemaVersion, 62, reason: 'checkpoint storage follows revisions at 61'); final db = await open(); - expect(await db.getVersion(), 61); + expect(await db.getVersion(), LocalDb.schemaVersion); expect(await _hasRevisions(db), isTrue); final tables = await LocalDb.tableNames(); expect(tables, isNot(contains('store_rev'))); @@ -175,7 +175,7 @@ void main() { expect(await db.rawQuery('PRAGMA table_info($table)'), legacyColumns[table], reason: 'rung 61 must not change raw columns/defaults'); } - expect(await db.getVersion(), 61); + expect(await db.getVersion(), LocalDb.schemaVersion); expect(await _hasRevisions(db), isTrue); final triggers = await db.rawQuery( "SELECT tbl_name, sql FROM sqlite_master WHERE type='trigger'", diff --git a/test/ed2a_substrate_transfer_test.dart b/test/ed2a_substrate_transfer_test.dart index de120beab..2c727d87e 100644 --- a/test/ed2a_substrate_transfer_test.dart +++ b/test/ed2a_substrate_transfer_test.dart @@ -241,7 +241,7 @@ void main() { markTestSkipped('committed golden is UTC; run this test with TZ=UTC'); return; } - expect(kAlgoVersion, 121, reason: "ED-2a keeps main's algorithm version"); + expect(kAlgoVersion, 124, reason: 'ED-2b repins analytics; transfer remains exact'); final fixture = _fixture(); final baseline = substrateFromDecodedPage(fixture.frames, fixture.rr); final transferred = Substrate.fromTransfer(baseline.toJson()); diff --git a/test/ed2b_algo_version_114_test.dart b/test/ed2b_algo_version_114_test.dart new file mode 100644 index 000000000..b3de3bd82 --- /dev/null +++ b/test/ed2b_algo_version_114_test.dart @@ -0,0 +1,35 @@ +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; + +void main() { + // WHY: finalized days need a new version when the analytics pin changes outputs. + test('ED-2b adoption and workout rest repin name their output changes', () { + expect(kAlgoVersion, 124); + expect(kAnalyticsPin, 'a438df40b38a4586a52a68ab7e1c100fca111ccd'); + final src = File('lib/compute/derivation_engine.dart').readAsStringSync(); + final at = src.indexOf('// 121 → 122'); + expect(at, isNonNegative); + final entry = src.substring(at, src.indexOf('const int kAlgoVersion', at)); + for (final change in ['f8e7d3c', '32', 'hrvTime', 'banisterTrimp', + 'ctlAtlTsb', 'cosinor', 'timeInZone', 'circadianNonparametric', + 'Lomb–Scargle', 'non-finite', 'irregular-screen gaps', 'D35', 'D36', + 'forward-only', 'staging labels unchanged']) { + expect(entry, contains(change)); + } + final repin = src.indexOf('// 122 → 123'); + expect(repin, isNonNegative); + final repinEntry = src.substring(repin, src.indexOf('// 123 → 124', repin)); + for (final change in ['801c825', '#98', 'workout rest bridge', + 'duration-cap', 'saved-session overlap']) { + expect(repinEntry, contains(change)); + } + final mainRepin = src.substring(src.indexOf('// 123 → 124')); + for (final change in ['a438df4', '#102-#110', '#110', 'regressing clocks', + '5%', 'Cosinor', 'SDNN', 'uncertainty', 'HrvFreqStream v2', + 'CardioRowCache v3', 'PRSA']) { + expect(mainRepin, contains(change)); + } + }); +} diff --git a/test/ed2b_cardio_cache_test.dart b/test/ed2b_cardio_cache_test.dart new file mode 100644 index 000000000..2339b257a --- /dev/null +++ b/test/ed2b_cardio_cache_test.dart @@ -0,0 +1,280 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/compute/vendor_sleep.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +const _day = '2020-01-02'; +int _sec(int hour) => DateTime(2020, 1, 2, hour).millisecondsSinceEpoch ~/ 1000; + +// RR plus accelerometry forces cardio rows through the real stager; a sleepy +// HR-only fixture could pass output guards without ever exercising the cache. +Substrate _night({int endHour = 7, int fromHour = -2}) { + final ts = [], hr = []; + final ax = [], ay = [], az = []; + final rr = [], rt = []; + var beat = _sec(fromHour) * 1000.0; + for (var t = _sec(fromHour); t < _sec(endHour); t++) { + final asleep = t >= _sec(-1) && t < _sec(6); + final bpm = asleep ? 52 + t % 7 : 85 + t % 13; + ts.add(t); hr.add(bpm); + ax.add(asleep ? 0 : .3 * math.sin(t * .21)); + ay.add(asleep ? 0 : .2 * math.cos(t * .13)); + az.add(asleep ? 1 : 1 + .05 * math.sin(t * .07)); + while (beat < (t + 1) * 1000) { + final ibi = 60000 / bpm + (asleep ? 40 : 12) * math.sin(t * .35); + rt.add(beat); rr.add(ibi); beat += ibi; + } + } + return Substrate(tsSec: ts, hr: hr, ax: ax, ay: ay, az: az, + rrMs: rr, rrTsMs: rt, skinTemp: List.filled(ts.length, 3000), + skinContact: List.filled(ts.length, 1), spo2Red: List.filled(ts.length, 1), + spo2Ir: List.filled(ts.length, 1), deviceFamily: 'gen4'); +} + +String _candidate(Substrate sub, ana.CardioRowCache? cache, + {SleepWindowOverride? override}) => jsonEncode(prepareSleepSessionCandidate( + sub, targetDay: _day, rowCache: cache, override: override).toJson()); + +Future _save(Substrate sub) async { + final db = await LocalDb.instance; final b = db.batch(); + for (var i = 0; i < sub.tsSec.length; i++) { + final t = sub.tsSec[i]; + b.insert('decoded_onehz', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'counter': i, 'hr': sub.hr[i], 'ax': sub.ax[i], + 'ay': sub.ay[i], 'az': sub.az[i], 'device_family': 'gen4'}); + } + final indexes = {}; + for (var i = 0; i < sub.rrMs.length; i++) { + final t = sub.rrTsMs[i] ~/ 1000; + final index = indexes.update(t, (v) => v + 1, ifAbsent: () => 0); + b.insert('decoded_rr', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'rr_ts_ms': sub.rrTsMs[i], 'rr_ms': sub.rrMs[i], + 'beat_index': index}); + } + await b.commit(noResult: true); +} + +void _nearPrsa(Object? got, Object? want, [String path = r'$']) { + if (got is double && want is double) { + final tolerance = path.endsWith('.confidence') ? 1e-12 : 1e-9 + 1e-9 * want.abs(); + expect((got - want).abs(), lessThanOrEqualTo(tolerance), reason: path); + } else if (want is Map) { + final g = got as Map; expect(g.keys.toList(), want.keys.toList()); + for (final k in want.keys) { _nearPrsa(g[k], want[k], '$path.$k'); } + } else if (want is List) { + final g = got as List; expect(g.length, want.length); + for (var i = 0; i < want.length; i++) { _nearPrsa(g[i], want[i], '$path[$i]'); } + } else { expect(got, want, reason: path); } +} + +void _checkNightOutput(Map input, Map output) { + final cold = deriveDayBundle(input); + final a = Map.from(output['clinical'] as Map); + final b = Map.from(cold['clinical'] as Map); + for (final k in ['prsa_dc', 'prsa_ac']) { _nearPrsa(a.remove(k), b.remove(k)); } + expect(jsonEncode({...output, 'clinical': a}), jsonEncode({...cold, 'clinical': b})); +} + +void main() { + for (final path in ['candidate auto', 'candidate manual', 'candidate confirmed', 'payload', 'calendar']) { + test('E6 $path shares the supplied CardioRowCache and stays byte-identical', () { + final sub = _night(); final cache = ana.CardioRowCache(); + String run(ana.CardioRowCache? rows) { + if (path == 'candidate auto') { return _candidate(sub, rows); } + if (path == 'candidate confirmed' || path == 'candidate manual') { return _candidate(sub, rows, + override: SleepWindowOverride(dayId: _day, onsetSec: _sec(0), + offsetSec: _sec(5), source: path == 'candidate manual' ? 'manual' : 'confirmed')); } + if (path == 'payload') { return jsonEncode(prepareDerivationPayload(sub, + targetDay: _day, rowCache: rows).days.map((d) => d.toJson()).toList()); } + return jsonEncode(calendarDays(sub, rowCache: rows) + .map((d) => d.sleep.toJson()).toList()); + } + final plain = run(null); expect(run(cache), plain); expect(run(cache), plain); + expect(cache.length, greaterThan(0), reason: 'every actual staging call must populate shared rows'); + expect(cache.hits, greaterThan(0)); + }); + } + + test('E6 HR-led fallback shares the cache through its forced stager', () { + // Include waking HR before the previous local noon so the fallback uses + // an actual daytime baseline rather than a flat night percentile. + final original = _night(fromHour: -13); + final sub = Substrate.fromJson({...original.toJson(), + // Keep valid moving vectors: all-zero missing accel would leave the + // forced stager with no feature rows to cache even after E6 lands. + 'ax': [for (final t in original.tsSec) .3 * math.sin(t * .21)], + 'ay': [for (final t in original.tsSec) .2 * math.cos(t * .13)], + 'az': [for (final t in original.tsSec) 1 + .05 * math.sin(t * .07)]}); + final plain = prepareSleepSessionCandidate(sub, targetDay: _day); + expect(plain.present, isTrue); + expect(plain.sleepSource, 'auto_fallback', reason: 'fixture must reach the HR-led forced-window call'); + final cache = ana.CardioRowCache(); + expect(_candidate(sub, cache), jsonEncode(plain.toJson())); + expect(_candidate(sub, cache), jsonEncode(plain.toJson())); + expect(cache.hits, greaterThan(0)); + }); + + test('E6 vendor-owned forced staging shares the same auto-stage row cache', () { + final sub = _night(); + final plain = prepareSleepSessionCandidate(sub, targetDay: _day); + expect(plain.present, isTrue); + final start = plain.sleepOnsetSec, end = plain.sleepOffsetSec - 1; + final step = (end - start) ~/ 3; + final vendor = VendorNight(deviceId: '', source: 'gen4', decodedAtSec: _sec(7), + epochs: [VendorEpoch(start, start + step, 'light'), + VendorEpoch(start + step, start + step * 2, 'deep'), + VendorEpoch(start + step * 2, end, 'rem')]); + String run(ana.CardioRowCache? cache) { + final got = prepareSleepSessionCandidate(sub, targetDay: _day, + vendorNights: [vendor], rowCache: cache); + expect(got.sleepSource, 'vendor_staged', reason: 'fixture must reach the vendor forced-window call'); + return jsonEncode(got.toJson()); + } + final cold = run(null), cache = ana.CardioRowCache(); + expect(run(cache), cold); expect(run(cache), cold); + expect(cache.hits, greaterThan(0)); + }); + + test('E6 an append reuses settled rows and preserves cold candidate bytes', () { + final cache = ana.CardioRowCache(); + _candidate(_night(endHour: 3), cache); + final full = _night(); final got = _candidate(full, cache); + expect(got, _candidate(full, null)); + expect(cache.hits, greaterThan(0), reason: 'earlier epochs survive a growing night'); + }); + + test('guard: #99 malformed cardio cache refuses whole payload', () { + for (final bad in [{'version': 99, 'rows': []}, + {'version': 2, 'rows': 'invalid'}, + {'version': 2, 'rows': [{'startSec': 1}]}]) { + expect(() => ana.CardioRowCache.fromJson(bad), throwsFormatException); + } + }); + + group('E6 engine owns encoded bytes and workers own row objects', () { + setUpAll(edInit); + setUp(() async { await edFresh('ed2b_cardio.db'); await _save(_night()); }); + tearDown(edDrop); + + test('real engine worker decodes, merges and prunes the shared row cache', () async { + final e = DerivationEngine()..debugDisableStagedSleepCache = true; + final first = await e.debugSleepCandidate(_day); + final second = await e.debugSleepCandidate(_day); + expect(jsonEncode(second.toJson()), jsonEncode(first.toJson())); + expect(e.debugCardioRowBytes, isA(), reason: 'UI isolate retains only the encoded payload'); + expect(e.debugCardioRowHits, greaterThan(0)); + expect(e.debugCardioRowWorkerOperations, containsAll(['decode:worker', 'merge:worker', 'prune:worker'])); + expect(e.debugCardioRowWorkerOperations.where((s) => s.endsWith(':caller')), isEmpty); + final cold = DerivationEngine()..debugDisableStagedSleepCache = true + ..debugDisableCardioRows = true; + expect(jsonEncode((await cold.debugSleepCandidate(_day)).toJson()), + jsonEncode(first.toJson())); + }); + + for (final key in ['input revision', 'generation', 'algorithm']) { + test('$key rejects bytes then replacement cache hits with cold-exact staging', () async { + final e = DerivationEngine()..debugDisableStagedSleepCache = true; + final first = await e.debugSleepCandidate(_day); + final firstMisses = e.debugCardioRowMisses; + if (key == 'input revision') { + final db = await LocalDb.instance; + final row = (await db.query('decoded_onehz', limit: 1)).first; + await db.update('decoded_onehz', {'hr': row['hr']}, + where: 'device_id = ? AND ts_ms = ?', whereArgs: [row['device_id'], row['ts_ms']]); + } else if (key == 'generation') { + await LocalDb.close(); await LocalDb.instance; + } else { + final fp = await LocalDb.inputDayFingerprint(_day); + e.debugRekeyCardioRows(inputRevision: fp.revisionSum, + generation: fp.generation, algorithm: 115); + } + final changed = await e.debugSleepCandidate(_day); + expect(jsonEncode(changed.toJson()), jsonEncode(first.toJson())); + expect(e.debugCardioRowMisses, greaterThan(firstMisses), + reason: 'new provenance must reject existing encoded rows'); + final misses = e.debugCardioRowMisses, hits = e.debugCardioRowHits; + await e.debugSleepCandidate(_day); + expect(e.debugCardioRowMisses, misses); + expect(e.debugCardioRowHits, greaterThan(hits), reason: 'new provenance must become reusable'); + }); + } + test('E7-E11 real runDays retains night streams across worker dispatches', () async { + await LocalDb.putSleepOverride(dayId: _day, onsetTs: _sec(1), + offsetTs: _sec(1) + 1200, source: 'confirmed'); + var observed = 0; + final e = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + e.debugOnNightBundle = (input, output) { + observed++; + _checkNightOutput(input, output); + }; + expect(await e.runDays(edProfile, {_day}), 1); + expect(observed, greaterThan(0)); + final before = e.debugNightStreams!; + expect(before.processedNightBeats, greaterThan(0)); + final work = {for (final name in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) + name: before.processedBeats(name)}; + expect(await e.runDays(edProfile, {_day}), 1); + expect(e.debugNightStreams!.processedNightBeats, before.processedNightBeats, + reason: 'the production dispatch must return and retain the RAM night owner'); + for (final entry in work.entries) { + expect(work[entry.key], greaterThan(0)); + expect(e.debugNightStreams!.processedBeats(entry.key), entry.value); + } + expect(e.debugNightStreams!.fullListDigests, greaterThan(0)); + expect(e.debugNightStreams!.maximumListDigestsPerIdentity, 1); + expect(e.debugNightStreams!.identityMemoEntries, 0); + }); + + for (final change in ['revision', 'generation', 'algorithm']) { + test('E7-E11 engine $change misses then retains the replacement worker state', () async { + await LocalDb.putSleepOverride(dayId: _day, onsetTs: _sec(1), + offsetTs: _sec(1) + 1200, source: 'confirmed'); + final e = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics() + ..debugOnNightBundle = _checkNightOutput; + // Isolate the RAM generation contract: disk checkpoints legitimately + // survive reopen and would otherwise hide the required RAM rebuild. + if (change == 'generation') e.debugCheckpoints.disabled = true; + expect(await e.runDays(edProfile, {_day}), 1); + final before = e.debugNightStreams!.processedNightBeats; + if (change == 'revision') { + final db = await LocalDb.instance; + final row = (await db.query('decoded_onehz', where: 'rec_ts = ?', + whereArgs: [_sec(1) + 60], limit: 1)).first; + await db.update('decoded_onehz', {'hr': row['hr']}, + where: 'device_id = ? AND ts_ms = ?', whereArgs: [row['device_id'], row['ts_ms']]); + } else if (change == 'generation') { + await LocalDb.close(); await LocalDb.instance; + } else { + final fp = await LocalDb.inputDayFingerprint(_day); + e.debugNightStreams!.context(inputRevision: fp.revisionSum, + generation: fp.generation, algorithm: 115); + } + expect(await e.runDays(edProfile, {_day}), 1); + final miss = e.debugNightStreams!.processedNightBeats; + expect(miss, greaterThan(before)); + final metricWork = {for (final s in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) + s: e.debugNightStreams!.processedBeats(s)}; + expect(await e.runDays(edProfile, {_day}), 1); + expect(e.debugNightStreams!.processedNightBeats, miss); + for (final s in metricWork.keys) { + expect(e.debugNightStreams!.processedBeats(s), metricWork[s]); + } + expect(e.debugNightStreams!.identityMemoEntries, 0); + }); + } + }); +} diff --git a/test/ed2b_checkpoints_test.dart b/test/ed2b_checkpoints_test.dart new file mode 100644 index 000000000..41bc68409 --- /dev/null +++ b/test/ed2b_checkpoints_test.dart @@ -0,0 +1,854 @@ +// E16 RED contracts. The standalone night/cardio payload uses format 2; +// fork day/minute/awake accumulator formats and schema numbers do not transfer. +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:io'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:crypto/crypto.dart'; +import 'package:path_provider_platform_interface/path_provider_platform_interface.dart'; +import 'package:openstrap_edge/compute/night_checkpoint.dart'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/recent_slice.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common/sqflite_logger.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +class _CheckpointPaths extends PathProviderPlatform { + _CheckpointPaths(this.root); + final String root; + @override + Future getTemporaryPath() async => root; +} + +Map _legacyRsa(Map row, {required bool bins}) { + final bytes = row['state'] as List; + final payload = jsonDecode(utf8.decode(bytes.sublist(32))) as Map; + final night = payload['night'] as Map; + final state = bins ? (night['rsa_bins'] as Map)['open'] as Map : night['rsa'] as Map; + state['version'] = 2; + final body = utf8.encode(jsonEncode(payload)); + return {'state': Uint8List.fromList([...sha256.convert(body).bytes, ...body])}; +} + +Future _hasCheckpoint(Database db) async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='day_checkpoint'")) + .isNotEmpty; + +Future _storage() async { + final db = await LocalDb.instance; + expect(await _hasCheckpoint(db), isTrue, + reason: 'E16 needs disposable day_checkpoint storage at rung 62'); + expect(await db.query('store_identity'), hasLength(1), + reason: 'rung 62 and repair must also create the durable identity row'); + return db; +} + +Map _sampleRow({String day = edDay, int? version}) => { + 'day_id': day, 'algo_version': version ?? kAlgoVersion, 'fmt': nightCheckpointFormat, + 'ctx_sig': 'night-cardio-context', 'cp_rec_ts': edStart ~/ 900 * 900, + 'rev_vec': Uint8List.fromList([0, 1, 2, 3, 4, 5, 6, 7]), + 'state': Uint8List.fromList([0, 255, 9, 10, 13, 128]), + 'night_ref': null, 'computed_at': 7, +}; + +Future>> _rows(Database db) => db.query( + 'day_checkpoint', where: 'day_id = ? AND algo_version = ?', + whereArgs: [edDay, kAlgoVersion]); + +// Supply a real sleep window and RR, not an absent-night fixture that could +// satisfy persistence by writing an empty payload. Two closed 15-minute buckets +// plus a tail exercise the resume boundary. +const _nightSeconds = 2100, _manualOffsetSeconds = 2000; +// One beat per second, including both manual-window endpoints. Clamp to the +// seeded RR extent; later 1 Hz-only appends add no beats. +int get _manualBeatCount => math.min(_nightSeconds, _manualOffsetSeconds + 1); + +Future _night() async { + await edRows(edStart, edStart + _nightSeconds, hr: 60, az: 1); + await LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + _manualOffsetSeconds, source: 'manual'); + final db = await LocalDb.instance; + final batch = db.batch(); + for (var i = 0; i < _nightSeconds; i++) { + final ts = edStart + i; + batch.insert('decoded_rr', {'device_id': '', 'ts_ms': ts * 1000, + 'rec_ts': ts, 'beat_index': 0, 'rr_ts_ms': ts * 1000, + 'rr_ms': 950.0 + 35 * math.sin(i * .13) + 13 * math.sin(i * .041)}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + await batch.commit(noResult: true); +} + +Future _derive(DerivationEngine engine) async { + expect(await engine.runDays(edProfile, {edDay}), 1, + reason: 'the checkpoint assertions require a real persisted derivation'); +} + +void main() { + final originalName = LocalDb.dbName; + final originalZone = Platform.environment['TZ'] ?? 'UTC'; + setUpAll(edInit); + setUp(() async { + edSetZone('UTC'); + await edFresh('ed2b_checkpoints.db'); + LocalDb.lastRebuild = null; + }); + tearDown(() async { + DerivationEngine.debugHoldTonight = {}; + await edDrop(); + LocalDb.dbName = originalName; + edSetZone(originalZone); + }); + + test('E16 fresh storage uses rung 62 and a versioned composite key', () async { + expect(LocalDb.schemaVersion, 62, + reason: '61 belongs to ED-2a revisions; checkpoints use the next rung'); + final db = await _storage(); + expect(await db.getVersion(), 62); + expect(await db.query('store_identity'), hasLength(1)); + expect(await LocalDb.storeIdentity(), matches(RegExp(r'^[0-9a-f]{32}$'))); + final columns = {for (final row in await db.rawQuery( + 'PRAGMA table_info(day_checkpoint)')) row['name']: row}; + expect(columns.keys.toSet(), {'day_id', 'algo_version', 'fmt', 'ctx_sig', + 'cp_rec_ts', 'rev_vec', 'state', 'night_ref', 'computed_at'}); + expect(columns['day_id']!['pk'], 1); + expect(columns['algo_version']!['pk'], 2); + for (final name in ['rev_vec', 'state']) { + expect(columns[name]!['type'], 'BLOB'); + expect(columns[name]!['notnull'], 1); + } + expect(await db.query('day_checkpoint'), isEmpty); + }); + + test('E16 61 to 62 preserves decoded rows and revisions without backfill', () async { + var db = await LocalDb.instance; + await edRows(edStart, edStart + 3); + await db.insert('decoded_rr', {'device_id': '', 'ts_ms': edStart * 1000, + 'rec_ts': edStart, 'rr_ts_ms': edStart * 1000, 'beat_index': 0, 'rr_ms': 900}); + final raw = await db.query('decoded_onehz'); + final rr = await db.query('decoded_rr'); + final revs = await db.query('input_rev', orderBy: 'bucket'); + final rawShape = await db.rawQuery('PRAGMA table_info(decoded_onehz)'); + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + await db.setVersion(61); + await LocalDb.close(); + db = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull, + reason: 'a migration must preserve the original DB, not quarantine it'); + expect(await db.query('decoded_onehz'), raw); + expect(await db.query('decoded_rr'), rr); + expect(await db.query('input_rev', orderBy: 'bucket'), revs); + expect(await db.rawQuery('PRAGMA table_info(decoded_onehz)'), rawShape); + await _storage(); + expect(await db.getVersion(), 62); + expect(await db.query('day_checkpoint'), isEmpty, + reason: 'migration creates empty scratch storage without deriving old days'); + }); + + test('E16 same-version repair creates storage and is idempotent', () async { + final db = await LocalDb.instance; + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + await LocalDb.repairOpenSchemaForTest(db); + await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final before = await db.query('day_checkpoint'); + final storeId = await LocalDb.storeIdentity(); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.close(); + final reopened = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull); + expect(await reopened.query('day_checkpoint'), before); + expect(await LocalDb.storeIdentity(), storeId); + }); + + test('E16 upgrade and open repair create scratch without scanning raw input', () async { + var db = await LocalDb.instance; + await edRows(edStart, edStart + 3); + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + await db.setVersion(61); + await LocalDb.close(); + final sql = []; + // Observe the actual sqflite calls, including batches, without changing + // migration code or providing a synthetic checkpoint implementation. + // ignore: experimental_member_use + databaseFactoryOrNull = SqfliteDatabaseFactoryLogger(databaseFactoryFfi, + options: SqfliteLoggerOptions( + type: SqfliteDatabaseFactoryLoggerType.invoke, + log: (event) { + if (event is! SqfliteLoggerInvokeEvent) return; + void collect(Object? value) { + if (value is Map) { + if (value['sql'] case final String statement) sql.add(statement); + for (final v in value.values) { collect(v); } + } else if (value is List) { + for (final v in value) { collect(v); } + } + } + collect(event.arguments); + }, + )); + try { + db = await LocalDb.instance; + await LocalDb.repairOpenSchemaForTest(db); + final openingSql = List.of(sql); + expect(openingSql, isNotEmpty, reason: 'SQL observation must be active'); + final rawAccess = RegExp( + r'\b(?:FROM|JOIN|UPDATE|INTO|DELETE\s+FROM)\s+["`\[]?' + r'(?:decoded_onehz|decoded_rr|samples|raw_archive)\b', + caseSensitive: false); + expect(openingSql.where((statement) { + final ddl = statement.trimLeft().toUpperCase(); + if (ddl.startsWith('CREATE TRIGGER') || ddl.startsWith('CREATE VIEW')) return false; + return rawAccess.hasMatch(statement); + }), isEmpty, reason: 'rung 62 and repair must perform no raw scan/backfill/rewrite'); + await _storage(); + expect(await db.query('day_checkpoint'), isEmpty); + } finally { + await LocalDb.close(); + databaseFactory = databaseFactoryFfi; + } + }); + + test('E16 actual same-version reopen repairs missing checkpoint storage', () async { + final db = await LocalDb.instance; + await db.execute('DROP TABLE IF EXISTS day_checkpoint'); + await db.execute('DROP TABLE IF EXISTS store_identity'); + final version = await db.getVersion(); + await LocalDb.close(); + final reopened = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull); + expect(await reopened.getVersion(), version); + await _storage(); + expect(await reopened.query('day_checkpoint'), isEmpty); + }); + + test('E16 rerunning rung 62 keeps checkpoint blobs byte for byte', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + await db.insert('day_checkpoint', _sampleRow(version: kAlgoVersion - 1)); + await db.insert('day_checkpoint', _sampleRow(day: '2020-01-03')); + final before = await db.query('day_checkpoint', orderBy: 'day_id,algo_version'); + final storeId = await LocalDb.storeIdentity(); + await db.setVersion(61); + await LocalDb.close(); + final reopened = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull); + expect(await reopened.query('day_checkpoint', orderBy: 'day_id,algo_version'), before); + expect(await LocalDb.storeIdentity(), storeId); + expect(await _rows(reopened), hasLength(1)); + expect((await _rows(reopened)).single['state'], _sampleRow()['state']); + }); + + test('E16 standalone payload carries night and cardio only, format 2', () async { + await _night(); + final engine = DerivationEngine(); + await _derive(engine); + final diagnostics = engine.debugCheckpoints; + expect(diagnostics.writes, greaterThan(0), + reason: 'the worker must emit a checkpoint from real derived inputs'); + expect(diagnostics.lastEncodedPayload, isNotNull); + expect(diagnostics.lastEncodedPayload, isNotEmpty); + final payload = diagnostics.lastDecodedPayload; + expect(payload, isNotNull, + reason: 'observe the worker decode, without decoding on the UI isolate'); + expect(payload!['v'], nightCheckpointFormat, + reason: 'RSA v3 and closed-bin states require the new format'); + expect(payload['night'], isNotNull, reason: 'fixture supplies a settled RR night'); + expect(payload['cardio'], isNotNull); + // Inspect the actual worker-emitted inner checkpoint. In #99 capped + // correctors carry their consumed input count at capped.n, not top-level n. + final night = payload['night'] as Map; + final tag = night['tag'] as Map; + final corrector = (night['corrector'] as Map).cast(); + expect(corrector['version'], 2); + expect((night['rsa'] as Map)['version'], 3); + expect((night['rsa_bins'] as Map)['v'], 2); + expect(corrector['type'], 'RrCorrector'); + expect(corrector['maxArtifactRun'], 32); + expect((corrector['capped'] as Map)['n'], tag['count'], + reason: 'the serialized capped corrector must describe exactly the tagged night prefix'); + expect(tag['count'], greaterThan(0)); + expect(ana.RrCorrector.fromJson(corrector).toJson(), corrector, + reason: '#99 must accept the real inner checkpoint without changing its state'); + for (final key in ['day', 'minutes', 'bills', 'awake', 'activity', 'curves']) { + expect(payload, isNot(contains(key)), reason: '$key is outside ED-2b adoption'); + } + }); + + test('E16 process restart resumes and equals cold derive byte for byte', () async { + await _night(); + // Close the entire selected night before the checkpoint boundary, so a + // restarted worker has no tail beats left to fold. + await edRows(edStart + 2100, edStart + 2700, hr: 60, az: 1); + final first = DerivationEngine(); + await _derive(first); + final before = await edStored(edDay); + final storeId = await LocalDb.storeIdentity(); + await LocalDb.close(); + await LocalDb.instance; + expect(await LocalDb.storeIdentity(), storeId); + final resumed = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(resumed); + final resumedBytes = await edStored(edDay); + expect(resumedBytes, before); + final cold = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(cold); + expect(await edStored(edDay), resumedBytes); + expect(resumed.debugCheckpoints.hits, greaterThan(0), + reason: 'a fresh engine after reopen must adopt the persisted night/cardio state'); + expect(resumed.debugNightStreams!.processedNightBeats, 0); + expect(cold.debugCheckpoints.hits, 0); + }); + + test('E16 resumed append equals cold derive on the identical extended input', () async { + await _night(); + await _derive(DerivationEngine()); + await edRows(edStart + 2100, edStart + 2400, hr: 61, az: 1); + final resumed = DerivationEngine(); + await _derive(resumed); + final actual = await edStored(edDay); + final cold = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(cold); + expect(await edStored(edDay), actual); + expect(resumed.debugCheckpoints.hits, greaterThan(0), + reason: 'new open-bucket tail must not discard the valid closed prefix'); + }); + + final damaged = Function(Map)>{ + 'truncated blob': (_) => {'state': Uint8List.fromList([1, 0, 255])}, + 'checksum': (row) { + final bytes = Uint8List.fromList(row['state'] as List); + bytes[bytes.length ~/ 2] ^= 1; + return {'state': bytes}; + }, + 'old format': (_) => {'fmt': 1}, + 'future format': (_) => {'fmt': 999}, + 'legacy RSA v2': (row) => _legacyRsa(row, bins: false), + 'legacy RSA-bin v2': (row) => _legacyRsa(row, bins: true), + 'old algorithm': (_) => {'algo_version': kAlgoVersion - 1}, + 'foreign context': (_) => {'ctx_sig': 'foreign-owner-profile-and-window'}, + 'misaligned boundary': (row) => {'cp_rec_ts': (row['cp_rec_ts'] as int) + 1}, + 'corrupt revision vector': (_) => {'rev_vec': Uint8List.fromList([1])}, + }; + for (final entry in damaged.entries) { + test('E16 ${entry.key} is refused whole and recomputed exactly', () async { + await _night(); + await _derive(DerivationEngine()); + final db = await _storage(); + final rows = await _rows(db); + expect(rows, hasLength(1), reason: 'real derive must persist a resume point'); + await db.update('day_checkpoint', entry.value(rows.single), + where: 'day_id = ? AND algo_version = ?', whereArgs: [edDay, kAlgoVersion]); + final engine = DerivationEngine(); + await _derive(engine); + final rejectedOutput = await edStored(edDay); + final cold = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(cold); + expect(await edStored(edDay), rejectedOutput); + expect(engine.debugCheckpoints.hits, 0, + reason: 'never salvage a night or cardio subset of an invalid checkpoint'); + if (entry.key == 'old algorithm') { + expect(engine.debugCheckpoints.misses, greaterThan(0), + reason: 'version-keyed lookup must miss the old algorithm row'); + } else { + expect(engine.debugCheckpoints.rejections, isNotEmpty, + reason: 'rejection must be observable after the real reader attempts it'); + } + expect(engine.debugCheckpoints.writes, greaterThan(0), reason: 'cold recompute replaces bad state'); + }); + } + + test('E16 changed closed input revision refuses an otherwise valid checkpoint', () async { + await _night(); + await _derive(DerivationEngine()); + await edRows(edStart, edStart + 1, hr: 60, az: 1); + final engine = DerivationEngine(); + await _derive(engine); + final revised = await edStored(edDay); + await _derive(DerivationEngine()..debugCheckpoints.disabled = true); + expect(await edStored(edDay), revised); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty, + reason: 'identical newest-wins replacement changes input_rev, so prefix must rebuild'); + }); + + for (final sameEngine in [true, false]) { + test('E16 reopen reuses checkpoints with unchanged revisions, ' + '${sameEngine ? 'retained' : 'fresh'} engine', () async { + await _night(); + await edRows(edStart + 2100, edStart + 2700, hr: 60, az: 1); + final storeId = await LocalDb.storeIdentity(); + final first = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(first); + final before = await LocalDb.inputDayFingerprint(edDay); + final output = await edStored(edDay); + final beats = first.debugNightStreams!.retainedNightBeats; + final work = first.debugNightStreams!.processedNightBeats; + final streamWork = {for (final key in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) + key: first.debugNightStreams!.processedBeats(key)}; + expect(beats, _manualBeatCount, + reason: 'one seeded RR beat per second, including the manual-window end'); + final saved = await _rows(await _storage()); + expect(saved, hasLength(1)); + await LocalDb.close(); + final db = await LocalDb.instance; + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(after.generation, greaterThan(before.generation)); + expect(await _rows(db), saved, + reason: 'ordinary reopen preserves the durable checkpoint'); + expect(await LocalDb.storeIdentity(), storeId); + final engine = sameEngine ? first : + (DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics()); + engine.debugCheckpoints.reset(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 1, + reason: 'the same store id permits disk adoption after RAM generation changes'); + expect(engine.debugCheckpoints.rejections, isEmpty); + expect(engine.debugNightStreams!.processedNightBeats, + sameEngine ? work : 0, + reason: 'resume adds no night work and preserves cumulative RAM counters'); + for (final entry in streamWork.entries) { + expect(engine.debugNightStreams!.processedBeats(entry.key), sameEngine ? entry.value : 0, + reason: '${entry.key} counters survive same-engine disk adoption'); + } + expect(await edStored(edDay), output); + final replacement = await _rows(db); + expect(replacement, hasLength(1)); + expect(replacement.single['ctx_sig'], saved.single['ctx_sig']); + final rebuilt = engine.debugNightStreams!.processedNightBeats; + await _derive(engine); + expect(engine.debugNightStreams!.processedNightBeats, rebuilt, + reason: 'the replacement RAM owner must be retained'); + expect(await edStored(edDay), output); + final resumed = DerivationEngine(); + await _derive(resumed); + expect(resumed.debugCheckpoints.hits, 1, + reason: 'the checkpoint remains reusable in the same store'); + expect(await edStored(edDay), output); + }); + } + + test('E16 committed import rejects stale disk state with unchanged raw revisions', () async { + await _night(); + final engine = DerivationEngine(); + await _derive(engine); + final before = await LocalDb.inputDayFingerprint(edDay); + final storeId = await LocalDb.storeIdentity(); + final saved = (await _rows(await _storage())).single; + final dir = await Directory.systemTemp.createTemp('ed2b_cp_generation_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE journal (date TEXT PRIMARY KEY, ' + "note TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL)"); + await source.insert('journal', {'date': edDay, 'note': 'generation-only', 'updated_at': 1}); + await source.close(); + await LocalDb.importFromDbFile(path); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(after.generation, greaterThan(before.generation), + reason: 'committed store replacement differs from a legitimate checkpoint restart'); + expect(await LocalDb.storeIdentity(), isNot(storeId)); + final db = await _storage(); + expect(await _rows(db), isEmpty); + // Defeat scratch clearing deliberately: identity must reject stale bytes + // even when the revision vector and raw inputs are unchanged. + await db.insert('day_checkpoint', saved); + await LocalDb.close(); + await LocalDb.instance; + engine.debugCheckpoints.reset(); + await _derive(engine); + final replaced = await edStored(edDay); + await _derive(DerivationEngine()..debugCheckpoints.disabled = true); + expect(await edStored(edDay), replaced); + expect(engine.debugCheckpoints.hits, 0, + reason: 'the original engine must not revive its pre-restore resume state'); + expect(engine.debugCheckpoints.misses, greaterThan(0), + reason: 'the new store identity must force a cold night/cardio lookup'); + expect(engine.debugCheckpoints.rejections, isNotEmpty); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + + test('E16 wipe and re-import reject stale checkpoint when revisions repeat', () async { + await _night(); + await _derive(DerivationEngine()); + final db = await _storage(); + final saved = (await _rows(db)).single; + final before = await LocalDb.inputDayFingerprint(edDay); + final storeId = await LocalDb.storeIdentity(); + final dir = await Directory.systemTemp.createTemp('ed2b_cp_wipe_'); + final path = p.join(dir.path, 'source.db'); + try { + await db.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'); + await File(db.path).copy(path); + await LocalDb.wipeAll(); + final wipedId = await LocalDb.storeIdentity(); + expect(wipedId, isNot(storeId)); + await LocalDb.importFromDbFile(path); + expect(await LocalDb.storeIdentity(), isNot(wipedId)); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum, + reason: 'input_rev is local, so reset and identical restore can repeat it'); + expect(after.revisionMax, before.revisionMax); + expect(await _rows(db), isEmpty); + await db.insert('day_checkpoint', saved); + await LocalDb.close(); + await LocalDb.instance; + final engine = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty); + expect(engine.debugNightStreams!.processedNightBeats, _manualBeatCount); + final actual = await edStored(edDay); + await _derive(DerivationEngine()..debugCheckpoints.disabled = true); + expect(await edStored(edDay), actual); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + + test('E16 replaced DB file rejects old checkpoint with identical inputs', () async { + await _night(); + await _derive(DerivationEngine()); + final original = await _storage(); + final saved = (await _rows(original)).single; + final before = await LocalDb.inputDayFingerprint(edDay); + final storeId = await LocalDb.storeIdentity(); + final liveName = LocalDb.dbName; + final livePath = original.path; + final replacementPath = p.join(p.dirname(livePath), 'ed2b_cp_replacement.db'); + await LocalDb.close(); + try { + await edFresh('ed2b_cp_replacement.db'); + await _night(); + final replacement = await _storage(); + final replacementId = await LocalDb.storeIdentity(); + expect(replacementId, isNot(storeId)); + await replacement.insert('day_checkpoint', saved); + await replacement.rawQuery('PRAGMA wal_checkpoint(TRUNCATE)'); + await LocalDb.close(); + await databaseFactory.deleteDatabase(livePath); + await File(replacementPath).copy(livePath); + LocalDb.dbName = liveName; + await LocalDb.instance; + expect(await LocalDb.storeIdentity(), replacementId); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(await _rows(await _storage()), [saved]); + final engine = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty, + reason: 'an intact old blob must not certify a different database file'); + expect(engine.debugNightStreams!.processedNightBeats, _manualBeatCount); + } finally { + await LocalDb.close(); + LocalDb.dbName = liveName; + await databaseFactory.deleteDatabase(replacementPath); + } + }); + + test('E16 rebuild with tolerant salvage rotates identity and rejects old state', () async { + await _night(); + await _derive(DerivationEngine()); + final db = await _storage(); + final saved = (await _rows(db)).single; + final storeId = await LocalDb.storeIdentity(); + final before = await LocalDb.inputDayFingerprint(edDay); + // Force the real ladder-rebuild path. The rename collision rolls back the + // failed open; all input tables remain available to tolerant salvage. + await db.execute('CREATE TABLE _raw_old (hex TEXT)'); + await db.execute('DROP TABLE baselines'); + await db.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await db.insert('baselines', {'key': 'unreadable-salvage-shape'}); + await db.setVersion(2); + await LocalDb.close(); + try { + final rebuilt = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNotNull); + final rebuild = LocalDb.lastRebuild!; + expect(rebuild.salvaged['baselines'], 0, + reason: 'tolerant salvage must continue past a failed table'); + expect(rebuild.salvaged['decoded_onehz'], _nightSeconds); + expect(rebuild.salvaged['decoded_rr'], _nightSeconds); + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect(await _rows(rebuilt), isEmpty); + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + await rebuilt.insert('day_checkpoint', saved); + final engine = DerivationEngine()..debugNightStreams = Ed2bStreamDiagnostics(); + await _derive(engine); + expect(engine.debugCheckpoints.hits, 0); + expect(engine.debugCheckpoints.rejections, isNotEmpty); + expect(engine.debugNightStreams!.processedNightBeats, _manualBeatCount); + } finally { + final rebuild = LocalDb.lastRebuild; + if (rebuild != null) await databaseFactory.deleteDatabase(rebuild.quarantinePath); + } + }); + + test('E16 wipe clears checkpoints with every other table', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final storeId = await LocalDb.storeIdentity(); + await LocalDb.wipeAll(); + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect(await db.query('day_checkpoint'), isEmpty); + expect(await _hasCheckpoint(db), isTrue, reason: 'wipe clears data and preserves schema'); + }); + + for (final fail in [false, true]) { + test('E16 ${fail ? 'partially failing' : 'successful'} restore clears local scratch', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final storeId = await LocalDb.storeIdentity(); + await db.execute("CREATE TRIGGER require_rotated_identity BEFORE INSERT ON journal " + "BEGIN SELECT CASE WHEN (SELECT id FROM store_identity WHERE singleton = 1) " + "= '$storeId' OR EXISTS(SELECT 1 FROM day_checkpoint) " + "THEN RAISE(ABORT, 'old checkpoint identity inside restore page') END; END"); + final dir = await Directory.systemTemp.createTemp('ed2b_cp_restore_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE journal (date TEXT PRIMARY KEY, ' + "note TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL)"); + await source.insert('journal', {'date': edDay, 'note': 'restored', 'updated_at': 1}); + // Even a source with matching schema/version must not import its + // checkpoint into a different store's revision history. + final ddl = (await db.rawQuery("SELECT sql FROM sqlite_master " + "WHERE type='table' AND name='day_checkpoint'")).single['sql'] as String; + await source.execute(ddl); + await source.insert('day_checkpoint', { + ..._sampleRow(), 'ctx_sig': 'foreign-store-checkpoint', + }); + if (fail) { + await source.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await source.insert('baselines', {'key': 'invalid-import'}); + } + await source.close(); + if (fail) { + await expectLater(LocalDb.importFromDbFile(path), throwsA(isA())); + } else { + await LocalDb.importFromDbFile(path); + } + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect((await db.query('journal')).single['note'], 'restored', + reason: 'at least one replacement page committed'); + expect(await db.query('day_checkpoint'), isEmpty, + reason: 'a committed restore invalidates local checkpoint inputs even if a later page fails'); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + } + + for (final fail in [false, true]) { + test('E16 ${fail ? 'rolled-back first page' : 'empty import'} preserves identity and scratch', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final saved = await _rows(db); + final storeId = await LocalDb.storeIdentity(); + final generation = LocalDb.storeGeneration; + final dir = await Directory.systemTemp.createTemp('ed2b_cp_no_commit_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + if (fail) await source.insert('baselines', {'key': 'invalid-first-page'}); + await source.close(); + if (fail) { + await expectLater(LocalDb.importFromDbFile(path), throwsA(isA())); + } else { + await LocalDb.importFromDbFile(path); + } + expect(await LocalDb.storeIdentity(), storeId); + expect(LocalDb.storeGeneration, generation); + expect(await _rows(db), saved); + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + } + + test('E16 issue-report slices explicitly classify disposable checkpoints', () { + expect(kSliceTables, contains('day_checkpoint'), + reason: 'new storage must be classified, not silently use fallback'); + expect(kSliceTables['day_checkpoint']!.$1, SliceKeep.none); + expect(kSliceTables['store_identity']!.$1, SliceKeep.none); + }); + + test('GUARD restore allow-list excludes disposable checkpoints', () { + expect(LocalDb.restoreTablesForTest, isNot(contains('day_checkpoint'))); + expect(LocalDb.restoreTablesForTest, isNot(contains('store_identity'))); + expect(LocalDb.salvageTablesForTest, isNot(contains('store_identity'))); + }); + + test('GUARD disabled checkpoint observations are inert and output-neutral', () async { + await _night(); + final normal = DerivationEngine(); + await _derive(normal); + final expected = await edStored(edDay); + final disabled = DerivationEngine()..debugCheckpoints.disabled = true; + await _derive(disabled); + expect(await edStored(edDay), expected); + expect(disabled.debugCheckpoints.hits, 0); + expect(disabled.debugCheckpoints.writes, 0); + disabled.debugCheckpoints.reset(); + expect(disabled.debugCheckpoints.disabled, isTrue); + expect(disabled.debugCheckpoints.lastEncodedPayload, isNull); + expect(jsonEncode(disabled.debugCheckpoints.lastDecodedPayload), 'null'); + }); + for (final mode in ['success', 'later table failure', 'later archive-page failure', 'first failure']) { + test('E16 archive-only $mode rotates identity atomically with committed pages', () async { + final db = await _storage(); + await db.insert('day_checkpoint', _sampleRow()); + final before = await _rows(db); + final storeId = await LocalDb.storeIdentity(); + await db.execute('CREATE TRIGGER require_archive_identity BEFORE INSERT ON ' + '${LocalDb.rawArchiveTable} BEGIN SELECT CASE WHEN ' + "(SELECT id FROM store_identity WHERE singleton = 1) = '$storeId' " + "OR EXISTS(SELECT 1 FROM day_checkpoint) THEN RAISE(ABORT, 'old archive identity') END; END"); + if (mode == 'first failure' || mode == 'later archive-page failure') { + final refused = mode == 'first failure' ? 1 : 501; + await db.execute('CREATE TRIGGER refuse_archive_page BEFORE INSERT ON ' + '${LocalDb.rawArchiveTable} WHEN NEW.counter = $refused ' + "BEGIN SELECT RAISE(ABORT, 'archive page failure'); END"); + } + final dir = await Directory.systemTemp.createTemp('ed2b_archive_identity_'); + final path = p.join(dir.path, 'source.db'); + try { + final source = await databaseFactory.openDatabase(path); + await source.execute('CREATE TABLE raw_archive (hex TEXT PRIMARY KEY, ' + 'counter INTEGER, packet_type INTEGER, rec_ts INTEGER, captured_at INTEGER, reason TEXT)'); + final batch = source.batch(); + final count = mode == 'later archive-page failure' ? 501 : 1; + for (var i = 1; i <= count; i++) { + batch.insert('raw_archive', {'hex': i.toRadixString(16).padLeft(4, '0'), + 'counter': i, 'packet_type': 0, 'rec_ts': edStart, + 'captured_at': edStart * 1000, 'reason': 'identity_test'}); + } + await batch.commit(noResult: true); + if (mode == 'later table failure') { + await source.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await source.insert('baselines', {'key': 'missing-payload'}); + } + await source.close(); + if (mode == 'success') { + await LocalDb.importFromDbFile(path); + } else { + await expectLater(LocalDb.importFromDbFile(path), throwsA(isA())); + } + final copied = mode == 'first failure' ? 0 : + mode == 'later archive-page failure' ? 500 : 1; + expect(await db.query(LocalDb.rawArchiveTable), hasLength(copied)); + if (mode == 'first failure') { + expect(await LocalDb.storeIdentity(), storeId); + expect(await _rows(db), before); + } else { + expect(await LocalDb.storeIdentity(), isNot(storeId)); + expect(await db.query('day_checkpoint'), isEmpty); + } + } finally { + await databaseFactory.deleteDatabase(path); + await dir.delete(recursive: true); + } + }); + } + + test('E16 deleting a day removes every disposable raw prefix and keeps unrelated input', () async { + await _night(); + final db = await _storage(); + final other = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(other, other + 2); + await db.insert('day_checkpoint', _sampleRow()); + await db.insert('day_checkpoint', _sampleRow(day: '2020-01-03')); + final identity = await LocalDb.storeIdentity(); + const learning = '{"nights":7,"folded_days":["2020-01-01"]}'; + await LocalDb.putBaseline('sleep_user_profile', learning); + await LocalDb.deleteDays({edDay}); + expect(await db.query('day_checkpoint'), isEmpty, + reason: 'a neighbouring checkpoint can contain the deleted evening too'); + expect(await LocalDb.storeIdentity(), isNot(identity)); + expect(await db.query('decoded_rr'), isEmpty); + final remaining = await db.query('decoded_onehz'); + expect(remaining.map((row) => row['rec_ts']), [other, other + 1]); + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], learning, + reason: 'learned state cannot be rebuilt once old input is pruned'); + }); + + test('E16 selected-day export omits scratch and uses a fresh identity', () async { + final db = await _storage(); + await edRows(edStart, edStart + 2); + final other = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(other, other + 2); + await db.insert('day_checkpoint', _sampleRow()); + final identity = await LocalDb.storeIdentity(); + final dir = await Directory.systemTemp.createTemp('ed2b_selected_export_'); + final provider = PathProviderPlatform.instance; + PathProviderPlatform.instance = _CheckpointPaths(dir.path); + try { + final path = await LocalDb.exportDaysDb({edDay}); + final out = await databaseFactory.openDatabase(path, options: OpenDatabaseOptions(readOnly: true)); + try { + expect(await _hasCheckpoint(out), isTrue); + expect(await out.query('day_checkpoint'), isEmpty); + expect((await out.query('store_identity')).single['id'], isNot(identity)); + expect((await out.query('decoded_onehz')).map((r) => r['rec_ts']), [edStart, edStart + 1]); + expect(await LocalDb.storeIdentity(), identity); + expect(await _rows(db), hasLength(1)); + } finally { + await out.close(); + } + } finally { + PathProviderPlatform.instance = provider; + await dir.delete(recursive: true); + } + }); + + test('E16 checkpoint and RAM reuse cannot finalize or fingerprint a held night', () async { + await _night(); + final later = DateTime(2020, 1, 5, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(later, later + 1); + DerivationEngine.debugHoldTonight = {edDay}; + await _derive(DerivationEngine()); + expect(await _rows(await _storage()), hasLength(1)); + final engine = DerivationEngine(); + for (var i = 0; i < 2; i++) { + await engine.run(edProfile, heavy: true); + expect(await LocalDb.finalizedDayIds(kAlgoVersion), isNot(contains(edDay))); + expect(await LocalDb.deriveFingerprint(edDay), isNull); + } + expect(engine.debugCheckpoints.hits, greaterThan(0), + reason: 'the hold must survive actual disk adoption'); + DerivationEngine.debugHoldTonight = {}; + await engine.run(edProfile, heavy: true); + expect(await LocalDb.finalizedDayIds(kAlgoVersion), contains(edDay)); + }); + +} diff --git a/test/ed2b_d35_ownership_endpoint_test.dart b/test/ed2b_d35_ownership_endpoint_test.dart new file mode 100644 index 000000000..07eb87861 --- /dev/null +++ b/test/ed2b_d35_ownership_endpoint_test.dart @@ -0,0 +1,85 @@ +@Timeout(Duration(minutes: 5)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/adapters/signals.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/coverage_resolver.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +void main() { + setUpAll(edInit); + setUp(() => edFresh('ed2b_d35.db')); + tearDown(edDrop); + + for (final signal in [InputSignal.hr1Hz, InputSignal.rrIntervals]) { + test('$signal inclusive endpoint is covered by the ownership read and spans', () async { + final end = edStart + 600; + final db = await LocalDb.instance; + // This interval starts exactly at the inclusive last sample. A query + // ending at `end` silently omits it, even if resolution is later fixed. + await db.insert('device_coverage', {'device_id': '', 'signal': signal.name, + 'start_ts': end, 'end_ts': end + 1}); + final (ownership, _) = await DerivationEngine().debugResolveOwnership(edStart, end); + expect(spanAt(ownership[signal]!, end)?.deviceId, '', + reason: 'both coverage query and resolution must extend through to+1'); + expect(ownership[signal]!.last.end, end + 1); + }); + } + + test('non-owner last-second HR does not leak into a single-second load', () async { + await edRows(edStart, edStart + 601, hr: 61); + await edRows(edStart + 600, edStart + 601, hr: 199, id: 'unranked'); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': edStart, 'end_ts': edStart + 601}); + final e = DerivationEngine()..debugDisableSubstrateRetention = true; + final last = await e.debugLoadOwnedSubstrate(edStart + 600, edStart + 600, day: edDay); + expect(last.hr, [61], reason: 'a zero-width half-open resolver must not disable the filter'); + }); + + test('non-owner last-second RR beats are excluded by the same inclusive ownership', () async { + await edRows(edStart, edStart + 601, hr: 61); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'rrIntervals', + 'start_ts': edStart, 'end_ts': edStart + 601}); + for (final id in ['', 'unranked']) { + await db.insert('decoded_rr', {'device_id': id, + 'ts_ms': (edStart + 600) * 1000, 'rec_ts': edStart + 600, + 'rr_ts_ms': (edStart + 600) * 1000, 'beat_index': 0, + 'rr_ms': id.isEmpty ? 900 : 1500}); + } + final last = await DerivationEngine().debugLoadOwnedSubstrate( + edStart + 600, edStart + 600, day: edDay); + expect(last.rrMs, [900], reason: 'the other device must not add a beat at to'); + }); + + test('same endpoint loaded alone or inside a larger range has identical owner data', () async { + await edRows(edStart, edStart + 601, hr: 61); + await edRows(edStart + 600, edStart + 601, hr: 199, id: 'unranked'); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': edStart, 'end_ts': edStart + 601}); + final e = DerivationEngine()..debugDisableSubstrateRetention = true; + final wide = await e.debugLoadOwnedSubstrate(edStart, edStart + 601, day: edDay); + final alone = await e.debugLoadOwnedSubstrate(edStart + 600, edStart + 600, day: edDay); + expect(jsonEncode(alone.toJson()), + jsonEncode(wide.slice(edStart + 600, edStart + 601).toJson())); + }); + + test('interior ownership stays primary and absent owner stays absent (guard)', () async { + await edRows(edStart, edStart + 600, hr: 61); + await edRows(edStart, edStart + 600, hr: 199, id: 'unranked'); + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': edStart, 'end_ts': edStart + 600}); + final sub = await DerivationEngine().debugLoadOwnedSubstrate(edStart, edStart + 600, day: edDay); + expect(sub.hr, everyElement(61)); + final (ownership, _) = await DerivationEngine().debugResolveOwnership(edStart - 60, edStart - 1); + expect(ownership[InputSignal.hr1Hz], everyElement(predicate((s) => s.deviceId == null))); + }); +} diff --git a/test/ed2b_d36_target_profile_test.dart b/test/ed2b_d36_target_profile_test.dart new file mode 100644 index 000000000..843f67f51 --- /dev/null +++ b/test/ed2b_d36_target_profile_test.dart @@ -0,0 +1,195 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/sleep_profile_policy.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +final _mid = DateTime(2020, 1, 2).millisecondsSinceEpoch ~/ 1000; +int _h(int hour) => _mid + hour * 3600; + +// Longer previous night, shorter target, real motion and HR contrast. No +// forced/manual window: those deliberately cannot train the profile. +Substrate _night({bool target = true, bool earlierEvening = false}) { + final ts = [], hr = []; + final ax = [], ay = [], az = []; + for (var t = _h(-36); t < _h(30); t++) { + final still = (t >= _h(earlierEvening ? -12 : -26) && + t < _h(earlierEvening ? -2 : -17)) || + (target && t >= _h(-1) && t < _h(5)); + ts.add(t); + hr.add(still ? 52 + t % 7 : 80 + (t ~/ 60) % 20 + t % 3); + ax.add(still ? 0 : .3 * math.sin(t * .21)); + ay.add(still ? 0 : .2 * math.cos(t * .13)); + az.add(still ? 1 : 1 + .05 * math.sin(t * .07)); + } + return Substrate.fromJson({'ts_sec': ts, 'hr': hr, 'ax': ax, 'ay': ay, + 'az': az, 'device_family': 'gen4'}); +} + +String _observation(ana.SleepNightObservation o) => jsonEncode({ + 'epochs': o.epochs, 'hr_floor_p5': o.hrFloorP5, 'hr_floor_p25': o.hrFloorP25, + 'hr_sleep_median': o.hrSleepMedian, 'hr_arousal': o.hrArousal, + 'rmssd_med': o.rmssdMed, 'rmssd_mad': o.rmssdMad, + 'enmo_still_cut': o.enmoStillCut, 'enmo_move_cut': o.enmoMoveCut, + 'lfhf_med': o.lfhfMed, 'rk_med': o.rkMed}); + +Future _save(Substrate s) async { + final b = (await LocalDb.instance).batch(); + for (var i = 0; i < s.length; i++) { + b.insert('decoded_onehz', {'device_id': '', 'ts_ms': s.tsSec[i] * 1000, + 'rec_ts': s.tsSec[i], 'counter': i, 'hr': s.hr[i], 'ax': s.ax[i], + 'ay': s.ay[i], 'az': s.az[i], 'device_family': 'gen4'}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + await b.commit(noResult: true); +} + +void main() { + setUpAll(edInit); + setUp(() => edFresh('ed2b_d36.db')); + tearDown(() async { + ana.cardioRecordObservations = false; + ana.cardioUserProfile = null; + ana.resetCardioObservations(); + await edDrop(); + }); + + for (final target in [true, false]) { + test('recordTargetOnly: ${target ? "one target observation" : "no target means none"}', () { + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + final sub = _night(target: target); + // An existing exact restricted walk supplies an independent oracle; the + // full walk must retain history while excluding its training side effects. + calendarDays(sub, targetDay: edDay, skipUnusedDays: true); + final expected = ana.takeCardioObservations().map(_observation).toList(); + expect(expected.length, target ? 1 : 0, reason: 'fixture sanity'); + ana.resetCardioObservations(); + final days = calendarDays(sub, targetDay: edDay, skipUnusedDays: false, + recordTargetOnly: true); + expect(days.length, greaterThan(1), reason: 'history walk remains real'); + expect(ana.takeCardioObservations().map(_observation).toList(), expected, + reason: 'earlier and later walked nights must never train under the target ID'); + expect(ana.cardioRecordObservations, isTrue); + }); + } + + test('production mayFold worker returns the target observation, never the longest walked night', () async { + // Production's folding pass must use the same exact target-only walk as + // the nonfolding path, and report that target's own training input. + final sub = _night(); + await _save(sub); + final e = DerivationEngine(); + final range = e.debugTargetDayWindow(edDay); + final search = await e.debugLoadSubstrate(range.$1, range.$2, day: edDay); + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: true); + final target = ana.takeCardioObservations(); + expect(target.length, 1, reason: 'the target has a usable real night'); + final observed = []; + e.debugOnStagingWalk = (fold, count, observation) { + expect(fold, isTrue); + expect(count, 1, reason: 'target-only folding permits the E15 skip during training'); + observed.add(observation); + }; + await e.debugSleepCandidate(edDay); + expect(observed, [_observation(target.single)]); + }); + + test('production absent target never folds the earlier walked evening under its ID', () async { + await _save(_night(target: false, earlierEvening: true)); + final observations = []; + final e = DerivationEngine()..debugOnStagingWalk = + (fold, _, observation) { + expect(fold, isTrue); + observations.add(observation); + }; + final candidate = await e.debugSleepCandidate(edDay); + expect(candidate.present, isFalse, reason: 'the only sleep ended before target midnight'); + expect(observations, [null], reason: 'no target observation means no training'); + expect(await LocalDb.baseline('sleep_user_profile'), isNull); + }); + + test('already folded profile and folded_days survive version adoption byte-for-byte (guard)', () async { + await _save(_night()); + final profile = jsonEncode({...const ana.SleepUserProfile().toJson(), + 'nights': 7, 'folded_days': ['2019-12-31', edDay]}); + await LocalDb.putBaseline('sleep_user_profile', profile); + final before = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; + await DerivationEngine().debugSleepCandidate(edDay); + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], before, + reason: 'D36 is forward-only; existing accumulated evidence is not reset or rebuilt'); + }); + + test('fold flags restored when segmentation throws (guard)', () { + for (final initial in [true, false]) { + ana.cardioRecordObservations = initial; + expect(() => calendarDays(_night(), targetDay: edDay, recordTargetOnly: true, + priorSleep: [for (var i = 1; i <= 14; i++) + (startSec: _h(-24 * i - 1), endSec: _h(-24 * i + 5), + dayKey: '2019-12-${i.toString().padLeft(2, '0')}')], + tzOffsetAt: (_) => throw StateError('offset failure')), throwsStateError); + expect(ana.cardioRecordObservations, initial); + } + }); + + test('disabled recording remains disabled on successful target walk (guard)', () { + ana.cardioRecordObservations = false; + ana.resetCardioObservations(); + calendarDays(_night(), targetDay: edDay, recordTargetOnly: true); + expect(ana.cardioRecordObservations, isFalse); + expect(ana.takeCardioObservations(), isEmpty); + }); + + + test('actual transactional folds preserve explicit deterministic order and idempotent replay (guard)', () async { + final inputs = [ + ('2020-01-01', _observation(const ana.SleepNightObservation( + epochs: 480, hrFloorP5: 51, hrSleepMedian: 56))), + (edDay, _observation(const ana.SleepNightObservation( + epochs: 720, hrFloorP5: 58, hrSleepMedian: 63))), + ]; + final profiles = []; + for (var repeat = 0; repeat < 2; repeat++) { + final db = await LocalDb.instance; + await db.delete('baselines', where: 'key = ?', whereArgs: ['sleep_user_profile']); + final order = []; + final e = DerivationEngine()..debugOnProfileFold = order.add; + for (final input in inputs) { + await e.debugFoldObservation(input.$1, input.$2); + } + expect(order, inputs.map((input) => input.$1).toList()); + final before = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; + for (final input in inputs.reversed) { + await e.debugFoldObservation(input.$1, input.$2); + } + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], before, + reason: 'already folded nights leave the whole profile byte-identical'); + final profile = jsonDecode(before as String) as Map; + profile.remove('updated_at_ms'); // operational fold timestamp only + profiles.add(jsonEncode(profile)); + } + expect(profiles.last, profiles.first); + }); + + test('folded day bookkeeping is deterministic regardless of insertion order (guard)', () { + final a = SleepProfilePolicy.withFoldedDays({'nights': 8}, + {'2020-01-01', '2019-12-30'}, edDay); + final b = SleepProfilePolicy.withFoldedDays({'nights': 8}, + {'2019-12-30', '2020-01-01'}, edDay); + expect(jsonEncode(a), jsonEncode(b)); + expect(a['folded_days'], ['2019-12-30', '2020-01-01', edDay]); + }); +} diff --git a/test/ed2b_e14_union_loading_test.dart b/test/ed2b_e14_union_loading_test.dart new file mode 100644 index 000000000..c62758a8c --- /dev/null +++ b/test/ed2b_e14_union_loading_test.dart @@ -0,0 +1,128 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +String _payload(PreparedDerivationDay day) => jsonEncode({ + 'prepared': day.toJson(), 'bundle': edBundle(day)}); + +Future _night() async { + final start = DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000; + await edRows(start, start + 10 * 3600, hr: 55, az: 1); + await edRows(edStart, edStart + 1200, hr: 85); + final batch = (await LocalDb.instance).batch(); + // Multiple beats, beat-only seconds and backwards raw placements straddle + // every revision bucket; worker cuts must restart the running maximum. + for (var t = start; t < start + 10 * 3600; t += 17) { + for (var i = 0; i < 2; i++) { + batch.insert('decoded_rr', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'rr_ts_ms': t * 1000, 'beat_index': i, + 'beat_ts_ms': t * 1000 - (t % 11 == 0 ? 9000 : i * 400), + 'rr_ms': 900 + t % 90}, conflictAlgorithm: ConflictAlgorithm.replace); + } + } + await batch.commit(noResult: true); +} + +DerivationEngine _cold() => DerivationEngine() + ..debugDisableUnionLoads = true + ..debugDisableSubstrateRetention = true + ..debugDisableStagedSleepCache = true; + +void main() { + setUpAll(edInit); + setUp(() async { await edFresh('ed2b_e14.db'); await _night(); }); + tearDown(edDrop); + + test('E14 search, calendar, nap and night come from one union load with worker cuts', () async { + final loads = <(int, int, String)>[]; + final e = DerivationEngine()..debugDisableSubstrateRetention = true + ..debugOnSubstrateLoad = (a, b, label) => loads.add((a, b, label)); + final day = await e.debugPrepareDay(edDay); + expect(day, isNotNull); + expect(day!.sleepSub.isEmpty, isFalse, reason: 'fixture must reach the staged-night cut'); + final cold = await _cold().debugPrepareDay(edDay); + expect(_payload(day), _payload(cold!)); + expect(loads.length, 1, + reason: 'overlapping search/day/night rows must be loaded once as a union'); + expect(e.debugUnionWorkerCuts, greaterThanOrEqualTo(3), + reason: 'UI transfers encoded substrate; workers cut all requested windows'); + }); + + test('E14 neighbouring days borrow overlap rather than loading it twice', () async { + final next = DateTime(2020, 1, 3, 5).millisecondsSinceEpoch ~/ 1000; + await edRows(next, next + 1200, hr: 57); + final e = DerivationEngine(); + await e.debugPrepareDay(edDay); + final before = e.snapshot()['raw_rows'] as int; + final warm = await e.debugPrepareDay('2020-01-03'); + final cold = await _cold().debugPrepareDay('2020-01-03'); + expect(_payload(warm!), _payload(cold!)); + final reread = (e.snapshot()['raw_rows'] as int) - before; + final coldEngine = _cold(); + await coldEngine.debugPrepareDay('2020-01-03'); + expect(reread, lessThan(coldEngine.snapshot()['raw_rows'] as int), + reason: 'neighbour union must retain its earlier shared hours'); + }); + + final changes = Function(DerivationEngine)>{ + 'input revision': (_) => edRows(edStart, edStart + 1, hr: 91), + 'late RR': (_) async { + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': edStart * 1000, 'rec_ts': edStart, 'rr_ts_ms': edStart * 1000, + 'beat_index': 0, 'rr_ms': 850}, conflictAlgorithm: ConflictAlgorithm.replace); + }, + 'generation': (_) async { await LocalDb.close(); await LocalDb.instance; }, + 'algorithm': (e) async { e.debugRekeyUnionAlgorithm(kAlgoVersion - 1); }, + }; + for (final change in changes.entries) { + test('E14 ${change.key}: refuse stale union, cold bytes exact, then reuse worker cuts', () async { + final e = DerivationEngine()..debugDisableStagedSleepCache = true; + await e.debugPrepareDay(edDay); + final before = e.snapshot()['raw_rows'] as int; + await change.value(e); + final warm = await e.debugPrepareDay(edDay); + final cold = await _cold().debugPrepareDay(edDay); + expect(_payload(warm!), _payload(cold!)); + final refreshed = e.snapshot()['raw_rows'] as int; + expect(refreshed, greaterThan(before), + reason: 'changed provenance must reload rather than hit a stale union'); + final cuts = e.debugUnionWorkerCuts; + await e.debugPrepareDay(edDay); + expect(e.snapshot()['raw_rows'], refreshed, + reason: 'unchanged replacement provenance must reuse the refreshed union'); + expect(e.debugUnionWorkerCuts, greaterThan(cuts), + reason: 'valid refreshed union supplies worker cuts after invalidation'); + }); + } + + test('E14 D35 endpoint: union cuts agree with a separately owned single second', () async { + final db = await LocalDb.instance; + final e = DerivationEngine(); + final range = e.debugTargetDayWindow(edDay); + final end = range.$2; + await edRows(end, end + 1, hr: 61); + await edRows(end, end + 1, hr: 199, id: 'non-owner'); + await db.insert('device_coverage', {'device_id': '', 'signal': 'hr1Hz', + 'start_ts': range.$1, 'end_ts': end + 1}); + final day = await e.debugPrepareDay(edDay); + expect(day!.daySub.hr.last, 61); + final last = await DerivationEngine().debugLoadOwnedSubstrate(end, end, day: edDay); + expect(last.hr, [61], reason: 'D35 is a prerequisite for exact union cuts'); + expect(last.hr, day.daySub.slice(end, end + 1).hr); + }); + + test('E14 disabled union diagnostic is behavior-neutral (guard)', () async { + final ordinary = await DerivationEngine().debugPrepareDay(edDay); + final disabled = await _cold().debugPrepareDay(edDay); + expect(_payload(ordinary!), _payload(disabled!)); + }); +} diff --git a/test/ed2b_merged_analytics_test.dart b/test/ed2b_merged_analytics_test.dart new file mode 100644 index 000000000..6889a66c2 --- /dev/null +++ b/test/ed2b_merged_analytics_test.dart @@ -0,0 +1,110 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/cardio_rows_cache.dart'; +import 'package:openstrap_edge/compute/night_rr_state.dart'; + +void main() { + // WHY: a capped reset can put corrected NN behind its previous segment even + // with ordered raw timestamps. Merged batch RSA/CVHR abstain; their streams + // throw on that clock. One absent metric must not abort the whole day. + // Proves CLAIM(merged-clock-refusal). + // Catches: feeding a regressing corrected clock into retained RSA/CVHR. + test('merged clock refusals preserve a complete day and retained frequency', () { + const start = 1577930400; + double wobble(int i) => 1500.0 + (i % 7) * 12 - (i % 3) * 9; + // Analytics test/output_changes/review_inputs_test.dart's capped-reset + // fixture: 400 good beats, 33 artifacts, 400 good beats at 1.4 s cadence. + final rr = [for (var i = 0; i < 400; i++) wobble(i), + ...List.filled(33, 3000), + for (var i = 0; i < 400; i++) wobble(i)]; + final times = [for (var i = 0; i < rr.length; i++) start * 1000.0 + i * 1400]; + final diagnostics = Ed2bStreamDiagnostics(); + List? windows; + diagnostics.onRespWindows = (rates) => windows = rates; + for (final count in [400, 650, 833]) { + final input = DayBundleInput(date: '2020-01-02', dayTsSec: const [], + dayHr: const [], sleepTsSec: const [], sleepHr: const [], + sleepRrMs: rr.sublist(0, count), sleepRrTsMs: times.sublist(0, count), + sleepSkinTemp: const [], sleepJson: const {}, hypnoStages: const [], + sleepOnsetSec: start, sleepOffsetSec: start + 1200, + profile: const {}, deviceFamily: 'gen4').toJson(); + final cold = deriveDayBundle(input); + final retained = deriveDayBundle(input, streamDiagnostics: diagnostics); + final respiration = retained['respiration'] as Map; + expect(respiration['rsa'], (cold['respiration'] as Map)['rsa']); + expect(respiration['cvhr_apnea'], (cold['respiration'] as Map)['cvhr_apnea']); + expect((retained['clinical'] as Map)['hrv_freq'], + (cold['clinical'] as Map)['hrv_freq']); + if (count == 833) { + for (final key in ['rsa', 'cvhr_apnea']) { + expect((respiration[key] as Map)['value'], '—'); + expect((respiration[key] as Map)['note'], 'regressing beat time'); + } + final frequency = (retained['clinical'] as Map)['hrv_freq'] as Map; + expect(frequency['value'], isA()); + expect(frequency['note'], contains('left out')); + final c = ana.correctRr(rr, rrTsMs: times); + expect(windows, respPerWindow(c.nn, c.nnTimesMs, + artifactFraction: 1 - c.cleanFraction)); + // Cutting this prefix would otherwise try to fold the earlier valid + // RSA stream under the now-invalid clock. + expect(diagnostics.nightState.checkpoint(start + 900), isNull); + } + } + expect(diagnostics.processedNightBeats, rr.length); + expect(diagnostics.streamRestarts('frequency'), 1); + expect(diagnostics.retainedStreamBeats('frequency'), greaterThan(0)); + // Check that the fixture still exercises a real corrected-clock regression. + final corrected = ana.correctRr(rr, rrTsMs: times); + expect([for (var i = 1; i < corrected.nnTimesMs.length; i++) + corrected.nnTimesMs[i] < corrected.nnTimesMs[i - 1]], contains(true)); + }); + // WHY: analytics main changed codec policy. Old reusable work must cost a + // cold calculation, never load a result derived under the old pin. + for (final codec in ['frequency v1', 'PRSA without policy']) { + test('merged night checkpoint refuses $codec', () { + final state = NightRrState(); + final rr = List.filled(500, 950); + final times = [for (var i = 0; i < rr.length; i++) 1000000.0 + i * 950]; + state.correct(rr, times, day: '2020-01-02', onsetSec: 1000); + state.nightHrvFreq(0); + state.nightPrsa(); + final checkpoint = jsonDecode(jsonEncode(state.checkpoint(2000))) as Map; + expect(NightRrState.fromCheckpoint( + checkpoint.cast(), 2000).retainedNightBeats, 500); + if (codec == 'frequency v1') { + final frequency = checkpoint['frequency'] as Map; + frequency['version'] = 1; + frequency.remove('dropped'); + } else { + // PRSA's lookback inner now binds direction and anchor-ratio cap. + final dc = (checkpoint['prsa'] as Map)['dc'] as List; + final policy = (dc[1] as Map)['inner'] as Map; + policy.remove('deceleration'); + policy.remove('anchorRatioCap'); + } + expect(() => NightRrState.fromCheckpoint( + checkpoint.cast(), 2000), throwsFormatException); + }); + } + + // WHY: old word-wise fingerprints could alias after sign-bit changes. + // The merged codec invalidates those rows so staging rebuilds them. + test('merged cardio codec treats a v2 chunk as a cold miss', () { + final cache = ana.CardioRowCache()..insert( + const ana.CardioRowKey(1000, 30, 42), + const ana.CardioEpochRow(0, 60, 1, 20, 1, .5, 30)); + final json = cache.toJson(); + final current = Uint8List.fromList(utf8.encode(jsonEncode(json))); + expect(decodeCardioRows([current]).length, 1); + json['version'] = 2; + expect(() => ana.CardioRowCache.fromJson(json), throwsFormatException); + final bytes = Uint8List.fromList(utf8.encode(jsonEncode(json))); + expect(decodeCardioRows([bytes]).length, 0); + }); +} diff --git a/test/ed2b_streams_test.dart b/test/ed2b_streams_test.dart new file mode 100644 index 000000000..4de8c4931 --- /dev/null +++ b/test/ed2b_streams_test.dart @@ -0,0 +1,361 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:isolate'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/aggregate.dart' as agg; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/ed2b_stream_diagnostics.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/night_resp_bins.dart'; + +// Deliberately no fork DayCalculationState, mode or revision framework. The +// optional diagnostics observe the real upstream batch calls until GREEN. +Map _input({int beats = 950, bool gaps = false, + bool artifacts = false}) { + const start = 1577930400; + final rr = [], times = []; + var t = start * 1000.0; + for (var i = 0; i < beats; i++) { + final x = 970 + 38 * math.sin(i * .071) + 24 * math.sin(i * 1.31); + rr.add(artifacts && i >= 430 && i < 465 ? 2400.0 : x); + times.add(t); + t += x; + if (gaps && i == 510) t += 90000; + } + return DayBundleInput(date: '2020-01-02', dayTsSec: const [], + dayHr: const [], sleepTsSec: const [], sleepHr: const [], + sleepRrMs: rr, sleepRrTsMs: times, sleepSkinTemp: const [], + sleepJson: const {}, hypnoStages: const [], sleepOnsetSec: start, + sleepOffsetSec: t ~/ 1000 + 1, profile: const {}, + deviceFamily: 'gen4').toJson(); +} + +Map _prefix(Map input, int beats) => { + ...input, + 'sleep_rr_ms': (input['sleep_rr_ms'] as List).take(beats).toList(), + 'sleep_rr_ts_ms': (input['sleep_rr_ts_ms'] as List).take(beats).toList(), +}; + +String _wire(Object? value) => jsonEncode(value); +List _rr(Map x) => x['sleep_rr_ms'] as List; +List _times(Map x) => x['sleep_rr_ts_ms'] as List; + +void _near(Object? got, Object? want, [String path = r'$']) { + if (got is num && want is num) { + // #99 PRSA profile/capacity abs+relative 1e-9; confidence abs 1e-12. + final tol = path.endsWith('.confidence') ? 1e-12 : + 1e-9 + 1e-9 * want.abs(); + expect((got - want).abs(), lessThanOrEqualTo(tol), reason: path); + } else if (want is Map) { + expect(got, isA(), reason: path); + final g = got as Map; + expect(g.keys.toList(), want.keys.toList(), reason: path); + for (final k in want.keys) { _near(g[k], want[k], '$path.$k'); } + } else if (want is List) { + expect(got, isA(), reason: path); + final g = got as List; + expect(g.length, want.length, reason: path); + for (var i = 0; i < want.length; i++) { _near(g[i], want[i], '$path[$i]'); } + } else { expect(got, want, reason: path); } +} + +void _oracles(Map input, Map bundle) { + final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); + final nn = c.nn, ts = c.nnTimesMs; + final af = (1 - c.cleanFraction).clamp(0.0, 1.0); + final h = ana.HrvFreqStream(), r = ana.RsaRespStream(), v = ana.CvhrStream(); + for (var i = 0; i < nn.length; i += 73) { + final end = math.min(i + 73, nn.length); + final a = nn.sublist(i, end), b = ts.sublist(i, end); + h.fold(a, b); r.fold(a, b); v.fold(a, b); + } + final clinical = bundle['clinical'] as Map; + final respiration = bundle['respiration'] as Map; + if (nn.length >= 20) { + final batch = ana.hrvFreq(nn, ts, artifactFraction: af).toJson((x) => x.toJson()); + expect(_wire(h.snapshot(artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + expect(_wire(clinical['hrv_freq']), _wire(batch)); + } + if (nn.length >= 30) { + final batch = ana.rsaRespRate(nn, ts, artifactFraction: af).toJson((x) => x.toJson()); + expect(_wire(r.snapshot(artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + expect(_wire(respiration['rsa']), _wire(batch)); + } + if (nn.length >= 60) { + final batch = ana.cvhrApneaScreen(nn, ts, artifactFraction: af).toJson((x) => x.toJson()); + expect(_wire(v.snapshot(artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + expect(_wire(respiration['cvhr_apnea']), _wire(batch)); + // #99 contains 914bb6a: an entirely provisional tail on a FRESH stream. + expect(_wire(ana.CvhrStream().snapshot(tailNnMs: nn, + tailNnTimesMs: ts, artifactFraction: af).toJson((x) => x.toJson())), _wire(batch)); + } + for (final deceleration in [true, false]) { + final a = agg.prsaAggregator(deceleration: deceleration); + var state = a.empty(); + final column = agg.DoubleColumn(Float64List.fromList(nn)); + for (var i = 0; i < nn.length; i += 37) { + state = a.addSlice(state, column, i, math.min(i + 37, nn.length)); + } + final batch = (deceleration ? ana.decelerationCapacity(nn) : + ana.accelerationCapacity(nn)).toJson((x) => x.toJson()); + _near(a.finish(state).toJson((x) => x.toJson()), batch); + _near(clinical[deceleration ? 'prsa_dc' : 'prsa_ac'], batch); + } +} + +void _bundleSame(Map got, Map cold) { + // #99 documents PRSA capacity/profile abs+relative 1e-9 and confidence + // abs 1e-12. Every other bundle byte, including notes and absence, is exact. + final gc = Map.from(got['clinical'] as Map); + final cc = Map.from(cold['clinical'] as Map); + for (final k in ['prsa_dc', 'prsa_ac']) { _near(gc.remove(k), cc.remove(k)); } + expect(_wire({...got, 'clinical': gc}), _wire({...cold, 'clinical': cc})); +} + +void _exact(Map input, Ed2bStreamDiagnostics diagnostics) { + final cold = deriveDayBundle(input); + final got = deriveDayBundle(input, streamDiagnostics: diagnostics); + _bundleSame(got, cold); + _oracles(input, got); +} + +void main() { + for (final style in ['smooth', 'gaps + capped artifact run']) { + final input = _input(gaps: style != 'smooth', artifacts: style != 'smooth'); + test('E7 $style chunked night NN is batch-exact and folds each beat once', () { + final d = Ed2bStreamDiagnostics(); + var observed = false; + d.onCorrected = (nn, times, fraction) { + observed = true; + final c = ana.correctRr(_rr(current), rrTsMs: _times(current)); + expect(_wire(nn), _wire(c.nn)); expect(_wire(times), _wire(c.nnTimesMs)); + expect(fraction, c.cleanFraction); + }; + // Cuts straddle the 248-beat capped corrector's provisional tail and + // the >32 artifact run. No cut is rounded to an epoch boundary. + for (final n in [1, 91, 247, 248, 249, 431, 464, 465, 701, 950]) { + current = _prefix(input, n); + _exact(current, d); + } + expect(observed, isTrue); + expect(d.processedNightBeats, _rr(input).length, + reason: 'RAM corrector must process only the appended raw beats'); + expect(d.retainedNightBeats, greaterThan(0)); + expect(d.nightRebuilds, 1); + }); + + for (final stream in ['frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) { + final contract = {'frequency': 'E8', 'rsa': 'E8', 'cvhr': 'E9', + 'prsa': 'E10', 'rsa_bins': 'E10'}[stream]; + test('$contract $style $stream streams across growing and repeated passes', () { + final d = Ed2bStreamDiagnostics(); + for (final n in [300, 641, 950]) { _exact(_prefix(input, n), d); } + final work = d.processedBeats(stream); + expect(work, greaterThan(0), reason: 'real metric path must be exercised'); + _exact(input, d); + expect(d.processedBeats(stream), work, + reason: 'repeat must reuse settled stream state'); + expect(d.retainedStreamBeats(stream), greaterThan(0)); + expect(d.streamRestarts(stream), 1); + }); + } + } + + for (final feature in ['rr', 'frequency', 'rsa', 'cvhr', 'prsa', 'rsa_bins']) { + for (final key in ['input revision', 'store generation', 'algorithm']) { + test('E7-E10 $feature refuses changed $key, equals cold, then reuses replacement', () { + final d = Ed2bStreamDiagnostics(); + final input = _input(); + d.context(inputRevision: 1, generation: 1, algorithm: 114); + _exact(input, d); + final before = feature == 'rr' ? d.processedNightBeats : d.processedBeats(feature); + d.context(inputRevision: key == 'input revision' ? 2 : 1, + generation: key == 'store generation' ? 2 : 1, + algorithm: key == 'algorithm' ? 115 : 114); + _exact(input, d); + final miss = feature == 'rr' ? d.processedNightBeats : d.processedBeats(feature); + expect(miss, greaterThan(before), reason: 'changed context must reject retained work'); + _exact(input, d); + expect(feature == 'rr' ? d.processedNightBeats : d.processedBeats(feature), miss, + reason: 'the replacement context must subsequently hit'); + }); + } + } + + for (final mutation in ['old value', 'old timestamp', 'shorter prefix', 'shifted onset']) { + test('E7-E10 $mutation rebuilds the night and preserves every bundle byte', () { + final d = Ed2bStreamDiagnostics(); + final input = _input(); _exact(input, d); + final changed = _prefix(input, mutation == 'shorter prefix' ? 700 : 950); + if (mutation == 'old value') (_rr(changed))[100] += 7; + if (mutation == 'old timestamp') (_times(changed))[100] += .125; + if (mutation == 'shifted onset') { + changed['sleep_rr_ms'] = _rr(changed).sublist(10); + changed['sleep_rr_ts_ms'] = _times(changed).sublist(10); + changed['sleep_onset_sec'] = (changed['sleep_onset_sec'] as int) + 10; + } + _exact(changed, d); + final miss = d.processedNightBeats; + _exact(changed, d); + expect(d.processedNightBeats, miss, reason: 'changed prefix must be retained after rebuild'); + expect(d.nightRebuilds, 2); + }); + } + + test('E7 RAM night owner survives a real isolate hop without changed output', () async { + final d = Ed2bStreamDiagnostics(); final input = _input(); + deriveDayBundle(input, streamDiagnostics: d); + final before = d.processedNightBeats; + final hopped = await Isolate.run(() { + final bundle = deriveDayBundle(input, streamDiagnostics: d); + return (bundle, d); + }); + _bundleSame(hopped.$1, deriveDayBundle(input)); + expect(hopped.$2.processedNightBeats, before); + expect(hopped.$2.retainedNightBeats, greaterThan(0)); + }); + + test('guard: #99 capped RR settles and snapshots equal batch at every chunk', () { + final input = _input(gaps: true, artifacts: true); + final stream = ana.RrCorrector(); + final settled = [], times = []; + var previous = 0; + for (final n in [1, 91, 247, 248, 249, 431, 464, 465, 701, 950]) { + final next = stream.fold(_rr(input).sublist(previous, n), + tsMs: _times(input).sublist(previous, n)); + settled.addAll(next.nn); times.addAll(next.nnTimes); + final tail = stream.snapshot(); + final batch = ana.correctRr(_rr(input).sublist(0, n), + rrTsMs: _times(input).sublist(0, n)); + expect(_wire([...settled, ...tail.tailNn]), _wire(batch.nn)); + expect(_wire([...times, ...tail.tailNnTimes]), _wire(batch.nnTimesMs)); + previous = n; + } + }); + + test('E11 exceptional exit releases occupied identity memo and retry is fresh', () { + final d = Ed2bStreamDiagnostics(); final input = _input(beats: 2300); + var digestsAtExit = 0, memoAtExit = 0; + d.onBeforePassReturn = () { + digestsAtExit = d.fullListDigests; memoAtExit = d.identityMemoEntries; + throw StateError('injected after the actual analytics pass'); + }; + expect(() => deriveDayBundle(input, streamDiagnostics: d), throwsStateError); + expect(digestsAtExit, greaterThan(0), reason: 'must exercise actual hash work before failure'); + expect(memoAtExit, greaterThan(0), reason: 'must fail while the pass owns real identity keys'); + expect(d.identityMemoEntries, 0, reason: 'finally must release lists after exceptions'); + d.onBeforePassReturn = null; + _rr(input)[90] += 13; _exact(input, d); + expect(d.identityMemoEntries, 0); + }); + + for (final key in ['input revision', 'generation', 'algorithm']) { + test('E11 $key never reuses a preceding pass identity digest', () { + final input = _input(beats: 2300), d = Ed2bStreamDiagnostics(); + d.context(inputRevision: 1, generation: 1, algorithm: 114); + _exact(input, d); final first = d.fullListDigests; + d.context(inputRevision: key == 'input revision' ? 2 : 1, + generation: key == 'generation' ? 2 : 1, + algorithm: key == 'algorithm' ? 115 : 114); + _rr(input)[89] += 9; _exact(input, d); + expect(d.fullListDigests, greaterThan(first)); + expect(d.maximumListDigestsPerIdentity, 1); + expect(d.identityMemoEntries, 0); + }); + } + + test('E10 RSA bins cross 30-minute edges; sequential PRSA is batch-toleranced', () { + final input = _input(beats: 4000, gaps: true); + final d = Ed2bStreamDiagnostics(); + List? actualBins; + d.onRespWindows = (rates) { actualBins = rates; }; + _exact(_prefix(input, 1861), d); _exact(input, d); + final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); + final bins = , List)>{}; + for (var i = 0; i < c.nn.length; i++) { + final key = ((c.nnTimesMs[i] - c.nnTimesMs.first) / 1800000).floor(); + final b = bins.putIfAbsent(key, () => ([], [])); + b.$1.add(c.nn[i]); b.$2.add(c.nnTimesMs[i]); + } + final rates = []; + for (final b in bins.values) { + if (b.$1.length < 60) continue; + final stream = ana.RsaRespStream(); + for (var i = 0; i < b.$1.length; i += 79) { + final end = math.min(i + 79, b.$1.length); + stream.fold(b.$1.sublist(i, end), b.$2.sublist(i, end)); + } + final batch = ana.rsaRespRate(b.$1, b.$2, artifactFraction: 0); + expect(_wire(stream.snapshot(artifactFraction: 0).toJson((v) => v.toJson())), + _wire(batch.toJson((v) => v.toJson()))); + final rate = batch.present ? batch.value!.brpm : null; + if (rate != null) rates.add(rate); + } + expect(_wire(actualBins), _wire(rates)); + expect(bins.length, greaterThanOrEqualTo(3)); + expect(d.slicesProcessed('rsa_bins'), greaterThanOrEqualTo(3)); + final before = d.processedBeats('rsa_bins'); _exact(input, d); + expect(d.processedBeats('rsa_bins'), before); + expect(d.retainedStreamBeats('rsa_bins'), greaterThan(0)); + expect(d.retainedStreamBeats('prsa'), greaterThan(0)); + }); + + test('E11 shared long night lists are hashed once each per pass and released', () { + final d = Ed2bStreamDiagnostics(); final input = _input(beats: 2300); + _exact(input, d); + expect(d.fullListDigests, greaterThan(0), reason: 'real dependency proofs must be measured'); + expect(d.maximumListDigestsPerIdentity, 1); + expect(d.identityMemoEntries, 0, reason: 'identity keys must not pin worker arrays between passes'); + final before = d.fullListDigests; + // Reuse the SAME input identities, mutate content between passes. A digest + // surviving endPass would conceal the changed old prefix. + _rr(input)[100] += 11; _exact(input, d); + expect(d.fullListDigests, greaterThan(before)); + expect(d.maximumListDigestsPerIdentity, 1); expect(d.identityMemoEntries, 0); + }); + + test('guard: diagnostics enabled or disabled never changes bundle bytes', () { + final input = _input(gaps: true, artifacts: true); + final d = Ed2bStreamDiagnostics()..disabled = true; + _exact(input, d); d.disabled = false; _exact(input, d); + }); + + test('guard: empty and thin inputs preserve absence and no identity retention', () { + for (final n in [0, 1, 19, 29, 59]) { + final input = _input(beats: n); final d = Ed2bStreamDiagnostics(); + final got = deriveDayBundle(input, streamDiagnostics: d); + expect(_wire(got), _wire(deriveDayBundle(input))); + expect(d.identityMemoEntries, 0); + if (n == 0) expect(d.processedNightBeats, 0); + } + }); + test('E10 closed RSA bins reapply the night artifact gate after append and JSON restart', () { + final input = _input(beats: 6200); + final c = ana.correctRr(_rr(input), rrTsMs: _times(input)); + final bins = NightRespBins(); + bins.fold(c.nn.sublist(0, 4000), c.nnTimesMs.sublist(0, 4000)); + List batch(double artifacts) => respPerWindow(c.nn, c.nnTimesMs, + artifactFraction: artifacts); + // A positive control ensures the artifact gate, rather than absent signal, + // is the only reason a high-artifact snapshot has no rates. + expect(batch(0), isNotEmpty); + bins.fold(c.nn.sublist(4000), c.nnTimesMs.sublist(4000)); + expect(bins.snapshot(const [], const [], artifactFraction: 0), batch(0)); + expect(batch(.5), isEmpty); + expect(bins.snapshot(const [], const [], artifactFraction: .5), batch(.5)); + final saved = (jsonDecode(jsonEncode(bins.toJson())) as Map).cast(); + final restored = NightRespBins.fromJson(saved, c.nn.length); + expect(restored.snapshot(const [], const [], artifactFraction: .5), batch(.5)); + expect(restored.snapshot(const [], const [], artifactFraction: 0), batch(0)); + }); + +} + +// Assigned immediately before each synchronous derive; used only to inspect +// the callback's actual corrected arrays without adding them to output JSON. +Map current = {}; diff --git a/test/fixtures/two_device_day_expected.json b/test/fixtures/two_device_day_expected.json index db883f8d2..a682b3071 100644 --- a/test/fixtures/two_device_day_expected.json +++ b/test/fixtures/two_device_day_expected.json @@ -3895,7 +3895,7 @@ "skin_temp_adc": 30005.0, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 14.14213555551533, + "sdnn": 14.142135555515845, "sdnn_window": 14.164940954653554, "prsa_dc": 15.0, "prsa_dc_anchors": 2879.0, diff --git a/test/fixtures/whoop_freeze/gen4_minus0400.json b/test/fixtures/whoop_freeze/gen4_minus0400.json index 0cbf310b9..8ba1db57b 100644 --- a/test/fixtures/whoop_freeze/gen4_minus0400.json +++ b/test/fixtures/whoop_freeze/gen4_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 227.5841832798056, + "value": 227.58418327980579, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23019,12 +23019,12 @@ "skin_temp_adc": 30405.99974110511, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -24898,11 +24898,11 @@ "readiness": 27.033995461954163, "skipped": 0, "partial": 0, - "payload_sha256": "b57127eaba2e220875324219ed002d3801c330506d1303b87a413572f89ec560" + "payload_sha256": "62a201619a8a80c7a58a4f8feefe51b9b9b17b71fb395a35456b5f06c2705908" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25132,7 +25132,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 221.15132594116318, + "value": 221.15132594116335, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47739,12 +47739,12 @@ "skin_temp_adc": 30405.999593165176, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49591,11 +49591,11 @@ "readiness": 27.03409387728019, "skipped": 0, "partial": 0, - "payload_sha256": "c665785d82540466a55ea1cbce98b9c46288e0fec0fb90c64878f908412e0e2e" + "payload_sha256": "52604af7568a6615643226ce4d7f339e594e6e9ef256b6e9d9720ae3ae91d98f" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -49831,7 +49831,7 @@ "note": "need_baseline:have=2,need=3" }, "trimp": { - "value": 218.15549360346512, + "value": 218.15549360346523, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -71054,12 +71054,12 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 29.23441483806156, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -72717,7 +72717,7 @@ "readiness": 66.11544177079463, "skipped": 0, "partial": 0, - "payload_sha256": "274e77c395da40e489da25b343bbe71e207dcd4fbe51571a521b6ef3d9a42c90" + "payload_sha256": "339b824c3825df771b722599c2b712a19fb697c719ccb46cb77deef5ec91fb2b" } ], "metric_series": [ @@ -72864,7 +72864,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -72909,7 +72909,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 227.5841832798056 + "value": 227.58418327980579 }, { "date": "2026-07-13", @@ -73074,7 +73074,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73119,7 +73119,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 221.15132594116318 + "value": 221.15132594116335 }, { "date": "2026-07-14", @@ -73284,7 +73284,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73329,7 +73329,7 @@ { "date": "2026-07-15", "key": "trimp", - "value": 218.15549360346512 + "value": 218.15549360346523 }, { "date": "2026-07-15", @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -76995,7 +76995,7 @@ "skin_temp_adc": 30405.99974110511, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "strain": null, "steps": null, "nap_min": 0.0, @@ -77208,7 +77208,7 @@ "skin_temp_adc": 30405.999593165176, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "strain": null, "steps": null, "nap_min": 45.0, @@ -77421,7 +77421,7 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "strain": null, "steps": null, "nap_min": 0.0, diff --git a/test/fixtures/whoop_freeze/gen4_plus0530.json b/test/fixtures/whoop_freeze/gen4_plus0530.json index 192d21684..3355c1d46 100644 --- a/test/fixtures/whoop_freeze/gen4_plus0530.json +++ b/test/fixtures/whoop_freeze/gen4_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 227.5841832798056, + "value": 227.58418327980579, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23019,12 +23019,12 @@ "skin_temp_adc": 30406.0, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -24898,11 +24898,11 @@ "readiness": 27.03382323567632, "skipped": 0, "partial": 0, - "payload_sha256": "cda75458f39702ac4580c5d03d4bf6a6b5c5120aee516e8c32d1c6fe33383def" + "payload_sha256": "9df2d6b463c1d4668fe083d815dde535441090b667f7923c7601a80b2cd98cc0" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25132,7 +25132,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 221.15132594116318, + "value": 221.15132594116335, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47739,12 +47739,12 @@ "skin_temp_adc": 30406.000332864856, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49591,11 +49591,11 @@ "readiness": 27.033601802905416, "skipped": 0, "partial": 0, - "payload_sha256": "3228493c5545d3a4ee5f2da9410317da44cf8e9947d17b049572ca698c63112d" + "payload_sha256": "2cece0333903e0338a57f753ae55610073e575f4bd1931f74f6555b982237c15" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -49831,7 +49831,7 @@ "note": "need_baseline:have=2,need=3" }, "trimp": { - "value": 218.15549360346512, + "value": 218.15549360346523, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -71054,12 +71054,12 @@ "skin_temp_adc": 30406.00018492492, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 29.23441483806156, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -72717,7 +72717,7 @@ "readiness": 66.1155535435143, "skipped": 0, "partial": 0, - "payload_sha256": "bf1c2270ea868effc012ed7f83c834eac0f08e642bbde8aa6894e8aa2b394d18" + "payload_sha256": "a29b3dd5f3759041399c9ac0681eb643c449ba95e59b96b1a1a993d6fcee53c7" } ], "metric_series": [ @@ -72864,7 +72864,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -72909,7 +72909,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 227.5841832798056 + "value": 227.58418327980579 }, { "date": "2026-07-13", @@ -73074,7 +73074,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73119,7 +73119,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 221.15132594116318 + "value": 221.15132594116335 }, { "date": "2026-07-14", @@ -73284,7 +73284,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73329,7 +73329,7 @@ { "date": "2026-07-15", "key": "trimp", - "value": 218.15549360346512 + "value": 218.15549360346523 }, { "date": "2026-07-15", @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -76995,7 +76995,7 @@ "skin_temp_adc": 30406.0, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "strain": null, "steps": null, "nap_min": 0.0, @@ -77208,7 +77208,7 @@ "skin_temp_adc": 30406.000332864856, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "strain": null, "steps": null, "nap_min": 45.0, @@ -77421,7 +77421,7 @@ "skin_temp_adc": 30406.00018492492, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "strain": null, "steps": null, "nap_min": 0.0, diff --git a/test/fixtures/whoop_freeze/gen4_utc.json b/test/fixtures/whoop_freeze/gen4_utc.json index 9688b6b32..1073f8a70 100644 --- a/test/fixtures/whoop_freeze/gen4_utc.json +++ b/test/fixtures/whoop_freeze/gen4_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 227.5841832798056, + "value": 227.58418327980579, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23019,12 +23019,12 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -24898,11 +24898,11 @@ "readiness": 27.03355259577805, "skipped": 0, "partial": 0, - "payload_sha256": "f13bcc5af9185bb1b8384a9bc439a551a03bc4367ea2fba85c5742ef910df9d8" + "payload_sha256": "a9027226526086f68120d3401a25663926b07ff7515415e7e4f25ccd592e12ac" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25132,7 +25132,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 221.15132594116318, + "value": 221.15132594116335, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47739,12 +47739,12 @@ "skin_temp_adc": 30406.00025889489, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49591,11 +49591,11 @@ "readiness": 27.033651010089166, "skipped": 0, "partial": 0, - "payload_sha256": "411649f61ac1083a68de60dd1ff589276d011a075e7d22d81f88d4a692c5346a" + "payload_sha256": "f171df77c61ba6ac83f14a9b68516fe1bf2fc433463febe2aaff4eed6531c827" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -49831,7 +49831,7 @@ "note": "need_baseline:have=2,need=3" }, "trimp": { - "value": 218.15549360346512, + "value": 218.15549360346523, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -71054,12 +71054,12 @@ "skin_temp_adc": 30406.000110954952, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": 1.0, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 29.23441483806156, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -72717,7 +72717,7 @@ "readiness": 66.11559080104756, "skipped": 0, "partial": 0, - "payload_sha256": "5e74cc212eca9dfc9b06827c32ad9b70b18ea48b899b34eaef557741382b7143" + "payload_sha256": "7ce4b9c113c012b9e87932da751c13e2c985386e077c114c17caa6f7a158fdc8" } ], "metric_series": [ @@ -72864,7 +72864,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -72909,7 +72909,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 227.5841832798056 + "value": 227.58418327980579 }, { "date": "2026-07-13", @@ -73074,7 +73074,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73119,7 +73119,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 221.15132594116318 + "value": 221.15132594116335 }, { "date": "2026-07-14", @@ -73284,7 +73284,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73329,7 +73329,7 @@ { "date": "2026-07-15", "key": "trimp", - "value": 218.15549360346512 + "value": 218.15549360346523 }, { "date": "2026-07-15", @@ -73355,7 +73355,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73363,7 +73363,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73371,7 +73371,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen4", "coverage_devices": "", @@ -73381,7 +73381,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -74589,7 +74589,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -75797,7 +75797,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -76995,7 +76995,7 @@ "skin_temp_adc": 30406.000406834824, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 227.5841832798056, + "trimp": 227.58418327980579, "strain": null, "steps": null, "nap_min": 0.0, @@ -77208,7 +77208,7 @@ "skin_temp_adc": 30406.00025889489, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 221.15132594116318, + "trimp": 221.15132594116335, "strain": null, "steps": null, "nap_min": 45.0, @@ -77421,7 +77421,7 @@ "skin_temp_adc": 30406.000110954952, "skin_temp_family": "gen4", "skin_temp_illness_ok": false, - "trimp": 218.15549360346512, + "trimp": 218.15549360346523, "strain": null, "steps": null, "nap_min": 0.0, diff --git a/test/fixtures/whoop_freeze/gen5_minus0400.json b/test/fixtures/whoop_freeze/gen5_minus0400.json index 9d36e9dea..1fffab38b 100644 --- a/test/fixtures/whoop_freeze/gen5_minus0400.json +++ b/test/fixtures/whoop_freeze/gen5_minus0400.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 267.9104825050325, + "value": 267.9104825050327, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23020,12 +23020,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -25076,11 +25076,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "a3f73d5823f32766fd6f8741f46e2a72ac064ea73b044321b9ebb50480b75058" + "payload_sha256": "7324cb5f4edff114920b2fa3d4ba97cb974cd712eadbbd27991ebd940299011c" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25310,7 +25310,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 260.0235516950477, + "value": 260.02355169504744, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47918,12 +47918,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49942,11 +49942,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "44312e737e4881671c0f43e00ddfbb7bc0c4b59a578ad331e9fcc9d850f34936" + "payload_sha256": "b0b0de9fba0fe2e9ba7e6cadc2eb602ea27798154e12c66488143b0a603c3382" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -71400,7 +71400,7 @@ "skin_temp_dev_c": 0.1, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, @@ -73230,7 +73230,7 @@ "readiness": 61.79377627786914, "skipped": 0, "partial": 0, - "payload_sha256": "52208554e4cb1c08d66b6b7467560e007151d583ea6155dc3327496f18b87e17" + "payload_sha256": "6c93cfdfb231c5b9c02369f29df360cbced3fda2c908355c6cd22314c53e6572" } ], "metric_series": [ @@ -73377,7 +73377,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -73422,7 +73422,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 267.9104825050325 + "value": 267.9104825050327 }, { "date": "2026-07-13", @@ -73587,7 +73587,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73632,7 +73632,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 260.0235516950477 + "value": 260.02355169504744 }, { "date": "2026-07-14", @@ -73797,7 +73797,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -77507,7 +77507,7 @@ "skin_temp_z": 3.808966711097082, "skin_temp_adc": 30405.99974110511, "skin_temp_family": "gen5", - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "strain": null, "steps": 18240.0, "nap_min": 0.0, @@ -77719,7 +77719,7 @@ "skin_temp_z": 2.581920140166515, "skin_temp_adc": 30405.999593165176, "skin_temp_family": "gen5", - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "strain": null, "steps": 17640.0, "nap_min": 45.0, diff --git a/test/fixtures/whoop_freeze/gen5_plus0530.json b/test/fixtures/whoop_freeze/gen5_plus0530.json index 69eb2e248..18e58f2b8 100644 --- a/test/fixtures/whoop_freeze/gen5_plus0530.json +++ b/test/fixtures/whoop_freeze/gen5_plus0530.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 267.9104825050325, + "value": 267.9104825050327, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23020,12 +23020,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -25076,11 +25076,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "a6bb812440c40efd53dc915e78474d6581b354a787fd83f1d8d82274e94b052b" + "payload_sha256": "9f3dde3ed4f3d2bd7988104ed1312ffd3bb78cf2f9c0dc6c3bbd949763c917be" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25310,7 +25310,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 260.0235516950477, + "value": 260.02355169504744, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47918,12 +47918,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49942,11 +49942,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "e6ee25535499c542ebd13b8a1df85566c229944efba4dbdb8a44ca980c6442dc" + "payload_sha256": "785a07813a59e1e553b90599fbc43708ad5cf6063202796c1777f11aa886aa18" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -71400,7 +71400,7 @@ "skin_temp_dev_c": 0.1, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, @@ -73230,7 +73230,7 @@ "readiness": 61.79377627786914, "skipped": 0, "partial": 0, - "payload_sha256": "95786f495b2c7013ed03c36aaf66f0fd7018dbb42776f145314869085111d68e" + "payload_sha256": "be6cda6e9bd97c996db7f60f380433e10f15b05d35949248b39ad35d530a420b" } ], "metric_series": [ @@ -73377,7 +73377,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -73422,7 +73422,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 267.9104825050325 + "value": 267.9104825050327 }, { "date": "2026-07-13", @@ -73587,7 +73587,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73632,7 +73632,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 260.0235516950477 + "value": 260.02355169504744 }, { "date": "2026-07-14", @@ -73797,7 +73797,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 240, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -77507,7 +77507,7 @@ "skin_temp_z": 3.8090587513754603, "skin_temp_adc": 30406.0, "skin_temp_family": "gen5", - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "strain": null, "steps": 18240.0, "nap_min": 0.0, @@ -77719,7 +77719,7 @@ "skin_temp_z": 2.5820748428079483, "skin_temp_adc": 30406.000332864856, "skin_temp_family": "gen5", - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "strain": null, "steps": 17640.0, "nap_min": 45.0, diff --git a/test/fixtures/whoop_freeze/gen5_utc.json b/test/fixtures/whoop_freeze/gen5_utc.json index 2ed458474..6bff9a346 100644 --- a/test/fixtures/whoop_freeze/gen5_utc.json +++ b/test/fixtures/whoop_freeze/gen5_utc.json @@ -2,7 +2,7 @@ "day_result": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-13", "day_confidence": 0.6, @@ -232,7 +232,7 @@ "note": "need_baseline:have=0,need=3" }, "trimp": { - "value": 267.9104825050325, + "value": 267.9104825050327, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -23020,12 +23020,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.601235828808456, - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -25076,11 +25076,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "64bd6e2d3cd9fb8a3762d32513c756dbfe5d33e204538d64f6188eb88b25afff" + "payload_sha256": "1bd3caf43bd58cb8ec23f7fbdc3ece0bf0c5c16d4e8ee0281da29dfb9f53d499" }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-14", "day_confidence": 0.6, @@ -25310,7 +25310,7 @@ "note": "need_baseline:have=1,need=3" }, "trimp": { - "value": 260.0235516950477, + "value": 260.02355169504744, "confidence": 0.6, "tier": "ESTIMATE", "inputs_used": [ @@ -47918,12 +47918,12 @@ "skin_temp_dev_c": 0.11, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, "dip_pct": 27.885325793318177, - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "odi_per_hour": null, "stress": 61.03520288391907, "stress_si": 159.4387755102041, @@ -49942,11 +49942,11 @@ "readiness": 72.95885235383324, "skipped": 0, "partial": 0, - "payload_sha256": "42d874e0691a2e1f19f3ca870dbf497fd820f6daa556da8bec1cbc7f62b661ca" + "payload_sha256": "1a383a9942303b44cfa356dd2d7c2d50c0edfa97a1eaa2b02e53d56cbcd1b1bf" }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "date": "2026-07-15", "day_confidence": 0.6, @@ -71400,7 +71400,7 @@ "skin_temp_dev_c": 0.1, "skin_temp_coverage_frac": 1.0, "skin_temp_settled_frac": null, - "sdnn": 77.77577302925846, + "sdnn": 77.77577302925867, "sdnn_window": 32.11774263281863, "prsa_dc": 10.513, "prsa_dc_anchors": 11257.0, @@ -73230,7 +73230,7 @@ "readiness": 61.79377627786914, "skipped": 0, "partial": 0, - "payload_sha256": "a7d36fabdb957577d3df84059f048e09a85ade8ecce048f891c00de4be9018fd" + "payload_sha256": "24ce9711e808adc5712473639cba81eb6c0fe1c84a7668b00c19ea8bcafcd362" } ], "metric_series": [ @@ -73377,7 +73377,7 @@ { "date": "2026-07-13", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-13", @@ -73422,7 +73422,7 @@ { "date": "2026-07-13", "key": "trimp", - "value": 267.9104825050325 + "value": 267.9104825050327 }, { "date": "2026-07-13", @@ -73587,7 +73587,7 @@ { "date": "2026-07-14", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-14", @@ -73632,7 +73632,7 @@ { "date": "2026-07-14", "key": "trimp", - "value": 260.0235516950477 + "value": 260.02355169504744 }, { "date": "2026-07-14", @@ -73797,7 +73797,7 @@ { "date": "2026-07-15", "key": "sdnn", - "value": 77.77577302925846 + "value": 77.77577302925867 }, { "date": "2026-07-15", @@ -73868,7 +73868,7 @@ "metric_series_version": [ { "date": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73876,7 +73876,7 @@ }, { "date": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73884,7 +73884,7 @@ }, { "date": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "source": "band", "device_family": "gen5", "coverage_devices": "", @@ -73894,7 +73894,7 @@ "wake_day_features": [ { "day_id": "2026-07-13", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-13", "active_min": 90, @@ -75102,7 +75102,7 @@ }, { "day_id": "2026-07-14", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-14", "active_min": 239, @@ -76310,7 +76310,7 @@ }, { "day_id": "2026-07-15", - "algo_version": 121, + "algo_version": 124, "payload_json": { "day_id": "2026-07-15", "active_min": 40, @@ -77507,7 +77507,7 @@ "skin_temp_z": 3.8092033860986256, "skin_temp_adc": 30406.000406834824, "skin_temp_family": "gen5", - "trimp": 267.9104825050325, + "trimp": 267.9104825050327, "strain": null, "steps": 18240.0, "nap_min": 0.0, @@ -77719,7 +77719,7 @@ "skin_temp_z": 2.581998719140345, "skin_temp_adc": 30406.00025889489, "skin_temp_family": "gen5", - "trimp": 260.0235516950477, + "trimp": 260.02355169504744, "strain": null, "steps": 17640.0, "nap_min": 45.0, diff --git a/test/import_bulk_restore_test.dart b/test/import_bulk_restore_test.dart index 0204f908f..c40b65b96 100644 --- a/test/import_bulk_restore_test.dart +++ b/test/import_bulk_restore_test.dart @@ -62,11 +62,14 @@ int _callsTouching(String table) { const _t0 = 1790000000; // 2026-09-21, an arbitrary fixed second /// Stamped with DateTime.now(): differs between any two -/// runs, so the equivalence dump leaves it out. +/// runs, so the equivalence dump leaves it out. store_identity.id is a fresh +/// random id on every committed restore by design (it keys night +/// checkpoints); the singleton row itself is still compared. const _wallClockColumns = { 'compute_freshness': {'updated_at'}, 'activity_review_meta': {'activated_at', 'alert_after_ts'}, 'substrate_archive': {'updated_at'}, + 'store_identity': {'id'}, }; Map archiveSecond(int ts, int hr) => { diff --git a/test/multidevice_coverage_derive_test.dart b/test/multidevice_coverage_derive_test.dart index 99d555102..6fb3c277c 100644 --- a/test/multidevice_coverage_derive_test.dart +++ b/test/multidevice_coverage_derive_test.dart @@ -134,10 +134,10 @@ Future>> _expectedOwnership() async { for (final sig in const [InputSignal.hr1Hz, InputSignal.rrIntervals]) { final raw = await LocalDb.signalPriority(sig); out[sig] = resolveOwnership( - coverage: await LocalDb.coverageIntervals(sig, unionFrom, unionTo), + coverage: await LocalDb.coverageIntervals(sig, unionFrom, unionTo + 1), priority: raw.isEmpty ? const [LocalDb.kPrimaryDeviceId] : raw, from: unionFrom, - to: unionTo, + to: unionTo + 1, signal: sig, ); } diff --git a/test/observation_of_night_test.dart b/test/observation_of_night_test.dart new file mode 100644 index 000000000..998fbdbf5 --- /dev/null +++ b/test/observation_of_night_test.dart @@ -0,0 +1,34 @@ +// Which recorded observation belongs to the night a derive chose. The search +// window also stages the day before's sessions; the longest observation of all +// is not necessarily the chosen night's. +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; + +ana.SleepNightObservation _o(int epochs, int? start, int? end) => + ana.SleepNightObservation(epochs: epochs, startTsSec: start, endTsSec: end); + +void main() { + test('takes the observation staged over the night, not the longest one', () { + final dayBefore = _o(900, 1000, 28000); // longer, but another day + final night = _o(500, 40000, 58000); + expect(observationOfNight([dayBefore, night], 40500, 57500), same(night)); + expect(observationOfNight([night, dayBefore], 40500, 57500), same(night)); + }); + + test('of two overlapping windows, the one overlapping the night most wins', () { + final partial = _o(300, 30000, 41000); + final whole = _o(500, 40000, 58000); + expect(observationOfNight([partial, whole], 40500, 57500), same(whole)); + }); + + test('no observation overlapping the night folds nothing', () { + expect(observationOfNight([_o(900, 1000, 28000)], 40500, 57500), isNull); + expect(observationOfNight(const [], 40500, 57500), isNull); + }); + + test('an observation with no window never matches', () { + expect(observationOfNight([_o(900, null, null)], 40500, 57500), isNull); + expect(observationOfNight([_o(900, 40000, null)], 40500, 57500), isNull); + }); +}