From 2afcf44897505bc037be2de9d961973be613633c Mon Sep 17 00:00:00 2001 From: uvtqt Date: Wed, 18 Nov 2026 15:59:16 +0100 Subject: [PATCH 1/4] stress: lower the score floor so low nights stop reading as a flat 0 (#631) The Baevsky SI -> 0-100 map floored at SI 20, so every genuinely low night (Baevsky's own 'low' band is SI < 50) came out exactly 0 and high-HRV sleepers saw 0 every night, which reads as no data. Floor is now kStressScoreFloorSi (5). Extract the mapping into a shared stressScoreFromSi so derive and the one-shot history rescale (stress_backfill.dart, mirroring strain_backfill) price an index identically. Bump kAlgoVersion 124 -> 125 with a changelog entry; no sibling repin. The floor value is a calibration choice flagged for maintainers. Co-Authored-By: Claude Opus 4.8 --- lib/compute/derivation_engine.dart | 23 +++- lib/compute/onehz_pipeline.dart | 27 +++- lib/compute/stress_backfill.dart | 179 +++++++++++++++++++++++++ test/ed2b_algo_version_114_test.dart | 12 +- test/stress_rescale_backfill_test.dart | 125 +++++++++++++++++ test/stress_score_floor_test.dart | 45 +++++++ 6 files changed, 403 insertions(+), 8 deletions(-) create mode 100644 lib/compute/stress_backfill.dart create mode 100644 test/stress_rescale_backfill_test.dart create mode 100644 test/stress_score_floor_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index d8af739ec..db5671987 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -32,6 +32,7 @@ import 'dart:math' as math; import 'quiet_level_seed.dart' show seedQuietHrrHistoryOnce; import 'strain_backfill.dart' show backfillStrainScale; +import 'stress_backfill.dart' show backfillStressScale; import 'readiness_calibration_backfill.dart' show backfillReadinessCalibration, seedRecentReadinessZ; @@ -1893,7 +1894,13 @@ import 'vendor_sleep.dart'; // no window does, instead of the longest of the search window's sessions. // HrvFreqStream v2, CardioRowCache v3 and policy-bound PRSA states require cold // rebuilds of old checkpoints, isolated by this algorithm version. -const int kAlgoVersion = 124; +// 124 → 125 (edge-side recalibration, no sibling repin): stress score. The +// Baevsky SI -> 0-100 map floored at SI 20, clamping every genuinely low night +// (SI < 50 is Baevsky's own "low" band) to a flat 0 and losing all variation +// for high-HRV sleepers (#631). `kStressScoreFloorSi` is now 5. +// stress_backfill.dart rescales stored history from each day's own `stress.si` +// so the trend has no step at the bump. +const int kAlgoVersion = 125; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. /// @@ -4205,6 +4212,20 @@ class DerivationEngine { } catch (e) { _log('[derive] strain rescale failed (kept old values): $e'); } + // Same one-shot shape as the strain rescale and for the same reason: + // low nights older than `rawRetentionDays` keep a floor-20 score that a + // version bump alone cannot reach. Pure function of each day's stored + // `stress.si`, so it needs no baseline (#631). + _diag['stage'] = 'stress_rescale'; + try { + final rescaled = await backfillStressScale(); + if (rescaled.didWork) { + _log('[derive] stress rescale: ${rescaled.bundleDays} day(s) ' + 'rebuilt, ${rescaled.skipped} skipped (no stored index)'); + } + } catch (e) { + _log('[derive] stress rescale failed (kept old values): $e'); + } // ONE-SHOT, after the sweep for the same reason as the strain rescale: // the rest of the readiness_z history, then a re-score of every day // below v112 that has no raw substrate left (substrate days were just diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 71d8545b7..40bc93f37 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -1672,11 +1672,7 @@ Map _deriveDayBundle(Map inputJson, // stress screen reads this {score, si, lf_hf, rmssd, level} block directly. final si = stress.present ? stress.value!.si : null; final lfhf = hrvF.present ? hrvF.value!.lfhf : null; - double? stressScore; - if (si != null && si > 0) { - final lo = math.log(20), hi = math.log(600); - stressScore = (100 * (math.log(si) - lo) / (hi - lo)).clamp(0.0, 100.0); - } + final stressScore = stressScoreFromSi(si); final stressBlock = { 'value': stressScore == null ? '—' : _round(stressScore, 1), 'score': stressScore == null ? null : _round(stressScore, 1), @@ -2371,6 +2367,27 @@ double? _stddev(List xs) { return math.sqrt(s / (xs.length - 1)); } +/// Baevsky Stress Index → a 0-100 resting-tension score, log-mapped over the +/// plausible resting SI range [kStressScoreFloorSi, kStressScoreCeilingSi]. +/// Null when there is no usable index (never 0 for absence — 0 is a real score +/// at the floor). +/// +/// CALIBRATION: the floor was 20, which clamped every genuinely low-tension +/// night (Baevsky's own "low" band is SI < 50) to a flat 0 and lost all +/// information for high-HRV sleepers (#631). A lower floor restores variation +/// at the bottom. Moving it shifts every user's number, so it is changed +/// deliberately, with a kAlgoVersion bump and a one-shot rescale of stored +/// history (stress_backfill.dart). ONE source for the mapping: the live derive +/// above and that rescale both call this. +const double kStressScoreFloorSi = 5; +const double kStressScoreCeilingSi = 600; + +double? stressScoreFromSi(double? si) { + if (si == null || si <= 0) return null; + final lo = math.log(kStressScoreFloorSi), hi = math.log(kStressScoreCeilingSi); + return (100 * (math.log(si) - lo) / (hi - lo)).clamp(0.0, 100.0); +} + double _round(double v, int dp) { final p = math.pow(10, dp); return (v * p).round() / p; diff --git a/lib/compute/stress_backfill.dart b/lib/compute/stress_backfill.dart new file mode 100644 index 000000000..c41d12862 --- /dev/null +++ b/lib/compute/stress_backfill.dart @@ -0,0 +1,179 @@ +// ONE-SHOT BACKFILL — stored stress scores onto the recalibrated floor (#631). +// +// The Baevsky SI → 0-100 map used to floor at SI 20, so every genuinely low +// night (Baevsky's own "low" band is SI < 50) read as a flat 0 and high-HRV +// sleepers saw 0 every single night. The floor is now `kStressScoreFloorSi`. +// Every day derived before that change carries a score on the old scale, so the +// stress trend and the day-detail screen would show a step at the release date +// rather than a real change. +// +// WHY NOT JUST RE-DERIVE: raw 1 Hz substrate is pruned `rawRetentionDays` behind +// the data edge, so a kAlgoVersion bump alone can only fix the last few days and +// leaves the rest flat at 0. +// +// It does not need raw. The new score is a PURE function of the Baevsky index, +// and each stored day_result already carries its own `stress.si`. So — unlike +// `backfillStrainScale`, which needs TRIMP, a wake window and a quiet-waking +// level — this rescale has every input in the bundle it is rewriting, needs no +// baseline and no warm-up, and can mark itself done on the first pass. +// +// ONE SOURCE for the mapping: this reads `stress.si` and calls the same +// `stressScoreFromSi` the live derive uses, so a rescaled day and a freshly +// derived one price an identical index identically. + +import 'dart:convert'; + +import 'package:flutter/foundation.dart' show visibleForTesting; + +import '../data/day_label.dart'; +import '../data/db.dart'; +import 'derivation_engine.dart' show kAlgoVersion, rawRetentionDays; +import 'onehz_pipeline.dart' show stressScoreFromSi; + +/// `compute_freshness` key marking the rescale as applied. Bumped with the algo +/// version so a later recalibration is a new one-shot rather than a no-op. +const String kStressRescaleKey = 'stress_rescale_v125'; + +class StressBackfillResult { + /// Days whose `metric_series` stress score was rewritten (trend / v_daily). + final int seriesDays; + + /// Days that got a fresh `day_result` row at the current algo version. + final int bundleDays; + + /// Days left exactly as they were because they could not be rescaled. + final int skipped; + + const StressBackfillResult({ + required this.seriesDays, + required this.bundleDays, + required this.skipped, + }); + + bool get didWork => seriesDays > 0 || bundleDays > 0; +} + +/// Rescale every stored day that can no longer be re-derived from raw. Returns +/// without marking done only when there is nothing yet to walk; otherwise it is +/// a one-shot. Set [force] to re-run. +Future backfillStressScale({bool force = false}) async { + const none = StressBackfillResult(seriesDays: 0, bundleDays: 0, skipped: 0); + if (!force && await LocalDb.computeFreshness(kStressRescaleKey) != null) { + return none; + } + + final stressRows = await LocalDb.metricSeries('stress'); + if (stressRows.isEmpty) { + await _markDone(); + return none; + } + + final days = [ + for (final r in stressRows) ?(r['date'] as String?), + ]..sort(); + // Days at or after the cutoff still have raw and are LEFT for a real + // re-derive: writing a patched row at kAlgoVersion here would satisfy the + // derive gate (which matches algo_version EXACTLY) and stand in for a full + // re-derivation of the day. + final cutoff = stressRescaleCutoff(days.last); + + var seriesDays = 0; + var bundleDays = 0; + var skipped = 0; + + // Days already complete at the current version were rescaled on a prior pass. + final alreadyCurrent = await LocalDb.dayResultIds(kAlgoVersion); + + for (final day in days) { + if (day.compareTo(cutoff) >= 0) continue; + if (alreadyCurrent.contains(day)) continue; + + final row = await LocalDb.dayResult(day); + if (row == null) { + // A series row with no bundle behind it: nothing to rewrite the detail + // screen from. Leave the trend point alone rather than diverge the two. + skipped++; + continue; + } + if (((row['algo_version'] as num?)?.toInt() ?? 0) >= kAlgoVersion) { + continue; + } + + final payload = _decode(row['payload_json']); + if (payload == null) { + skipped++; + continue; + } + final stressBlk = payload['stress']; + if (stressBlk is! Map) { + skipped++; + continue; + } + final si = (stressBlk['si'] as num?)?.toDouble(); + final next = stressScoreFromSi(si); + if (next == null) { + // No usable index on this day. A rescaled day must not become 0, which is + // a score, not an absence — leave it exactly as it was. + skipped++; + continue; + } + final rounded = _round1(next); + // The block is {value, score, si, level, ...}; `value` and `score` are the + // rounded headline the screens read, `si`/`level` are unchanged. + stressBlk['score'] = rounded; + stressBlk['value'] = rounded; + + final partial = (row['partial'] as num?)?.toInt() == 1; + await LocalDb.putDayResult( + dayId: day, + algoVersion: kAlgoVersion, + payloadJson: jsonEncode(payload), + windowJson: (row['window_json'] as String?) ?? '{}', + finalized: (row['finalized'] as num?)?.toInt() == 1, + skipped: (row['skipped'] as num?)?.toInt() == 1, + partial: partial, + rhr: (row['rhr'] as num?)?.toDouble(), + rmssd: (row['rmssd'] as num?)?.toDouble(), + readiness: (row['readiness'] as num?)?.toDouble(), + // Rewriting a band-derived day's stress from its own stored index — same + // provenance it already had. + source: 'band', + // `putDayResult` skips the series write for a partial row, so only count + // the trend as rewritten when it actually was. + series: {'stress': rounded}, + ); + bundleDays++; + if (!partial) seriesDays++; + } + + await _markDone(); + return StressBackfillResult( + seriesDays: seriesDays, + bundleDays: bundleDays, + skipped: skipped, + ); +} + +Future _markDone() => + LocalDb.putComputeFreshness(kStressRescaleKey, jsonEncode({'done': true})); + +double _round1(double v) => (v * 10).round() / 10; + +Map? _decode(Object? json) { + if (json is! String) return null; + try { + final v = jsonDecode(json); + return v is Map ? v.cast() : null; + } catch (_) { + return null; + } +} + +/// The first day still treated as having raw, [rawRetentionDays] CALENDAR days +/// before the data edge [lastDay]. Calendar arithmetic (not an absolute +/// `Duration`), matching the pruner's cut on local day labels, so a DST day does +/// not pull the cutoff a day early. A malformed label falls back to itself, +/// which leaves every day for a re-derive — the safe side. +@visibleForTesting +String stressRescaleCutoff(String lastDay) => + dayLabelBefore(lastDay, rawRetentionDays) ?? lastDay; diff --git a/test/ed2b_algo_version_114_test.dart b/test/ed2b_algo_version_114_test.dart index b3de3bd82..93dc86202 100644 --- a/test/ed2b_algo_version_114_test.dart +++ b/test/ed2b_algo_version_114_test.dart @@ -6,7 +6,7 @@ import 'package:openstrap_edge/compute/derivation_engine.dart'; void main() { // WHY: finalized days need a new version when the analytics pin changes outputs. test('ED-2b adoption and workout rest repin name their output changes', () { - expect(kAlgoVersion, 124); + expect(kAlgoVersion, 125); expect(kAnalyticsPin, 'a438df40b38a4586a52a68ab7e1c100fca111ccd'); final src = File('lib/compute/derivation_engine.dart').readAsStringSync(); final at = src.indexOf('// 121 → 122'); @@ -25,11 +25,19 @@ void main() { 'duration-cap', 'saved-session overlap']) { expect(repinEntry, contains(change)); } - final mainRepin = src.substring(src.indexOf('// 123 → 124')); + final mainRepin = src.substring( + src.indexOf('// 123 → 124'), src.indexOf('// 124 → 125')); for (final change in ['a438df4', '#102-#110', '#110', 'regressing clocks', '5%', 'Cosinor', 'SDNN', 'uncertainty', 'HrvFreqStream v2', 'CardioRowCache v3', 'PRSA']) { expect(mainRepin, contains(change)); } + // A bump must name its output change: v125 is the stress recalibration. + final stressEntry = src.substring( + src.indexOf('// 124 → 125'), src.indexOf('const int kAlgoVersion')); + for (final change in ['#631', 'stress', 'kStressScoreFloorSi', + 'stress_backfill']) { + expect(stressEntry, contains(change)); + } }); } diff --git a/test/stress_rescale_backfill_test.dart b/test/stress_rescale_backfill_test.dart new file mode 100644 index 000000000..db2f2ba5c --- /dev/null +++ b/test/stress_rescale_backfill_test.dart @@ -0,0 +1,125 @@ +// One-shot backfill of stored stress scores onto the recalibrated floor (#631). +// +// Raw 1 Hz substrate is pruned `rawRetentionDays` behind the data edge, so the +// stress history CANNOT be re-derived from raw and a kAlgoVersion bump alone +// leaves old low nights flat at 0. It does not need raw: the score is a pure +// function of the stored Baevsky index, which every day_result already carries, +// so the rescale reads `stress.si` and re-maps it with `stressScoreFromSi`. + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'package:openstrap_edge/compute/onehz_pipeline.dart' + show stressScoreFromSi; +import 'package:openstrap_edge/compute/stress_backfill.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; + +/// Seed a derived day the way the engine would have on the OLD (floor-20) scale. +Future seedStressDay( + String day, { + required double? si, + required double? score, + String level = 'low', + bool withBlock = true, + int algoVersion = 120, +}) async { + await LocalDb.putDayResult( + dayId: day, + algoVersion: algoVersion, + payloadJson: jsonEncode({ + 'date': day, + 'scalars': const {'rhr': 55}, + if (withBlock) + 'stress': { + 'value': score, + 'score': score, + 'si': si, + 'level': level, + }, + }), + windowJson: '{}', + finalized: true, + series: {'stress': ?score, 'stress_si': ?si}, + ); +} + +Future seriesValue(String key, String day) async { + for (final r in await LocalDb.metricSeries(key)) { + if (r['date'] == day) return (r['value'] as num?)?.toDouble(); + } + return null; +} + +Future bundleScore(String day) async { + final row = await LocalDb.dayResult(day); + if (row == null) return null; + final payload = jsonDecode(row['payload_json'] as String) as Map; + final blk = payload['stress']; + return blk is Map ? (blk['score'] as num?)?.toDouble() : null; +} + +void main() { + setUpAll(() async { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + LocalDb.dbName = 'openstrap_stress_backfill_test.db'; + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + }); + + tearDownAll(() async { + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + }); + + group('backfillStressScale — the stored history', () { + // Newest row 2026-06-10; a day this far back is comfortably before the + // rawRetentionDays cutoff, so it is rescaled, while 2026-06-09 is not. + const oldDay = '2026-05-01'; + final recentDay = dayLabelBefore('2026-06-10', 1)!; // within retention + const oddDay = '2026-05-02'; // has a trend score, but no SI in the bundle + const newestDay = '2026-06-10'; + + test('a flat-0 low night is rescaled from its own stored index', () async { + // SI 12 was exactly 0 under the floor of 20. + await seedStressDay(oldDay, si: 12, score: 0); + await seedStressDay(oddDay, si: null, score: 40); // bundle SI missing + await seedStressDay(recentDay, si: 12, score: 0); + await seedStressDay(newestDay, si: 300, score: 73); + + _result = await backfillStressScale(); + expect(_result!.didWork, isTrue); + + final want = ((stressScoreFromSi(12)! * 10).round()) / 10; + expect(want, greaterThan(0)); // the whole point of #631 + expect(await seriesValue('stress', oldDay), closeTo(want, 1e-9)); + expect(await bundleScore(oldDay), closeTo(want, 1e-9)); + }); + + test('a day still inside the raw window is left for a real re-derive', + () async { + expect(await seriesValue('stress', recentDay), 0); + expect(await seriesValue('stress', newestDay), 73); + }); + + test('a day whose index is missing is left alone, never zeroed', () async { + expect(_result!.skipped, greaterThan(0)); + // Its trend score is unchanged — abstaining, not rewriting to 0. + expect(await seriesValue('stress', oddDay), 40); + }); + + test('it is a one-shot: a second pass rewrites nothing', () async { + expect(await LocalDb.computeFreshness(kStressRescaleKey), isNotNull); + final again = await backfillStressScale(); + expect(again.didWork, isFalse); + }); + }); +} + +// The first pass's result, shared so later tests can assert on it. +StressBackfillResult? _result; diff --git a/test/stress_score_floor_test.dart b/test/stress_score_floor_test.dart new file mode 100644 index 000000000..cb50bbb86 --- /dev/null +++ b/test/stress_score_floor_test.dart @@ -0,0 +1,45 @@ +// #631: the Baevsky SI → 0-100 stress score clamped every genuinely low night +// to a flat 0, because the scale floor sat at SI 20 while Baevsky's own "low" +// band is SI < 50. High-HRV sleepers (nightly SI ~10-14) saw 0 on every night, +// which reads as "no data" rather than "low". The floor is now 5, so the low +// end varies again. This pins the mapping, which the live derive and the +// one-shot history rescale both call. +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart' + show stressScoreFromSi, kStressScoreFloorSi, kStressScoreCeilingSi; + +void main() { + group('stressScoreFromSi', () { + test('no usable index is absence, never 0', () { + expect(stressScoreFromSi(null), isNull); + expect(stressScoreFromSi(0), isNull); + expect(stressScoreFromSi(-1), isNull); + }); + + test('the floor is 0 and the ceiling is 100', () { + expect(stressScoreFromSi(kStressScoreFloorSi), 0.0); + expect(stressScoreFromSi(kStressScoreCeilingSi), 100.0); + expect(stressScoreFromSi(kStressScoreCeilingSi * 2), 100.0); // clamped + expect(stressScoreFromSi(kStressScoreFloorSi / 2), 0.0); // clamped + }); + + test('genuinely low nights no longer flatten to 0 (the #631 symptom)', () { + // Under the old floor of 20 every one of these was exactly 0. + for (final si in const [10.0, 12.0, 14.0, 18.0]) { + expect(stressScoreFromSi(si), greaterThan(0.0), + reason: 'SI $si used to clamp to 0'); + } + expect(stressScoreFromSi(10), closeTo(14.5, 0.6)); + expect(stressScoreFromSi(20), closeTo(29.0, 0.6)); + }); + + test('monotonic in the index', () { + double last = -1; + for (final si in const [5.0, 10.0, 20.0, 50.0, 150.0, 600.0]) { + final s = stressScoreFromSi(si)!; + expect(s, greaterThanOrEqualTo(last)); + last = s; + } + }); + }); +} From 72aa8a0c097d126141c97d00c0cb4e85bdbc79d9 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:28:30 +0530 Subject: [PATCH 2/4] stress rescale: patch scalars.stress too, unrounded series, independent tests --- lib/compute/derivation_engine.dart | 3 +- lib/compute/stress_backfill.dart | 7 +++- test/stress_rescale_backfill_test.dart | 54 ++++++++++++++++++++------ 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 3593810bd..cb522887a 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -1899,7 +1899,8 @@ import 'vendor_sleep.dart'; // (SI < 50 is Baevsky's own "low" band) to a flat 0 and losing all variation // for high-HRV sleepers (#631). `kStressScoreFloorSi` is now 5. // stress_backfill.dart rescales stored history from each day's own `stress.si` -// so the trend has no step at the bump. +// so the trend has no step at the bump. User-visible: every stress score moves +// up once (e.g. SI 160 reads ~72, was ~61); nights that read 0 now read low. const int kAlgoVersion = 125; /// The sibling SHAs this version was derived against, asserted against /// pubspec.yaml in test/db_serve_version_and_reads_test.dart. diff --git a/lib/compute/stress_backfill.dart b/lib/compute/stress_backfill.dart index c41d12862..436fb04bf 100644 --- a/lib/compute/stress_backfill.dart +++ b/lib/compute/stress_backfill.dart @@ -122,6 +122,10 @@ Future backfillStressScale({bool force = false}) async { // rounded headline the screens read, `si`/`level` are unchanged. stressBlk['score'] = rounded; stressBlk['value'] = rounded; + // The derive also writes the unrounded score into `scalars`, which the + // rollup and cross-day reads use, so it moves with the block. + final scalars = payload['scalars']; + if (scalars is Map) scalars['stress'] = next; final partial = (row['partial'] as num?)?.toInt() == 1; await LocalDb.putDayResult( @@ -140,7 +144,8 @@ Future backfillStressScale({bool force = false}) async { source: 'band', // `putDayResult` skips the series write for a partial row, so only count // the trend as rewritten when it actually was. - series: {'stress': rounded}, + // Unrounded, like the derive's `'stress': sc('stress')` series write. + series: {'stress': next}, ); bundleDays++; if (!partial) seriesDays++; diff --git a/test/stress_rescale_backfill_test.dart b/test/stress_rescale_backfill_test.dart index db2f2ba5c..15243b66e 100644 --- a/test/stress_rescale_backfill_test.dart +++ b/test/stress_rescale_backfill_test.dart @@ -32,7 +32,7 @@ Future seedStressDay( algoVersion: algoVersion, payloadJson: jsonEncode({ 'date': day, - 'scalars': const {'rhr': 55}, + 'scalars': {'rhr': 55, 'stress': ?score}, if (withBlock) 'stress': { 'value': score, @@ -62,11 +62,25 @@ Future bundleScore(String day) async { return blk is Map ? (blk['score'] as num?)?.toDouble() : null; } +Future scalarStress(String day) async { + final row = await LocalDb.dayResult(day); + if (row == null) return null; + final payload = jsonDecode(row['payload_json'] as String) as Map; + final sc = payload['scalars']; + return sc is Map ? (sc['stress'] as num?)?.toDouble() : null; +} + void main() { setUpAll(() async { sqfliteFfiInit(); databaseFactory = databaseFactoryFfi; LocalDb.dbName = 'openstrap_stress_backfill_test.db'; + }); + + // Every test starts from an empty database, so none depends on another's + // writes or on run order. + setUp(() async { + await LocalDb.close(); final dir = await databaseFactory.getDatabasesPath(); await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); }); @@ -88,38 +102,54 @@ void main() { test('a flat-0 low night is rescaled from its own stored index', () async { // SI 12 was exactly 0 under the floor of 20. await seedStressDay(oldDay, si: 12, score: 0); - await seedStressDay(oddDay, si: null, score: 40); // bundle SI missing - await seedStressDay(recentDay, si: 12, score: 0); await seedStressDay(newestDay, si: 300, score: 73); - _result = await backfillStressScale(); - expect(_result!.didWork, isTrue); + final result = await backfillStressScale(); + expect(result.didWork, isTrue); - final want = ((stressScoreFromSi(12)! * 10).round()) / 10; + final raw = stressScoreFromSi(12)!; + final want = ((raw * 10).round()) / 10; expect(want, greaterThan(0)); // the whole point of #631 - expect(await seriesValue('stress', oldDay), closeTo(want, 1e-9)); + // Trend and scalars carry the unrounded score, as the derive writes them. + expect(await seriesValue('stress', oldDay), closeTo(raw, 1e-9)); + expect(await scalarStress(oldDay), closeTo(raw, 1e-9)); expect(await bundleScore(oldDay), closeTo(want, 1e-9)); }); test('a day still inside the raw window is left for a real re-derive', () async { + await seedStressDay(recentDay, si: 12, score: 0); + await seedStressDay(newestDay, si: 300, score: 73); + + await backfillStressScale(); + expect(await seriesValue('stress', recentDay), 0); + expect(await scalarStress(recentDay), 0); expect(await seriesValue('stress', newestDay), 73); }); test('a day whose index is missing is left alone, never zeroed', () async { - expect(_result!.skipped, greaterThan(0)); - // Its trend score is unchanged — abstaining, not rewriting to 0. + await seedStressDay(oddDay, si: null, score: 40); // bundle SI missing + await seedStressDay(newestDay, si: 300, score: 73); + + final result = await backfillStressScale(); + + expect(result.skipped, 1); + expect(result.didWork, isFalse); + // Trend, scalars and block are unchanged: abstaining, not rewriting to 0. expect(await seriesValue('stress', oddDay), 40); + expect(await scalarStress(oddDay), 40); + expect(await bundleScore(oddDay), 40); }); test('it is a one-shot: a second pass rewrites nothing', () async { + await seedStressDay(oldDay, si: 12, score: 0); + await seedStressDay(newestDay, si: 300, score: 73); + + expect((await backfillStressScale()).didWork, isTrue); expect(await LocalDb.computeFreshness(kStressRescaleKey), isNotNull); final again = await backfillStressScale(); expect(again.didWork, isFalse); }); }); } - -// The first pass's result, shared so later tests can assert on it. -StressBackfillResult? _result; From 8a5f5ef22c35199751bb4db38599fc2acc4f92fd Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:50:26 +0530 Subject: [PATCH 3/4] stress rescale in place, keep versions and provenance, rebuild stale rollup --- lib/compute/derivation_engine.dart | 32 ++-- lib/compute/onehz_pipeline.dart | 2 +- lib/compute/stress_backfill.dart | 210 ++++++++++++------------- test/crossday_as_of_zone_test.dart | 83 ++++++---- test/stress_rescale_backfill_test.dart | 187 +++++++++++----------- test/stress_score_floor_test.dart | 2 + 6 files changed, 278 insertions(+), 238 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index cb522887a..65584a4cb 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -1898,8 +1898,8 @@ import 'vendor_sleep.dart'; // Baevsky SI -> 0-100 map floored at SI 20, clamping every genuinely low night // (SI < 50 is Baevsky's own "low" band) to a flat 0 and losing all variation // for high-HRV sleepers (#631). `kStressScoreFloorSi` is now 5. -// stress_backfill.dart rescales stored history from each day's own `stress.si` -// so the trend has no step at the bump. User-visible: every stress score moves +// stress_backfill.dart rescales stored history in place from each day's own +// `stress.si` so the trend has no step at the bump. User-visible: every stress score moves // up once (e.g. SI 160 reads ~72, was ~61); nights that read 0 now read low. const int kAlgoVersion = 125; /// The sibling SHAs this version was derived against, asserted against @@ -4218,14 +4218,20 @@ class DerivationEngine { // version bump alone cannot reach. Pure function of each day's stored // `stress.si`, so it needs no baseline (#631). _diag['stage'] = 'stress_rescale'; + var stressRescaled = false; try { final rescaled = await backfillStressScale(); if (rescaled.didWork) { - _log('[derive] stress rescale: ${rescaled.bundleDays} day(s) ' - 'rebuilt, ${rescaled.skipped} skipped (no stored index)'); + stressRescaled = true; + _log('[derive] stress rescale: ${rescaled.bundleDays} row(s), ' + '${rescaled.seriesDays} trend point(s) rescaled in place, ' + '${rescaled.skipped} skipped (no stored index or imported)'); } } catch (e) { - _log('[derive] stress rescale failed (kept old values): $e'); + // Each day commits on its own, so days before the failure stay + // rescaled; the marker is not written, so the next pass resumes. + stressRescaled = true; + _log('[derive] stress rescale stopped, will retry next pass: $e'); } // ONE-SHOT, after the sweep for the same reason as the strain rescale: // the rest of the readiness_z history, then a re-score of every day @@ -4258,10 +4264,10 @@ class DerivationEngine { } catch (e) { _log('[derive] readiness calibration failed (will retry): $e'); } - // The strain rescale rewrote stored days after the rollup of this pass - // was built from the old values. When the calibration above did not + // The strain or stress rescale rewrote stored days after the rollup of + // this pass was built from the old values. When the calibration above did not // rebuild it (no work, or it failed), rebuild it here. - if (strainRescaled && !crossDayRebuilt) { + if ((strainRescaled || stressRescaled) && !crossDayRebuilt) { try { await _refreshBaselines(); await _runCrossDay(profile); @@ -8632,7 +8638,8 @@ class DerivationEngine { String name, int nowOffsetMin, int janOffsetMin, int julOffsetMin) => '$name|$nowOffsetMin|$janOffsetMin|$julOffsetMin'; - /// Whether the stored cross-day ROLLUP was built in another zone. The + /// Whether the stored cross-day ROLLUP was built in another zone or by + /// another algorithm version. The /// rollup, not its inputs: [_refreshBaselines] re-stamps the inputs before /// the rollup is written, so a rollup that then timed out would read as /// current on the inputs' stamp and never be retried. @@ -8641,7 +8648,12 @@ class DerivationEngine { if (raw is! String || raw.isEmpty) return false; try { final decoded = jsonDecode(raw); - return decoded is Map && decoded['zone'] != crossDayZoneStamp(); + // A rollup from another algorithm version is rejected by its readers + // (`getInsights`), so it is as stale as one from another zone: rebuild it + // from stored days even when nothing else in the pass has work. + return decoded is Map && + (decoded['zone'] != crossDayZoneStamp() || + (decoded['algo_version'] as num?)?.toInt() != kAlgoVersion); } catch (_) { return false; } diff --git a/lib/compute/onehz_pipeline.dart b/lib/compute/onehz_pipeline.dart index 40bc93f37..312008ffa 100644 --- a/lib/compute/onehz_pipeline.dart +++ b/lib/compute/onehz_pipeline.dart @@ -2383,7 +2383,7 @@ const double kStressScoreFloorSi = 5; const double kStressScoreCeilingSi = 600; double? stressScoreFromSi(double? si) { - if (si == null || si <= 0) return null; + if (si == null || !si.isFinite || si <= 0) return null; final lo = math.log(kStressScoreFloorSi), hi = math.log(kStressScoreCeilingSi); return (100 * (math.log(si) - lo) / (hi - lo)).clamp(0.0, 100.0); } diff --git a/lib/compute/stress_backfill.dart b/lib/compute/stress_backfill.dart index 436fb04bf..e2f7d1f4e 100644 --- a/lib/compute/stress_backfill.dart +++ b/lib/compute/stress_backfill.dart @@ -8,26 +8,30 @@ // rather than a real change. // // WHY NOT JUST RE-DERIVE: raw 1 Hz substrate is pruned `rawRetentionDays` behind -// the data edge, so a kAlgoVersion bump alone can only fix the last few days and -// leaves the rest flat at 0. +// the data edge, so a kAlgoVersion bump alone can only fix the last few days. // // It does not need raw. The new score is a PURE function of the Baevsky index, -// and each stored day_result already carries its own `stress.si`. So — unlike -// `backfillStrainScale`, which needs TRIMP, a wake window and a quiet-waking -// level — this rescale has every input in the bundle it is rewriting, needs no -// baseline and no warm-up, and can mark itself done on the first pass. +// and each stored day_result already carries its own `stress.si`. // -// ONE SOURCE for the mapping: this reads `stress.si` and calls the same -// `stressScoreFromSi` the live derive uses, so a rescaled day and a freshly -// derived one price an identical index identically. +// IN PLACE, NOT A NEW VERSION ROW. The rescale rewrites only the stress score +// (`stress.score`/`value`, `scalars.stress`, the `stress` trend point) of the +// rows that exist, at the version they already have. It does not stamp a day +// current, so: +// * a day that still has raw is still re-derived by the version gate (no +// guess at where the raw window ends is needed); +// * the strain and readiness one-shots, which key on the row's version, +// still see every day they are owed; +// * provenance (device family, coverage, method stamps, naps) is untouched. +// The rewrite is idempotent (score = f(stored si)), so a row a fresh derive +// already wrote is a no-op. +// +// ONE source for the mapping: `stressScoreFromSi`, the same function the live +// derive calls. import 'dart:convert'; -import 'package:flutter/foundation.dart' show visibleForTesting; - -import '../data/day_label.dart'; import '../data/db.dart'; -import 'derivation_engine.dart' show kAlgoVersion, rawRetentionDays; +import '../data/series_codec.dart'; import 'onehz_pipeline.dart' show stressScoreFromSi; /// `compute_freshness` key marking the rescale as applied. Bumped with the algo @@ -38,10 +42,10 @@ class StressBackfillResult { /// Days whose `metric_series` stress score was rewritten (trend / v_daily). final int seriesDays; - /// Days that got a fresh `day_result` row at the current algo version. + /// `day_result` rows whose stress score was rewritten in place. final int bundleDays; - /// Days left exactly as they were because they could not be rescaled. + /// Rows left exactly as they were: no usable index, imported, or unreadable. final int skipped; const StressBackfillResult({ @@ -53,105 +57,94 @@ class StressBackfillResult { bool get didWork => seriesDays > 0 || bundleDays > 0; } -/// Rescale every stored day that can no longer be re-derived from raw. Returns -/// without marking done only when there is nothing yet to walk; otherwise it is -/// a one-shot. Set [force] to re-run. +/// Rescale every stored stress score in place. A one-shot once it has walked a +/// non-empty store; an empty store is not marked, so rows a restore merges in +/// later are still rescaled. Set [force] to re-run. Future backfillStressScale({bool force = false}) async { const none = StressBackfillResult(seriesDays: 0, bundleDays: 0, skipped: 0); if (!force && await LocalDb.computeFreshness(kStressRescaleKey) != null) { return none; } - final stressRows = await LocalDb.metricSeries('stress'); - if (stressRows.isEmpty) { - await _markDone(); - return none; - } - - final days = [ - for (final r in stressRows) ?(r['date'] as String?), - ]..sort(); - // Days at or after the cutoff still have raw and are LEFT for a real - // re-derive: writing a patched row at kAlgoVersion here would satisfy the - // derive gate (which matches algo_version EXACTLY) and stand in for a full - // re-derivation of the day. - final cutoff = stressRescaleCutoff(days.last); + final db = await LocalDb.instance; + final days = [ + for (final r in await db.rawQuery( + 'SELECT DISTINCT day_id FROM day_result WHERE day_id IS NOT NULL ' + 'ORDER BY day_id')) + r['day_id'] as String, + ]; + if (days.isEmpty) return none; var seriesDays = 0; var bundleDays = 0; var skipped = 0; - // Days already complete at the current version were rescaled on a prior pass. - final alreadyCurrent = await LocalDb.dayResultIds(kAlgoVersion); - for (final day in days) { - if (day.compareTo(cutoff) >= 0) continue; - if (alreadyCurrent.contains(day)) continue; - - final row = await LocalDb.dayResult(day); - if (row == null) { - // A series row with no bundle behind it: nothing to rewrite the detail - // screen from. Leave the trend point alone rather than diverge the two. - skipped++; - continue; - } - if (((row['algo_version'] as num?)?.toInt() ?? 0) >= kAlgoVersion) { - continue; - } - - final payload = _decode(row['payload_json']); - if (payload == null) { - skipped++; - continue; - } - final stressBlk = payload['stress']; - if (stressBlk is! Map) { - skipped++; - continue; - } - final si = (stressBlk['si'] as num?)?.toDouble(); - final next = stressScoreFromSi(si); - if (next == null) { - // No usable index on this day. A rescaled day must not become 0, which is - // a score, not an absence — leave it exactly as it was. - skipped++; - continue; - } - final rounded = _round1(next); - // The block is {value, score, si, level, ...}; `value` and `score` are the - // rounded headline the screens read, `si`/`level` are unchanged. - stressBlk['score'] = rounded; - stressBlk['value'] = rounded; - // The derive also writes the unrounded score into `scalars`, which the - // rollup and cross-day reads use, so it moves with the block. - final scalars = payload['scalars']; - if (scalars is Map) scalars['stress'] = next; - - final partial = (row['partial'] as num?)?.toInt() == 1; - await LocalDb.putDayResult( - dayId: day, - algoVersion: kAlgoVersion, - payloadJson: jsonEncode(payload), - windowJson: (row['window_json'] as String?) ?? '{}', - finalized: (row['finalized'] as num?)?.toInt() == 1, - skipped: (row['skipped'] as num?)?.toInt() == 1, - partial: partial, - rhr: (row['rhr'] as num?)?.toDouble(), - rmssd: (row['rmssd'] as num?)?.toDouble(), - readiness: (row['readiness'] as num?)?.toDouble(), - // Rewriting a band-derived day's stress from its own stored index — same - // provenance it already had. - source: 'band', - // `putDayResult` skips the series write for a partial row, so only count - // the trend as rewritten when it actually was. - // Unrounded, like the derive's `'stress': sc('stress')` series write. - series: {'stress': next}, - ); - bundleDays++; - if (!partial) seriesDays++; + // One transaction per day: the rows are read and rewritten together, so a + // derive in another isolate cannot land between the read and the write + // and be overwritten by an older payload. + await db.transaction((txn) async { + final rows = await txn.query('day_result', + columns: ['algo_version', 'payload_json', 'partial'], + where: 'day_id = ?', + whereArgs: [day], + orderBy: 'algo_version DESC'); + double? seriesScore; + var seriesPicked = false; + for (final row in rows) { + final payload = _decode(row['payload_json']); + final blk = payload?['stress']; + final si = blk is Map ? _finite(blk['si']) : null; + final next = stressScoreFromSi(si); + // No usable index, or another source's day: a rescaled day must not + // become 0 (a score, not an absence), and an imported score is not + // ours to re-price. + if (payload == null || next == null || payload['imported'] == true) { + skipped++; + continue; + } + // The trend point was written by the newest COMPLETE row (a partial + // row never writes the series), so it is rescaled from that row's index. + if (!seriesPicked && (row['partial'] as num?)?.toInt() != 1) { + seriesPicked = true; + seriesScore = next; + } + final rounded = _round1(next); + final scalars = payload['scalars']; + if ((blk as Map)['score'] == rounded && + blk['value'] == rounded && + (scalars is! Map || scalars['stress'] == next)) { + continue; // already on the new scale + } + // `value`/`score` are the rounded headline the screens read; the + // derive also writes the unrounded score into `scalars`. + blk['score'] = rounded; + blk['value'] = rounded; + if (scalars is Map) scalars['stress'] = next; + await txn.update( + 'day_result', + {'payload_json': SeriesCodec.encodePayloadJson(jsonEncode(payload))}, + where: 'day_id = ? AND algo_version = ?', + whereArgs: [day, row['algo_version']], + ); + bundleDays++; + } + if (seriesScore == null) return; + // UPDATE only: never create a trend point the derive did not write, and + // leave a day whose series another source produced alone. + final n = await txn.rawUpdate( + "UPDATE metric_series SET value = ? WHERE date = ? AND key = 'stress' " + 'AND (value IS NULL OR value <> ?) AND NOT EXISTS (SELECT 1 FROM ' + 'metric_series_version v WHERE v.date = ? AND v.source IS NOT NULL ' + "AND v.source <> 'band')", + [seriesScore, day, seriesScore, day], + ); + if (n > 0) seriesDays++; + }); } - await _markDone(); + await LocalDb.putComputeFreshness( + kStressRescaleKey, jsonEncode({'done': true})); return StressBackfillResult( seriesDays: seriesDays, bundleDays: bundleDays, @@ -159,8 +152,12 @@ Future backfillStressScale({bool force = false}) async { ); } -Future _markDone() => - LocalDb.putComputeFreshness(kStressRescaleKey, jsonEncode({'done': true})); +/// A finite number, or null — a wrong-typed index is skipped, not thrown on. +double? _finite(Object? v) { + if (v is! num) return null; + final d = v.toDouble(); + return d.isFinite ? d : null; +} double _round1(double v) => (v * 10).round() / 10; @@ -173,12 +170,3 @@ Map? _decode(Object? json) { return null; } } - -/// The first day still treated as having raw, [rawRetentionDays] CALENDAR days -/// before the data edge [lastDay]. Calendar arithmetic (not an absolute -/// `Duration`), matching the pruner's cut on local day labels, so a DST day does -/// not pull the cutoff a day early. A malformed label falls back to itself, -/// which leaves every day for a re-derive — the safe side. -@visibleForTesting -String stressRescaleCutoff(String lastDay) => - dayLabelBefore(lastDay, rawRetentionDays) ?? lastDay; diff --git a/test/crossday_as_of_zone_test.dart b/test/crossday_as_of_zone_test.dart index bc8ae34fe..94134576a 100644 --- a/test/crossday_as_of_zone_test.dart +++ b/test/crossday_as_of_zone_test.dart @@ -44,6 +44,36 @@ Future _putInput(String zone) async { ); } +Future _seedRestoredDays() async { + for (var i = 0; i < 5; i++) { + final day = '2026-09-0${i + 2}'; + final onset = + DateTime.utc(2026, 9, i + 1, 22).millisecondsSinceEpoch ~/ 1000; + await LocalDb.putDayResult( + dayId: day, + algoVersion: kAlgoVersion, + finalized: true, + rhr: 52, + readiness: 70, + windowJson: '{}', + payloadJson: jsonEncode({ + 'scalars': {'rhr': 52, 'readiness': 70, 'efficiency': 95}, + 'sleep': { + 'window': { + 'value': { + 'onset_ms': onset * 1000, + 'offset_ms': (onset + 8 * 3600) * 1000, + }, + }, + 'accounting': { + 'value': {'tst_sec': 27000}, + }, + }, + }), + ); + } +} + void main() { setUpAll(() async { sqfliteFfiInit(); @@ -82,33 +112,7 @@ void main() { test('a restored backup with no band data rebuilds an old-zone rollup', () async { // Stored days, no decoded rows, and a rollup from before a flight. - for (var i = 0; i < 5; i++) { - final day = '2026-09-0${i + 2}'; - final onset = - DateTime.utc(2026, 9, i + 1, 22).millisecondsSinceEpoch ~/ 1000; - await LocalDb.putDayResult( - dayId: day, - algoVersion: kAlgoVersion, - finalized: true, - rhr: 52, - readiness: 70, - windowJson: '{}', - payloadJson: jsonEncode({ - 'scalars': {'rhr': 52, 'readiness': 70, 'efficiency': 95}, - 'sleep': { - 'window': { - 'value': { - 'onset_ms': onset * 1000, - 'offset_ms': (onset + 8 * 3600) * 1000, - }, - }, - 'accounting': { - 'value': {'tst_sec': 27000}, - }, - }, - }), - ); - } + await _seedRestoredDays(); await LocalDb.putBaseline( 'crossday', jsonEncode({ @@ -131,6 +135,31 @@ void main() { expect(rollup['zone'], DerivationEngine.crossDayZoneStamp()); }); + test('a restored backup with no band data rebuilds an old-version rollup', + () async { + // Same zone, but written by the previous algorithm version, which the + // insights reader rejects. + await _seedRestoredDays(); + await LocalDb.putBaseline( + 'crossday', + jsonEncode({ + 'algo_version': kAlgoVersion - 1, + 'zone': DerivationEngine.crossDayZoneStamp(), + 'recent': const [], + }), + ); + await _putInput(DerivationEngine.crossDayZoneStamp()); + await DerivationEngine().run(const Profile( + ageYears: 35, + sex: 'm', + weightKg: 75, + heightCm: 178, + )); + final rollup = jsonDecode( + (await LocalDb.baseline('crossday'))!['payload_json'] as String) as Map; + expect(rollup['algo_version'], kAlgoVersion); + }); + test('a stored rollup from another zone is not served as current', () async { await _putInput('Elsewhere|-420'); diff --git a/test/stress_rescale_backfill_test.dart b/test/stress_rescale_backfill_test.dart index 15243b66e..9eab06be2 100644 --- a/test/stress_rescale_backfill_test.dart +++ b/test/stress_rescale_backfill_test.dart @@ -4,7 +4,8 @@ // stress history CANNOT be re-derived from raw and a kAlgoVersion bump alone // leaves old low nights flat at 0. It does not need raw: the score is a pure // function of the stored Baevsky index, which every day_result already carries, -// so the rescale reads `stress.si` and re-maps it with `stressScoreFromSi`. +// so the rescale reads `stress.si` and re-maps it with `stressScoreFromSi`, +// in place, at the version each row already has. import 'dart:convert'; @@ -15,16 +16,15 @@ import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:openstrap_edge/compute/onehz_pipeline.dart' show stressScoreFromSi; import 'package:openstrap_edge/compute/stress_backfill.dart'; -import 'package:openstrap_edge/data/day_label.dart'; import 'package:openstrap_edge/data/db.dart'; /// Seed a derived day the way the engine would have on the OLD (floor-20) scale. Future seedStressDay( String day, { - required double? si, + required Object? si, required double? score, - String level = 'low', - bool withBlock = true, + bool partial = false, + bool imported = false, int algoVersion = 120, }) async { await LocalDb.putDayResult( @@ -32,46 +32,39 @@ Future seedStressDay( algoVersion: algoVersion, payloadJson: jsonEncode({ 'date': day, + 'imported': ?(imported ? true : null), 'scalars': {'rhr': 55, 'stress': ?score}, - if (withBlock) - 'stress': { - 'value': score, - 'score': score, - 'si': si, - 'level': level, - }, + 'stress': {'value': score, 'score': score, 'si': si, 'level': 'low'}, }), windowJson: '{}', finalized: true, - series: {'stress': ?score, 'stress_si': ?si}, + partial: partial, + series: {'stress': ?score}, ); } -Future seriesValue(String key, String day) async { - for (final r in await LocalDb.metricSeries(key)) { +Future seriesValue(String day) async { + for (final r in await LocalDb.metricSeries('stress')) { if (r['date'] == day) return (r['value'] as num?)?.toDouble(); } return null; } -Future bundleScore(String day) async { - final row = await LocalDb.dayResult(day); - if (row == null) return null; +Future> stored(String day, int version) async { + final db = await LocalDb.instance; + final row = (await db.query('day_result', + where: 'day_id = ? AND algo_version = ?', whereArgs: [day, version])) + .single; final payload = jsonDecode(row['payload_json'] as String) as Map; - final blk = payload['stress']; - return blk is Map ? (blk['score'] as num?)?.toDouble() : null; -} - -Future scalarStress(String day) async { - final row = await LocalDb.dayResult(day); - if (row == null) return null; - final payload = jsonDecode(row['payload_json'] as String) as Map; - final sc = payload['scalars']; - return sc is Map ? (sc['stress'] as num?)?.toDouble() : null; + return { + 'score': (payload['stress'] as Map)['score'], + 'value': (payload['stress'] as Map)['value'], + 'scalar': (payload['scalars'] as Map)['stress'], + }; } void main() { - setUpAll(() async { + setUpAll(() { sqfliteFfiInit(); databaseFactory = databaseFactoryFfi; LocalDb.dbName = 'openstrap_stress_backfill_test.db'; @@ -91,65 +84,81 @@ void main() { await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); }); - group('backfillStressScale — the stored history', () { - // Newest row 2026-06-10; a day this far back is comfortably before the - // rawRetentionDays cutoff, so it is rescaled, while 2026-06-09 is not. - const oldDay = '2026-05-01'; - final recentDay = dayLabelBefore('2026-06-10', 1)!; // within retention - const oddDay = '2026-05-02'; // has a trend score, but no SI in the bundle - const newestDay = '2026-06-10'; - - test('a flat-0 low night is rescaled from its own stored index', () async { - // SI 12 was exactly 0 under the floor of 20. - await seedStressDay(oldDay, si: 12, score: 0); - await seedStressDay(newestDay, si: 300, score: 73); - - final result = await backfillStressScale(); - expect(result.didWork, isTrue); - - final raw = stressScoreFromSi(12)!; - final want = ((raw * 10).round()) / 10; - expect(want, greaterThan(0)); // the whole point of #631 - // Trend and scalars carry the unrounded score, as the derive writes them. - expect(await seriesValue('stress', oldDay), closeTo(raw, 1e-9)); - expect(await scalarStress(oldDay), closeTo(raw, 1e-9)); - expect(await bundleScore(oldDay), closeTo(want, 1e-9)); - }); - - test('a day still inside the raw window is left for a real re-derive', - () async { - await seedStressDay(recentDay, si: 12, score: 0); - await seedStressDay(newestDay, si: 300, score: 73); - - await backfillStressScale(); - - expect(await seriesValue('stress', recentDay), 0); - expect(await scalarStress(recentDay), 0); - expect(await seriesValue('stress', newestDay), 73); - }); - - test('a day whose index is missing is left alone, never zeroed', () async { - await seedStressDay(oddDay, si: null, score: 40); // bundle SI missing - await seedStressDay(newestDay, si: 300, score: 73); - - final result = await backfillStressScale(); - - expect(result.skipped, 1); - expect(result.didWork, isFalse); - // Trend, scalars and block are unchanged: abstaining, not rewriting to 0. - expect(await seriesValue('stress', oddDay), 40); - expect(await scalarStress(oddDay), 40); - expect(await bundleScore(oddDay), 40); - }); - - test('it is a one-shot: a second pass rewrites nothing', () async { - await seedStressDay(oldDay, si: 12, score: 0); - await seedStressDay(newestDay, si: 300, score: 73); - - expect((await backfillStressScale()).didWork, isTrue); - expect(await LocalDb.computeFreshness(kStressRescaleKey), isNotNull); - final again = await backfillStressScale(); - expect(again.didWork, isFalse); - }); + final raw = stressScoreFromSi(12)!; + final rounded = ((raw * 10).round()) / 10; + + test('a flat-0 low night is rescaled in place from its own index', () async { + // SI 12 was exactly 0 under the floor of 20. + await seedStressDay('2026-05-01', si: 12, score: 0); + + final result = await backfillStressScale(); + expect(result.bundleDays, 1); + expect(result.seriesDays, 1); + + expect(rounded, greaterThan(0)); // the whole point of #631 + // Same row, same version: the derive gate, the strain and readiness + // one-shots and the day's provenance all see what they saw before. + final s = await stored('2026-05-01', 120); + expect(s['score'], rounded); + expect(s['value'], rounded); + expect(s['scalar'], closeTo(raw, 1e-9)); // unrounded, as the derive writes + expect(await seriesValue('2026-05-01'), closeTo(raw, 1e-9)); + }); + + test('a missing or wrong-typed index is skipped, never zeroed or thrown on', + () async { + await seedStressDay('2026-05-01', si: null, score: 40); + await seedStressDay('2026-05-02', si: '12', score: 41); + await seedStressDay('2026-05-03', si: 12, score: 0); + + final result = await backfillStressScale(); + + expect(result.skipped, 2); + for (final (day, v) in [('2026-05-01', 40.0), ('2026-05-02', 41.0)]) { + final s = await stored(day, 120); + expect(s['score'], v); + expect(s['scalar'], v); + expect(await seriesValue(day), v); + } + // A bad row does not stop the walk. + expect(await seriesValue('2026-05-03'), closeTo(raw, 1e-9)); + }); + + test('a partial newest row: the trend point follows the complete row', + () async { + await seedStressDay('2026-05-01', si: 12, score: 0); // wrote the series + await seedStressDay('2026-05-01', + si: 300, score: 73, partial: true, algoVersion: 121); + + await backfillStressScale(); + + expect(await seriesValue('2026-05-01'), closeTo(raw, 1e-9)); + expect((await stored('2026-05-01', 121))['score'], + ((stressScoreFromSi(300)! * 10).round()) / 10); + }); + + test('an imported day keeps its score', () async { + await seedStressDay('2026-05-01', si: 12, score: 0, imported: true); + + final result = await backfillStressScale(); + + expect(result.didWork, isFalse); + expect((await stored('2026-05-01', 120))['score'], 0); + expect(await seriesValue('2026-05-01'), 0); + }); + + test('an empty store is not marked done', () async { + expect((await backfillStressScale()).didWork, isFalse); + expect(await LocalDb.computeFreshness(kStressRescaleKey), isNull); + }); + + test('it is a one-shot: a second pass rewrites nothing', () async { + await seedStressDay('2026-05-01', si: 12, score: 0); + + expect((await backfillStressScale()).didWork, isTrue); + expect(await LocalDb.computeFreshness(kStressRescaleKey), isNotNull); + expect((await backfillStressScale()).didWork, isFalse); + // Forced, an already-rescaled store is a no-op. + expect((await backfillStressScale(force: true)).didWork, isFalse); }); } diff --git a/test/stress_score_floor_test.dart b/test/stress_score_floor_test.dart index cb50bbb86..c4ff48d21 100644 --- a/test/stress_score_floor_test.dart +++ b/test/stress_score_floor_test.dart @@ -14,6 +14,8 @@ void main() { expect(stressScoreFromSi(null), isNull); expect(stressScoreFromSi(0), isNull); expect(stressScoreFromSi(-1), isNull); + expect(stressScoreFromSi(double.nan), isNull); + expect(stressScoreFromSi(double.infinity), isNull); }); test('the floor is 0 and the ceiling is 100', () { From d235114f452a8ee398fd8e204829d9d02b7d8e58 Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:59:58 +0530 Subject: [PATCH 4/4] stress rescale: trend only from its writer row --- lib/compute/stress_backfill.dart | 22 ++++++++++++++++------ test/stress_rescale_backfill_test.dart | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/lib/compute/stress_backfill.dart b/lib/compute/stress_backfill.dart index e2f7d1f4e..b7b58f09a 100644 --- a/lib/compute/stress_backfill.dart +++ b/lib/compute/stress_backfill.dart @@ -89,6 +89,13 @@ Future backfillStressScale({bool force = false}) async { where: 'day_id = ?', whereArgs: [day], orderBy: 'algo_version DESC'); + // The build that last wrote this day's series, when stamped. + final stamp = ((await txn.query('metric_series_version', + columns: ['algo_version'], + where: 'date = ?', + whereArgs: [day])) + .firstOrNull?['algo_version'] as num?) + ?.toInt(); double? seriesScore; var seriesPicked = false; for (final row in rows) { @@ -96,6 +103,15 @@ Future backfillStressScale({bool force = false}) async { final blk = payload?['stress']; final si = blk is Map ? _finite(blk['si']) : null; final next = stressScoreFromSi(si); + // The trend point comes ONLY from the row that wrote it: the stamped + // writer, else the newest complete row (a partial row never writes the + // series). If that row has no usable score the trend stays as it is. + if (!seriesPicked && + (row['partial'] as num?)?.toInt() != 1 && + (stamp == null || (row['algo_version'] as num?)?.toInt() == stamp)) { + seriesPicked = true; + if (payload?['imported'] != true) seriesScore = next; + } // No usable index, or another source's day: a rescaled day must not // become 0 (a score, not an absence), and an imported score is not // ours to re-price. @@ -103,12 +119,6 @@ Future backfillStressScale({bool force = false}) async { skipped++; continue; } - // The trend point was written by the newest COMPLETE row (a partial - // row never writes the series), so it is rescaled from that row's index. - if (!seriesPicked && (row['partial'] as num?)?.toInt() != 1) { - seriesPicked = true; - seriesScore = next; - } final rounded = _round1(next); final scalars = payload['scalars']; if ((blk as Map)['score'] == rounded && diff --git a/test/stress_rescale_backfill_test.dart b/test/stress_rescale_backfill_test.dart index 9eab06be2..d922bee16 100644 --- a/test/stress_rescale_backfill_test.dart +++ b/test/stress_rescale_backfill_test.dart @@ -137,6 +137,28 @@ void main() { ((stressScoreFromSi(300)! * 10).round()) / 10); }); + test('the trend follows only its writer row, never an older one', () async { + await seedStressDay('2026-05-01', si: 12, score: 0); + // Newest complete row has no usable index: the trend stays as it wrote it. + await seedStressDay('2026-05-01', si: null, score: 40, algoVersion: 121); + + await backfillStressScale(); + + expect(await seriesValue('2026-05-01'), 40); + // The older bundle is still rescaled on its own. + expect((await stored('2026-05-01', 120))['score'], rounded); + }); + + test('a stamped series writer wins over a newer complete row', () async { + await seedStressDay('2026-05-01', si: 300, score: 73, algoVersion: 121); + // A rolled-back build rewrote the series (and its stamp) at v120. + await seedStressDay('2026-05-01', si: 12, score: 0); + + await backfillStressScale(); + + expect(await seriesValue('2026-05-01'), closeTo(raw, 1e-9)); + }); + test('an imported day keeps its score', () async { await seedStressDay('2026-05-01', si: 12, score: 0, imported: true);