Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 40 additions & 6 deletions lib/compute/derivation_engine.dart
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import 'dart:math' as math;

import 'quiet_level_seed.dart' show seedQuietHrrHistoryOnce;
import 'strain_backfill.dart' show backfillStrainScale;
import 'stress_backfill.dart' show backfillStressScale;
import 'readiness_calibration_backfill.dart'
show backfillReadinessCalibration, seedRecentReadinessZ;

Expand Down Expand Up @@ -1893,7 +1894,14 @@ import 'vendor_sleep.dart';
// no window does, instead of the longest of the search window's sessions.
// HrvFreqStream v2, CardioRowCache v3 and policy-bound PRSA states require cold
// rebuilds of old checkpoints, isolated by this algorithm version.
const int kAlgoVersion = 124;
// 124 → 125 (edge-side recalibration, no sibling repin): stress score. The
// Baevsky SI -> 0-100 map floored at SI 20, clamping every genuinely low night
// (SI < 50 is Baevsky's own "low" band) to a flat 0 and losing all variation
// for high-HRV sleepers (#631). `kStressScoreFloorSi` is now 5.
// stress_backfill.dart rescales stored history in place from each day's own
// `stress.si` so the trend has no step at the bump. User-visible: every stress score moves
// up once (e.g. SI 160 reads ~72, was ~61); nights that read 0 now read low.
const int kAlgoVersion = 125;
Comment thread
abdulsaheel marked this conversation as resolved.
Resolved
/// The sibling SHAs this version was derived against, asserted against
/// pubspec.yaml in test/db_serve_version_and_reads_test.dart.
///
Expand Down Expand Up @@ -4205,6 +4213,26 @@ class DerivationEngine {
} catch (e) {
_log('[derive] strain rescale failed (kept old values): $e');
}
// Same one-shot shape as the strain rescale and for the same reason:
// low nights older than `rawRetentionDays` keep a floor-20 score that a
// version bump alone cannot reach. Pure function of each day's stored
// `stress.si`, so it needs no baseline (#631).
_diag['stage'] = 'stress_rescale';
var stressRescaled = false;
try {
final rescaled = await backfillStressScale();
Comment thread
abdulsaheel marked this conversation as resolved.
Resolved
if (rescaled.didWork) {
stressRescaled = true;
_log('[derive] stress rescale: ${rescaled.bundleDays} row(s), '
'${rescaled.seriesDays} trend point(s) rescaled in place, '
'${rescaled.skipped} skipped (no stored index or imported)');
}
} catch (e) {
// Each day commits on its own, so days before the failure stay
// rescaled; the marker is not written, so the next pass resumes.
stressRescaled = true;
_log('[derive] stress rescale stopped, will retry next pass: $e');
}
// ONE-SHOT, after the sweep for the same reason as the strain rescale:
// the rest of the readiness_z history, then a re-score of every day
// below v112 that has no raw substrate left (substrate days were just
Expand Down Expand Up @@ -4236,10 +4264,10 @@ class DerivationEngine {
} catch (e) {
_log('[derive] readiness calibration failed (will retry): $e');
}
// The strain rescale rewrote stored days after the rollup of this pass
// was built from the old values. When the calibration above did not
// The strain or stress rescale rewrote stored days after the rollup of
// this pass was built from the old values. When the calibration above did not
// rebuild it (no work, or it failed), rebuild it here.
if (strainRescaled && !crossDayRebuilt) {
if ((strainRescaled || stressRescaled) && !crossDayRebuilt) {
try {
await _refreshBaselines();
await _runCrossDay(profile);
Expand Down Expand Up @@ -8610,7 +8638,8 @@ class DerivationEngine {
String name, int nowOffsetMin, int janOffsetMin, int julOffsetMin) =>
'$name|$nowOffsetMin|$janOffsetMin|$julOffsetMin';

/// Whether the stored cross-day ROLLUP was built in another zone. The
/// Whether the stored cross-day ROLLUP was built in another zone or by
/// another algorithm version. The
/// rollup, not its inputs: [_refreshBaselines] re-stamps the inputs before
/// the rollup is written, so a rollup that then timed out would read as
/// current on the inputs' stamp and never be retried.
Expand All @@ -8619,7 +8648,12 @@ class DerivationEngine {
if (raw is! String || raw.isEmpty) return false;
try {
final decoded = jsonDecode(raw);
return decoded is Map && decoded['zone'] != crossDayZoneStamp();
// A rollup from another algorithm version is rejected by its readers
// (`getInsights`), so it is as stale as one from another zone: rebuild it
// from stored days even when nothing else in the pass has work.
return decoded is Map &&
(decoded['zone'] != crossDayZoneStamp() ||
(decoded['algo_version'] as num?)?.toInt() != kAlgoVersion);
} catch (_) {
return false;
}
Expand Down
27 changes: 22 additions & 5 deletions lib/compute/onehz_pipeline.dart
Original file line number Diff line number Diff line change
Expand Up @@ -1672,11 +1672,7 @@ Map<String, dynamic> _deriveDayBundle(Map<String, dynamic> inputJson,
// stress screen reads this {score, si, lf_hf, rmssd, level} block directly.
final si = stress.present ? stress.value!.si : null;
final lfhf = hrvF.present ? hrvF.value!.lfhf : null;
double? stressScore;
if (si != null && si > 0) {
final lo = math.log(20), hi = math.log(600);
stressScore = (100 * (math.log(si) - lo) / (hi - lo)).clamp(0.0, 100.0);
}
final stressScore = stressScoreFromSi(si);
Comment thread
abdulsaheel marked this conversation as resolved.
Resolved
final stressBlock = <String, dynamic>{
'value': stressScore == null ? '—' : _round(stressScore, 1),
'score': stressScore == null ? null : _round(stressScore, 1),
Expand Down Expand Up @@ -2371,6 +2367,27 @@ double? _stddev(List<double> xs) {
return math.sqrt(s / (xs.length - 1));
}

/// Baevsky Stress Index → a 0-100 resting-tension score, log-mapped over the
/// plausible resting SI range [kStressScoreFloorSi, kStressScoreCeilingSi].
/// Null when there is no usable index (never 0 for absence — 0 is a real score
/// at the floor).
///
/// CALIBRATION: the floor was 20, which clamped every genuinely low-tension
/// night (Baevsky's own "low" band is SI < 50) to a flat 0 and lost all
/// information for high-HRV sleepers (#631). A lower floor restores variation
/// at the bottom. Moving it shifts every user's number, so it is changed
/// deliberately, with a kAlgoVersion bump and a one-shot rescale of stored
/// history (stress_backfill.dart). ONE source for the mapping: the live derive
/// above and that rescale both call this.
const double kStressScoreFloorSi = 5;
const double kStressScoreCeilingSi = 600;

double? stressScoreFromSi(double? si) {
if (si == null || !si.isFinite || si <= 0) return null;
final lo = math.log(kStressScoreFloorSi), hi = math.log(kStressScoreCeilingSi);
return (100 * (math.log(si) - lo) / (hi - lo)).clamp(0.0, 100.0);
}

double _round(double v, int dp) {
final p = math.pow(10, dp);
return (v * p).round() / p;
Expand Down
182 changes: 182 additions & 0 deletions lib/compute/stress_backfill.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
// ONE-SHOT BACKFILL — stored stress scores onto the recalibrated floor (#631).
//
// The Baevsky SI → 0-100 map used to floor at SI 20, so every genuinely low
// night (Baevsky's own "low" band is SI < 50) read as a flat 0 and high-HRV
// sleepers saw 0 every single night. The floor is now `kStressScoreFloorSi`.
// Every day derived before that change carries a score on the old scale, so the
// stress trend and the day-detail screen would show a step at the release date
// rather than a real change.
//
// WHY NOT JUST RE-DERIVE: raw 1 Hz substrate is pruned `rawRetentionDays` behind
// the data edge, so a kAlgoVersion bump alone can only fix the last few days.
//
// It does not need raw. The new score is a PURE function of the Baevsky index,
// and each stored day_result already carries its own `stress.si`.
//
// IN PLACE, NOT A NEW VERSION ROW. The rescale rewrites only the stress score
// (`stress.score`/`value`, `scalars.stress`, the `stress` trend point) of the
// rows that exist, at the version they already have. It does not stamp a day
// current, so:
// * a day that still has raw is still re-derived by the version gate (no
// guess at where the raw window ends is needed);
// * the strain and readiness one-shots, which key on the row's version,
// still see every day they are owed;
// * provenance (device family, coverage, method stamps, naps) is untouched.
// The rewrite is idempotent (score = f(stored si)), so a row a fresh derive
// already wrote is a no-op.
//
// ONE source for the mapping: `stressScoreFromSi`, the same function the live
// derive calls.

import 'dart:convert';

import '../data/db.dart';
import '../data/series_codec.dart';
import 'onehz_pipeline.dart' show stressScoreFromSi;

/// `compute_freshness` key marking the rescale as applied. Bumped with the algo
/// version so a later recalibration is a new one-shot rather than a no-op.
const String kStressRescaleKey = 'stress_rescale_v125';

class StressBackfillResult {
/// Days whose `metric_series` stress score was rewritten (trend / v_daily).
final int seriesDays;

/// `day_result` rows whose stress score was rewritten in place.
final int bundleDays;

/// Rows left exactly as they were: no usable index, imported, or unreadable.
final int skipped;

const StressBackfillResult({
required this.seriesDays,
required this.bundleDays,
required this.skipped,
});

bool get didWork => seriesDays > 0 || bundleDays > 0;
}

/// Rescale every stored stress score in place. A one-shot once it has walked a
/// non-empty store; an empty store is not marked, so rows a restore merges in
/// later are still rescaled. Set [force] to re-run.
Future<StressBackfillResult> backfillStressScale({bool force = false}) async {
const none = StressBackfillResult(seriesDays: 0, bundleDays: 0, skipped: 0);
if (!force && await LocalDb.computeFreshness(kStressRescaleKey) != null) {
return none;
}

final db = await LocalDb.instance;
final days = [
for (final r in await db.rawQuery(
'SELECT DISTINCT day_id FROM day_result WHERE day_id IS NOT NULL '
'ORDER BY day_id'))
r['day_id'] as String,
];
if (days.isEmpty) return none;

var seriesDays = 0;
var bundleDays = 0;
var skipped = 0;

for (final day in days) {
// One transaction per day: the rows are read and rewritten together, so a
// derive in another isolate cannot land between the read and the write
// and be overwritten by an older payload.
await db.transaction((txn) async {
final rows = await txn.query('day_result',
columns: ['algo_version', 'payload_json', 'partial'],
where: 'day_id = ?',
whereArgs: [day],
orderBy: 'algo_version DESC');
// The build that last wrote this day's series, when stamped.
final stamp = ((await txn.query('metric_series_version',
columns: ['algo_version'],
where: 'date = ?',
whereArgs: [day]))
.firstOrNull?['algo_version'] as num?)
?.toInt();
double? seriesScore;
var seriesPicked = false;
for (final row in rows) {
final payload = _decode(row['payload_json']);
final blk = payload?['stress'];
final si = blk is Map ? _finite(blk['si']) : null;
final next = stressScoreFromSi(si);
// The trend point comes ONLY from the row that wrote it: the stamped
// writer, else the newest complete row (a partial row never writes the
// series). If that row has no usable score the trend stays as it is.
if (!seriesPicked &&
(row['partial'] as num?)?.toInt() != 1 &&
(stamp == null || (row['algo_version'] as num?)?.toInt() == stamp)) {
seriesPicked = true;
if (payload?['imported'] != true) seriesScore = next;
}
// No usable index, or another source's day: a rescaled day must not
// become 0 (a score, not an absence), and an imported score is not
// ours to re-price.
if (payload == null || next == null || payload['imported'] == true) {
skipped++;
continue;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
final rounded = _round1(next);
final scalars = payload['scalars'];
if ((blk as Map)['score'] == rounded &&
blk['value'] == rounded &&
(scalars is! Map || scalars['stress'] == next)) {
continue; // already on the new scale
}
// `value`/`score` are the rounded headline the screens read; the
// derive also writes the unrounded score into `scalars`.
blk['score'] = rounded;
blk['value'] = rounded;
if (scalars is Map) scalars['stress'] = next;
await txn.update(
'day_result',
{'payload_json': SeriesCodec.encodePayloadJson(jsonEncode(payload))},
where: 'day_id = ? AND algo_version = ?',
whereArgs: [day, row['algo_version']],
);
bundleDays++;
}
if (seriesScore == null) return;
// UPDATE only: never create a trend point the derive did not write, and
// leave a day whose series another source produced alone.
final n = await txn.rawUpdate(
"UPDATE metric_series SET value = ? WHERE date = ? AND key = 'stress' "
'AND (value IS NULL OR value <> ?) AND NOT EXISTS (SELECT 1 FROM '
'metric_series_version v WHERE v.date = ? AND v.source IS NOT NULL '
"AND v.source <> 'band')",
[seriesScore, day, seriesScore, day],
);
if (n > 0) seriesDays++;
});
}

await LocalDb.putComputeFreshness(
kStressRescaleKey, jsonEncode({'done': true}));
return StressBackfillResult(
seriesDays: seriesDays,
bundleDays: bundleDays,
skipped: skipped,
);
}

/// A finite number, or null — a wrong-typed index is skipped, not thrown on.
double? _finite(Object? v) {
if (v is! num) return null;
final d = v.toDouble();
return d.isFinite ? d : null;
}

double _round1(double v) => (v * 10).round() / 10;

Map<String, dynamic>? _decode(Object? json) {
if (json is! String) return null;
try {
final v = jsonDecode(json);
return v is Map ? v.cast<String, dynamic>() : null;
} catch (_) {
return null;
}
}
Loading