Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions lib/compute/cardio_rows_cache.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import 'dart:convert';
import 'dart:typed_data';

import 'package:openstrap_analytics/onehz.dart' as ana;

/// Caller-owned bytes and provenance. Row objects exist only in the worker.
class CardioRowsChunk {
CardioRowsChunk(this.bytes, this.loSec, this.hiSec, this.revisions,
this.generation, this.algorithm);
final Uint8List bytes;
final int loSec, hiSec;
final Map<String, (int, int, int)> revisions;
int generation, algorithm;

bool overlaps(int lo, int hi) => hiSec >= lo && loSec <= hi;

bool matches(Map<String, (int, int, int)> current, int gen, int algo) =>
generation == gen && algorithm == algo &&
current.entries.every((e) =>
!revisions.containsKey(e.key) || revisions[e.key] == e.value);
}

/// Worker only. Damage or an unknown codec version costs a cold staging.
ana.CardioRowCache decodeCardioRows(List<Uint8List> chunks) {
final rows = ana.CardioRowCache();
for (final bytes in chunks) {
try {
rows.merge(ana.CardioRowCache.fromJson(
(jsonDecode(utf8.decode(bytes)) as Map).cast<String, dynamic>()));
} on FormatException {
// Refuse the whole chunk.
} on TypeError {
// A structurally invalid JSON value is also a miss.
}
}
return rows;
}

/// Keep at most one search window plus slack and 8,000 row variants. A limit
/// only evicts reusable work; analytics always recomputes a missing row.
Uint8List encodeCardioRows(ana.CardioRowCache rows, int lo, int hi) {
var kept = 0;
rows.retainWhere((k) =>
k.startSec >= lo && k.startSec <= hi && kept++ < 8000);
return Uint8List.fromList(utf8.encode(jsonEncode(rows.toJson())));
}
669 changes: 638 additions & 31 deletions lib/compute/derivation_engine.dart

Large diffs are not rendered by default.

45 changes: 44 additions & 1 deletion lib/compute/derive_prepare.dart
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import 'package:openstrap_analytics/onehz.dart' as ana;
import '../ble/adapters/signals.dart';
import '../data/coverage_resolver.dart';
import 'substrate.dart';
import 'substrate_retention.dart';
import 'substrate_union.dart';
import 'vendor_sleep.dart';

class PreparedDerivationDay {
Expand Down Expand Up @@ -317,6 +319,9 @@ void derivationPrepareWorker(SendPort mainSendPort) {
final state = _PrepareAccumulator();
String? targetDay;
var mode = 'prepared_day';
SubstrateUnionWindow? unionWindow;
var loadFrom = 0, loadTo = -1;
Map<InputSignal, List<OwnedSpan>> loadOwnership = const {};
mainSendPort.send(port.sendPort);

void handle(Object? message) {
Expand Down Expand Up @@ -348,6 +353,10 @@ void derivationPrepareWorker(SendPort mainSendPort) {
final cfgMode = message['mode']?.toString();
if (cfgMode != null && cfgMode.isNotEmpty) mode = cfgMode;
state.retain = message['retain'] == true;
unionWindow = message['union_window'] as SubstrateUnionWindow?;
loadFrom = message['load_from'] as int? ?? 0;
loadTo = message['load_to'] as int? ?? -1;
loadOwnership = message['ownership'] as Map<InputSignal, List<OwnedSpan>>? ?? const {};
return;
}
if (type == 'seed') {
Expand All @@ -358,11 +367,20 @@ void derivationPrepareWorker(SendPort mainSendPort) {
try {
final substrate = state.buildSubstrate();
if (mode == 'substrate') {
final extras = state.retainable ? state.retainExtras() : null;
final request = unionWindow;
final window = request == null || extras == null ? null :
cutSubstrateUnion(RetainedSubstrate(version: 0, context: '',
revisions: const {}, sub: substrate, extras: extras, cut: loadFrom,
from: loadFrom, to: loadTo, ownership: loadOwnership),
request.from, request.to, request.ownership,
retainMetadata: false)?.sub;
mainSendPort.send({
'type': 'result',
'kind': 'substrate',
'payload': substrate.toJson(),
if (state.retainable) 'retain': state.retainExtras(),
'retain': ?extras,
if (window != null) 'union_window': window.toJson(),
});
} else {
final payload = prepareDerivationPayload(
Expand Down Expand Up @@ -415,13 +433,17 @@ PreparedDerivationPayload prepareDerivationPayload(
List<({int startSec, int endSec, String dayKey})> priorSleep = const [],
List<VendorNight> vendorNights = const [],
bool skipUnusedDays = false,
ana.CardioRowCache? rowCache,
bool recordTargetOnly = false,
}) {
if (sub.isEmpty || sub.lastTs == null) {
return const PreparedDerivationPayload(dataNowSec: 0, days: []);
}
final days = <PreparedDerivationDay>[];
for (final day in calendarDays(
sub,
rowCache: rowCache,
recordTargetOnly: recordTargetOnly,
override: override,
targetDay: targetDay,
skipUnusedDays: skipUnusedDays,
Expand Down Expand Up @@ -480,6 +502,8 @@ SleepSessionCandidate prepareSleepSessionCandidate(
List<({int startSec, int endSec, String dayKey})> priorSleep = const [],
List<VendorNight> vendorNights = const [],
bool skipUnusedDays = false,
ana.CardioRowCache? rowCache,
bool recordTargetOnly = false,
}) {
// An UNCLAIMED device night (our rows saw under half of it) ending today IS
// the night: above our detection, below the user's word. It never reaches
Expand Down Expand Up @@ -523,6 +547,8 @@ SleepSessionCandidate prepareSleepSessionCandidate(
claimed.add(n);
}
final payload = prepareDerivationPayload(sub,
rowCache: rowCache,
recordTargetOnly: recordTargetOnly,
targetDay: targetDay,
override: override,
skipUnusedDays: skipUnusedDays,
Expand Down Expand Up @@ -653,6 +679,7 @@ class _PrepareAccumulator {
final List<int> _rrStart = [];
final Map<int, _BucketAcc> _buckets = {};
Float64List? _rrTsRaw;
final List<List<Object?>> _provenance = [];

void seed(Map message) {
List<int> ints(String k) => message[k] as List<int>;
Expand Down Expand Up @@ -681,6 +708,11 @@ class _PrepareAccumulator {
rrTsMs.addAll(dbls('rr_ts_raw').getRange(0, beats));
rrMs.addAll(dbls('rr_ms').getRange(0, beats));
_rrStart.addAll(starts.getRange(0, n));
for (final run in (message['provenance'] as List?) ?? const []) {
final r = (run as List).cast<Object?>();
if ((r[0] as int) >= n) break;
_provenance.add(List<Object?>.from(r));
}
for (final wire in message['meta'] as List) {
final m = wire as List;
final bucket = m[0] as int;
Expand All @@ -698,6 +730,7 @@ class _PrepareAccumulator {
Map<String, Object?> retainExtras() => {
'rr_start': Int32List.fromList(_rrStart),
'rr_ts_raw': _rrTsRaw!,
'provenance': _provenance,
'meta': [for (final e in _buckets.entries)
[e.key, e.value.families.toList(), e.value.deviceIds.toList(),
e.value.skinUnit]],
Expand Down Expand Up @@ -779,6 +812,16 @@ class _PrepareAccumulator {
if (retain) {
_rrStart.add(rrMs.length);
_buckets.putIfAbsent(recTs ~/ 900, _BucketAcc.new).note(row);
final family = row?['device_family'];
final id = row?['device_id'];
final unit = row?['skin_temp_raw'] is num ? 'raw' :
(row?['skin_temp_c'] is num ? 'c' : null);
final f = family is String && family.isNotEmpty ? family : null;
final d = id is String ? id : null;
final last = _provenance.isEmpty ? null : _provenance.last;
if (last == null || last[1] != f || last[2] != d || last[3] != unit) {
_provenance.add([tsSec.length, f, d, unit]);
}
}
tsSec.add(recTs);
// NULL (absent), a beat-only second (no row at all) and an impossible
Expand Down
24 changes: 24 additions & 0 deletions lib/compute/ed2b_checkpoint_diagnostics.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import 'dart:typed_data';

/// Observations of the real checkpoint path. No storage, codec or resume policy
/// lives here. Counters remain zero until the GREEN implementation reports work.
class Ed2bCheckpointDiagnostics {
bool disabled = false;
int reads = 0;
int writes = 0;
int hits = 0;
int misses = 0;
final List<String> rejections = [];
Uint8List? lastEncodedPayload;
Map<String, Object?>? lastDecodedPayload;

void reset() {
reads = 0;
writes = 0;
hits = 0;
misses = 0;
rejections.clear();
lastEncodedPayload = null;
lastDecodedPayload = null;
}
}
45 changes: 45 additions & 0 deletions lib/compute/ed2b_stream_diagnostics.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import 'night_rr_state.dart';

// Work is counted beside the real owner and the remaining batch metric calls.
class Ed2bStreamDiagnostics {
int passes = 0;
NightRrState nightState = NightRrState();
int get processedNightBeats => nightState.processedNightBeats;
int get nightRebuilds => nightState.nightRebuilds;
final Map<String, int> _batchBeats = {};
final Map<String, int> _batchCalls = {};
bool disabled = false;
void Function(List<double> nn, List<double> times, double cleanFraction)?
onCorrected;
void Function()? onBeforePassReturn;
void Function(List<double>)? onRespWindows;

void beginPass() { passes++; }
void endPass() { onBeforePassReturn?.call(); }
void context({required int inputRevision, required int generation,
required int algorithm}) => nightState.context(inputRevision: inputRevision,
generation: generation, algorithm: algorithm);
// Total input work: GREEN counts only newly appended/settled beats here,
// regardless of whether the operation uses a batch call or an addSlice.
int get retainedNightBeats => nightState.retainedNightBeats;
int get settledNightBeats => nightState.settledNightBeats;
int get identityMemoEntries => nightState.identityMemoEntries;
int get fullListDigests => nightState.fullListDigests;
int get maximumListDigestsPerIdentity => nightState.maximumListDigestsPerIdentity;
int retainedStreamBeats(String stream) => nightState.retainedStreamBeats(stream);
int processedBeats(String stream) => nightState.processedBeats(stream) +
(_batchBeats[stream] ?? 0);
int slicesProcessed(String stream) => nightState.slicesProcessed(stream) +
(_batchCalls[stream] ?? 0);
int streamRestarts(String stream) => nightState.streamRestarts(stream);

void observeNightCorrection(int rawBeats, List<double> nn,
List<double> times, double cleanFraction) {
onCorrected?.call(nn, times, cleanFraction);
}

void observeBatch(String stream, int beats) {
_batchCalls[stream] = (_batchCalls[stream] ?? 0) + 1;
_batchBeats[stream] = (_batchBeats[stream] ?? 0) + beats;
}
}
115 changes: 115 additions & 0 deletions lib/compute/night_checkpoint.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import 'dart:convert';
import 'dart:typed_data';

import 'package:crypto/crypto.dart';
import 'package:openstrap_analytics/onehz.dart' as ana;

import 'cardio_rows_cache.dart';
import 'night_rr_state.dart';

// Format 2 carries RSA v3 and closed-bin spectral states, not saved rates.
// Older closed windows cannot be certified under the current RSA floor gate.
const nightCheckpointFormat = 2;
const maxNightCheckpointBytes = 8 * 1024 * 1024;

/// One request's provenance, read before loading substrate. No decoded rows.
class NightCheckpointPlan {
NightCheckpointPlan(this.from, this.through, this.revisions, this.generation,
this.epoch, this.storeId);
final int from, through, generation, epoch;
final String storeId;
final Map<int, int> revisions;
String? context;
int boundary = 0;
List<Uint8List> cardio = [];
NightRrState? restored;
}

Uint8List checkpointRevisions(Map<int, int> revisions, int from, int boundary) =>
Uint8List.fromList(utf8.encode(jsonEncode([
for (var b = from ~/ 900; b < boundary ~/ 900; b++)
[b, revisions[b] ?? 0],
])));

typedef NightCheckpointRead = ({NightRrState? night, Uint8List? cardio,
Map<String, Object?>? payload, String? rejection});
typedef NightCheckpointWrite = ({Uint8List bytes, Map<String, Object?> payload});

/// Worker only. SHA-256 covers both sections and all SQL provenance. No section
/// is returned unless every section and the complete revision vector validate.
NightCheckpointRead decodeNightCheckpoint(Map<String, Object?> row,
String day, int algorithm, String context, int from, int through,
Map<int, int> revisions, {List<double>? searchRr, List<double>? searchTimes}) {
try {
final boundary = row['cp_rec_ts'];
if (row['day_id'] != day || row['algo_version'] != algorithm ||
row['fmt'] != nightCheckpointFormat || row['ctx_sig'] != context ||
row['night_ref'] != null || boundary is! int || boundary % 900 != 0 ||
boundary <= from || boundary > through + 1) {
throw const FormatException('checkpoint provenance');
}
final expectedRevisions = checkpointRevisions(revisions, from, boundary);
if (base64Encode(row['rev_vec'] as List<int>) != base64Encode(expectedRevisions)) {
throw const FormatException('checkpoint revisions');
}
final bytes = row['state'] as List<int>;
if (bytes.length <= 32 || bytes.length > maxNightCheckpointBytes) {
throw const FormatException('checkpoint length');
}
final body = bytes.sublist(32);
if (base64Encode(sha256.convert(body).bytes) != base64Encode(bytes.sublist(0, 32))) {
throw const FormatException('checkpoint checksum');
}
final payload = (jsonDecode(utf8.decode(body)) as Map).cast<String, Object?>();
if (payload.length != 4 || payload['v'] != nightCheckpointFormat ||
jsonEncode(payload['provenance']) != jsonEncode([
day, algorithm, context, boundary, base64Encode(expectedRevisions),
])) {
throw const FormatException('checkpoint envelope');
}
final night = NightRrState.fromCheckpoint(
(payload['night'] as Map).cast<String, dynamic>(), boundary);
if (((payload['night'] as Map)['tag'] as Map)['day'] != day ||
(searchRr != null && (searchTimes == null ||
!night.acceptsSearchPrefix(searchRr, searchTimes, day)))) {
throw const FormatException('checkpoint raw prefix');
}
final cardioJson = (payload['cardio'] as Map).cast<String, dynamic>();
final cardio = ana.CardioRowCache.fromJson(cardioJson);
if (cardio.length > 8000) throw const FormatException('checkpoint cardio size');
var outside = false;
cardio.retainWhere((k) {
if (k.startSec < from - 3600 || k.startSec + k.epochSec > boundary) outside = true;
return true;
});
if (outside) throw const FormatException('checkpoint cardio window');
return (night: night,
cardio: Uint8List.fromList(utf8.encode(jsonEncode(cardioJson))),
payload: payload, rejection: null);
} catch (e) {
return (night: null, cardio: null, payload: null,
rejection: 'checkpoint refused: ${e.runtimeType}');
}
}

/// Worker only. Oversized or absent nights stay on the ordinary cold path.
NightCheckpointWrite? encodeNightCheckpoint(NightRrState? night,
List<Uint8List> chunks, String day, int algorithm, String context,
int from, int boundary, Uint8List revisions) {
final nightJson = night?.checkpoint(boundary);
if (nightJson == null) return null;
final cardio = decodeCardioRows(chunks);
var kept = 0;
cardio.retainWhere((k) => k.startSec >= from - 3600 &&
k.startSec + k.epochSec <= boundary && kept++ < 8000);
final payload = <String, Object?>{
'v': nightCheckpointFormat,
'provenance': [day, algorithm, context, boundary, base64Encode(revisions)],
'night': nightJson, 'cardio': cardio.toJson(),
};
final body = utf8.encode(jsonEncode(payload));
if (body.length + 32 > maxNightCheckpointBytes) return null;
final bytes = Uint8List.fromList([...sha256.convert(body).bytes, ...body]);
return (bytes: bytes,
payload: (jsonDecode(utf8.decode(body)) as Map).cast<String, Object?>());
}
Loading
Loading