From 12246ea8e2e5255365a13c24cb5e32ec1525a096 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 15:30:14 -0700 Subject: [PATCH 1/6] test: archive failures keep live rows; stuck bucket must not starve background archiving A bucket whose archive fails stays live whatever its age, error type or failure count: the archive is the only durable copy once live rows are deleted. A bucket that keeps failing must stop spending the background pass's single attempt. The former assertions that required deleting unarchived rows past the hard floor change deliberately. These tests fail on cd64eaa89. --- test/substrate_archive_db_test.dart | 177 +++++++++++++++------------- 1 file changed, 96 insertions(+), 81 deletions(-) diff --git a/test/substrate_archive_db_test.dart b/test/substrate_archive_db_test.dart index 0fc3b18fc..db4a24e86 100644 --- a/test/substrate_archive_db_test.dart +++ b/test/substrate_archive_db_test.dart @@ -4,7 +4,6 @@ // export day, restore/salvage, schema health, the prune guard). import 'dart:async'; -import 'dart:convert'; import 'dart:io'; import 'dart:math' as math; import 'dart:typed_data'; @@ -101,7 +100,6 @@ Future _clear() async { SubstrateArchive.debugCorruptEncode = false; SubstrateArchive.debugBeforeBucket = null; SubstrateArchive.debugThrowInBucket = null; - SubstrateArchive.debugBeforeFallbackDelete = null; LocalDb.debugFailExportStrip = false; } @@ -398,8 +396,9 @@ void main() { expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); }); - test('verify failure keeps rows live inside the hold window and deletes ' - 'past hardFloorSec', () async { + // WHY: the archive is the only durable copy once live rows are deleted. + // CLAIM(archive-before-delete); catches: hard-floor fallback deletion. + test('verify failure keeps rows live even past hardFloorSec', () async { await _seed(); final liveBefore = await _live( 'decoded_onehz', @@ -422,14 +421,16 @@ void main() { expect(logs, hasLength(4)); expect(logs.first, contains('kept live')); - // Past the bounded hold the old behaviour applies: deleted unarchived. + // Age never authorizes deleting rows whose archive failed verification. await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, hardFloorSec: cutoff, ); - expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); - expect(await _live('decoded_rr', 'rec_ts < ?', [cutoff]), isEmpty); + expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + liveBefore); + expect(await _live('decoded_rr', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + isNotEmpty); expect(await (await _db()).query('substrate_archive'), isEmpty); }); @@ -1469,6 +1470,57 @@ void main() { ); }); + // WHY: background passes archive ONE bucket (maxArchiveBuckets: 1), oldest + // first. A bucket that can never archive must not spend that one attempt on + // every pass, or background archiving would stall behind it for good. + // CLAIM(archive-before-delete); catches: a stuck oldest bucket starving the rest. + test('a bucket that keeps failing stops consuming the background budget', + () async { + await _seed(); + // Fails only the FIRST (oldest) bucket of every pass. The hook runs once + // per bucket, in order, before the pass decides whether to attempt it. + var seen = 0; + SubstrateArchive.debugBeforeBucket = () async { + SubstrateArchive.debugThrowInBucket = + seen++ == 0 ? StateError('codec rejected bucket') : null; + }; + Future pass() async { + seen = 0; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, hardFloorSec: cutoff, maxArchiveBuckets: 1); + } + + for (var n = 1; n <= 3; n++) { + await pass(); + expect(await (await _db()).query('substrate_archive'), isEmpty, + reason: 'pass $n spends its one attempt on the failing bucket'); + } + await pass(); + expect(await (await _db()).query('substrate_archive'), hasLength(1), + reason: 'after three failures the next bucket gets the attempt'); + expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + isNotEmpty, reason: 'the stuck bucket itself is still live'); + }); + + // WHY: skipping is a background-only economy; a foreground pass has no + // bucket cap, so it must retry the stuck bucket and clear its count. + // CLAIM(archive-before-delete); catches: a bucket skipped forever. + test('a foreground pass retries a bucket the background skips', () async { + await _seed(); + SubstrateArchive.debugThrowInBucket = StateError('codec rejected bucket'); + for (var n = 0; n < 3; n++) { + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, hardFloorSec: cutoff, maxArchiveBuckets: 1); + } + SubstrateArchive.debugThrowInBucket = null; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy); + expect(await (await _db()).query('substrate_archive'), hasLength(4)); + expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); + final counts = await (await _db()).query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]); + expect(counts.single['payload_json'], '{}'); + }); + test( 'readBlob returns a blob larger than one chunk, byte for byte', () async { @@ -1546,91 +1598,54 @@ void main() { }, ); - test('a failed unarchived delete neither stops the pass nor loses the ' - 'failure counts', () async { + // WHY: a bad bucket must stay live while independent buckets still archive. + // CLAIM(archive-before-delete); catches: aborting the loop on first error. + test('a failed bucket stays live while later buckets archive', () async { await _seed(); - SubstrateArchive.debugThrowInBucket = Exception('out of memory'); - for (var pass = 1; pass < SubstrateArchive.maxFailedPasses; pass++) { - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); - } - // The pass that would delete unarchived: the FIRST bucket's delete fails - // (a concurrent writer holding the lock). var calls = 0; - SubstrateArchive.debugBeforeFallbackDelete = () async { - if (calls++ == 0) throw Exception('database is locked'); + SubstrateArchive.debugBeforeBucket = () async { + SubstrateArchive.debugThrowInBucket = calls++ == 0 + ? StateError('codec rejected bucket') : null; }; - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); - SubstrateArchive.debugBeforeFallbackDelete = null; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, hardFloorSec: cutoff); + expect(calls, 4); + expect(await (await _db()).query('substrate_archive'), hasLength(3)); + expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + isNotEmpty); + SubstrateArchive.debugBeforeBucket = null; SubstrateArchive.debugThrowInBucket = null; - expect(calls, greaterThan(1), reason: 'the later buckets still ran'); - final left = await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [ - cutoff, - ]); - expect(left, isNotEmpty, reason: 'the failed bucket stays live'); - final days = {for (final r in left) (r['rec_ts'] as int) ~/ 86400}; - expect(days, hasLength(1)); - // The rest of the prune ran: the undatable row is gone. - expect(await _live('decoded_onehz', 'rec_ts <= 0', const []), isEmpty); - // Every bucket's count was saved; only the failed one is still counted. - final row = await (await _db()).query( - 'compute_freshness', - where: 'key = ?', - whereArgs: [SubstrateArchive.failuresKey], - ); - final counts = jsonDecode(row.single['payload_json'] as String) as Map; - expect(counts.values.toSet(), {SubstrateArchive.maxFailedPasses}); - expect(counts.keys.every((k) => (k as String).endsWith('|${days.single}')), - isTrue); - // The next pass finishes the job. - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, hardFloorSec: cutoff); + expect(await (await _db()).query('substrate_archive'), hasLength(4)); expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); }); - test('a bucket that keeps failing past the hold is deleted after ' - '${SubstrateArchive.maxFailedPasses} passes', () async { + // WHY: repeated resource failures must never become permission to lose input. + // CLAIM(archive-before-delete); catches: deletion after three failed passes. + test('repeated failures past the hold retain every row until archival succeeds', () async { await _seed(); - final before = await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [ - cutoff, - ]); + final before = await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]); + final beats = await _live('decoded_rr', 'rec_ts > 0 AND rec_ts < ?', [cutoff]); SubstrateArchive.debugThrowInBucket = Exception('out of memory'); - for (var pass = 1; pass < SubstrateArchive.maxFailedPasses; pass++) { - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); - expect( - await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), - hasLength(before.length), - reason: 'pass $pass keeps it live', - ); + for (var pass = 1; pass <= 4; pass++) { + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, hardFloorSec: cutoff); + expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + before, reason: 'pass $pass'); + expect(await _live('decoded_rr', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + beats, reason: 'pass $pass'); + expect(await (await _db()).query('substrate_archive'), isEmpty); } - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); SubstrateArchive.debugThrowInBucket = null; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, hardFloorSec: cutoff); expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); - expect(await (await _db()).query('substrate_archive'), isEmpty); - // The counts are cleared with the buckets they counted. - final row = await (await _db()).query( - 'compute_freshness', - where: 'key = ?', - whereArgs: [SubstrateArchive.failuresKey], - ); - expect(row.single['payload_json'], '{}'); + expect(await _live('decoded_rr', 'rec_ts < ?', [cutoff]), isEmpty); + expect(await (await _db()).query('substrate_archive'), hasLength(4)); + final counts = await (await _db()).query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]); + expect(counts.single['payload_json'], '{}'); }); + } From b7fcce21f857a20e13ef6cbb404eda3cc6f763dc Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 15:33:37 -0700 Subject: [PATCH 2/6] data: keep live substrate whenever archival fails; skip stuck buckets in the background Archive-and-delete owns the prune transaction. Remove its unarchived fallback for deterministic failures and for failures repeated past the hard floor, so seconds and beats stay live until a verified archive commits. The hardFloorSec parameter had no other use and goes with it. One bucket that always fails must not starve background archiving: after three failed passes a bounded pass skips it without using its one-bucket budget. Foreground passes still retry it every time, and success clears the count. --- lib/compute/derivation_engine.dart | 1 - lib/data/db.dart | 5 +- lib/data/substrate_archive.dart | 91 +++++++++-------------------- test/substrate_archive_db_test.dart | 19 +++--- 4 files changed, 37 insertions(+), 79 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index d8af739ec..8133baa6c 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -9106,7 +9106,6 @@ class DerivationEngine { final deleted = await LocalDb.pruneDecodedBeforeRecTs( cutoffSec, cursorName: _prunedBeforeCursor, - hardFloorSec: dataNowSec - _maxRawHoldDays * 86400, log: _log, shouldYield: offloadActive, // A background engine runs on a short, throttled budget in its own diff --git a/lib/data/db.dart b/lib/data/db.dart index dc799032e..9cc34ec1e 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -16240,8 +16240,7 @@ class LocalDb { /// through [SubstrateArchive.archiveAndDeleteBefore] first: one transaction /// per (device_id, utc_day) bucket, deleted only once its compressed copy /// has been written and read back. A bucket that fails stays live for the - /// next pass unless it lies wholly behind [hardFloorSec] (the end of the - /// bounded raw hold), where it is deleted unarchived as before. Rows with + /// next pass, whatever its age. Rows with /// `rec_ts <= 0` cannot be placed on any day and are deleted unarchived. /// [log] receives the per-bucket failures (this file has no logger). /// [shouldYield] stops the archive step for this pass (see @@ -16255,7 +16254,6 @@ class LocalDb { int cutoffSec, { String? cursorName, SubstrateArchivePolicy? archive, - int? hardFloorSec, void Function(String message)? log, bool Function()? shouldYield, int? maxArchiveBuckets, @@ -16270,7 +16268,6 @@ class LocalDb { deleted += await SubstrateArchive.archiveAndDeleteBefore( db, cutoffSec, - hardFloorSec: hardFloorSec, log: log, shouldYield: shouldYield, maxBuckets: maxArchiveBuckets, diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index e8920bb03..7d5c552b3 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -102,12 +102,12 @@ abstract final class SubstrateArchive { static const String policyKey = 'substrate_archive_policy'; /// `compute_freshness` key counting, per `device|utc_day`, the passes on - /// which a bucket past the hold failed for a non-deterministic reason. + /// which a bucket failed to archive. static const String failuresKey = 'substrate_archive_failures'; - /// Failed passes past the hold after which ANY failure counts as - /// permanent — see [archiveAndDeleteBefore]. - static const int maxFailedPasses = 3; + /// Failed passes after which a BOUNDED (background) pass stops attempting a + /// bucket — see [archiveAndDeleteBefore]. Foreground passes always retry. + static const int backgroundSkipAfterFailures = 3; static const int _pageSize = 5000; @@ -126,11 +126,6 @@ abstract final class SubstrateArchive { @visibleForTesting static Object? debugThrowInBucket; - /// Test seam: awaited inside the past-the-hold unarchived delete's - /// transaction, so a test can make that delete fail. - @visibleForTesting - static Future Function()? debugBeforeFallbackDelete; - @visibleForTesting static int get pageSizeForTest => _pageSize; @@ -358,21 +353,23 @@ abstract final class SubstrateArchive { /// the setting, see [writePolicy]) must not be followed by this prune /// writing a fresh bucket after it. /// - /// A bucket that fails STAYS LIVE and is retried on the next pass. The one - /// exception is a DETERMINISTIC failure — the codec refusing the data or - /// failing its own read-back, or an existing blob this build cannot read — - /// on a bucket entirely behind [hardFloorSec] (the end of the bounded - /// hold). That failure would repeat on every pass, so the bucket is deleted - /// unarchived exactly as before this table existed, and one bad bucket - /// cannot switch retention off. Anything else (SQLITE_BUSY, an isolate that - /// could not start, I/O) is treated as transient and retried — but a bucket - /// past the hold that has failed on [maxFailedPasses] passes (an encode that - /// runs out of memory on a huge day fails "transiently" forever) is treated - /// as permanent too. The count persists in `compute_freshness` - /// ([failuresKey]) and is cleared when the bucket archives. + /// CLAIM(archive-before-delete): a failed bucket stays live regardless of + /// age, error type or retry count [substrate_archive_db_test]. Neither a + /// corrupt encode nor repeated resource failures authorize data loss. The + /// failure counts persist in [failuresKey] and clear when a bucket archives. + /// A permanently failing bucket can therefore exceed the live retention + /// budget until archival succeeds or the user disables archiving. /// /// [maxBuckets] caps how many buckets one pass archives; the rest stay live /// for the next pass (background engines run on a short, throttled budget). + /// A bounded pass is also the BACKGROUND pass, and one bad bucket must not + /// starve it: buckets run oldest first, so a bucket that always fails would + /// spend the single attempt of every pass (up to ~3 s of write lock per full + /// day) and nothing behind it would ever archive. A bucket that has failed + /// [backgroundSkipAfterFailures] passes is therefore skipped by bounded + /// passes without using the budget. Unbounded (foreground) passes still + /// retry it every time, so a fix or a quieter moment can archive it; success + /// clears the count. /// /// The `UNION` over both tables is required: gen4 history can carry /// beat-only seconds with no 1 Hz parent row. @@ -392,7 +389,6 @@ abstract final class SubstrateArchive { static Future archiveAndDeleteBefore( Database db, int cutoffSec, { - int? hardFloorSec, void Function(String message)? log, bool Function()? shouldYield, int? maxBuckets, @@ -420,6 +416,11 @@ abstract final class SubstrateArchive { final hi = math.min((day + 1) * 86400, cutoffSec); await debugBeforeBucket?.call(); if (maxBuckets != null && attempted >= maxBuckets) break; + final failureKey = '$deviceId|$day'; + if (maxBuckets != null && + (failures[failureKey] ?? 0) >= backgroundSkipAfterFailures) { + continue; + } attempted++; if (shouldYield?.call() ?? false) { log?.call( @@ -428,7 +429,6 @@ abstract final class SubstrateArchive { ); break; } - final failureKey = '$deviceId|$day'; try { deleted += await db.transaction((txn) async { final n = await _archiveBucket(txn, deviceId, day, lo, hi); @@ -437,45 +437,12 @@ abstract final class SubstrateArchive { }); if (failures.remove(failureKey) != null) failuresChanged = true; } catch (e) { - final pastHold = hardFloorSec != null && hi <= hardFloorSec; - var permanent = - e is StateError || e is ArgumentError || e is FormatException; - if (pastHold && !permanent) { - final n = (failures[failureKey] ?? 0) + 1; - failures[failureKey] = n; - failuresChanged = true; - permanent = n >= maxFailedPasses; - } - if (permanent && pastHold) { - // Guarded: one bucket whose delete fails (a concurrent writer - // holding the lock) must not end the pass for every other - // bucket. It stays live and keeps its count, so the next pass - // tries again. - try { - deleted += await db.transaction((txn) async { - await debugBeforeFallbackDelete?.call(); - final n = await _deleteLive(txn, deviceId, lo, hi); - await onDeleted?.call(txn, hi); - return n; - }); - if (failures.remove(failureKey) != null) failuresChanged = true; - log?.call( - 'substrate archive failed for $deviceId/$day ($e); past the ' - 'hold, deleted unarchived', - ); - } catch (e2) { - log?.call( - 'substrate archive failed for $deviceId/$day ($e) and the ' - 'unarchived delete failed too ($e2); kept live for the next ' - 'pass', - ); - } - } else { - log?.call( - 'substrate archive failed for $deviceId/$day ($e); kept live for ' - 'the next pass', - ); - } + failures[failureKey] = (failures[failureKey] ?? 0) + 1; + failuresChanged = true; + log?.call( + 'substrate archive failed for $deviceId/$day ($e); kept live for ' + 'the next pass', + ); } } } finally { diff --git a/test/substrate_archive_db_test.dart b/test/substrate_archive_db_test.dart index db4a24e86..83cd4600a 100644 --- a/test/substrate_archive_db_test.dart +++ b/test/substrate_archive_db_test.dart @@ -398,7 +398,7 @@ void main() { // WHY: the archive is the only durable copy once live rows are deleted. // CLAIM(archive-before-delete); catches: hard-floor fallback deletion. - test('verify failure keeps rows live even past hardFloorSec', () async { + test('verify failure keeps rows live however often the prune runs', () async { await _seed(); final liveBefore = await _live( 'decoded_onehz', @@ -410,7 +410,6 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: _d0, // every bucket is still inside the hold log: logs.add, ); expect( @@ -425,7 +424,6 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: cutoff, ); expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), liveBefore); @@ -1173,7 +1171,6 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: cutoff, log: logs.add, shouldYield: () => buckets > 1, // the offload begins after bucket 1 ); @@ -1192,7 +1189,6 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: cutoff, ); expect(await db.query('substrate_archive'), hasLength(4)); final (archived, _) = await _archived(); @@ -1444,7 +1440,6 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: cutoff, // every bucket is past the hold ); SubstrateArchive.debugThrowInBucket = null; expect( @@ -1487,7 +1482,7 @@ void main() { Future pass() async { seen = 0; await LocalDb.pruneDecodedBeforeRecTs(cutoff, - archive: policy, hardFloorSec: cutoff, maxArchiveBuckets: 1); + archive: policy, maxArchiveBuckets: 1); } for (var n = 1; n <= 3; n++) { @@ -1510,7 +1505,7 @@ void main() { SubstrateArchive.debugThrowInBucket = StateError('codec rejected bucket'); for (var n = 0; n < 3; n++) { await LocalDb.pruneDecodedBeforeRecTs(cutoff, - archive: policy, hardFloorSec: cutoff, maxArchiveBuckets: 1); + archive: policy, maxArchiveBuckets: 1); } SubstrateArchive.debugThrowInBucket = null; await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy); @@ -1608,7 +1603,7 @@ void main() { ? StateError('codec rejected bucket') : null; }; await LocalDb.pruneDecodedBeforeRecTs(cutoff, - archive: policy, hardFloorSec: cutoff); + archive: policy); expect(calls, 4); expect(await (await _db()).query('substrate_archive'), hasLength(3)); expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), @@ -1616,7 +1611,7 @@ void main() { SubstrateArchive.debugBeforeBucket = null; SubstrateArchive.debugThrowInBucket = null; await LocalDb.pruneDecodedBeforeRecTs(cutoff, - archive: policy, hardFloorSec: cutoff); + archive: policy); expect(await (await _db()).query('substrate_archive'), hasLength(4)); expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); }); @@ -1630,7 +1625,7 @@ void main() { SubstrateArchive.debugThrowInBucket = Exception('out of memory'); for (var pass = 1; pass <= 4; pass++) { await LocalDb.pruneDecodedBeforeRecTs(cutoff, - archive: policy, hardFloorSec: cutoff); + archive: policy); expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), before, reason: 'pass $pass'); expect(await _live('decoded_rr', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), @@ -1639,7 +1634,7 @@ void main() { } SubstrateArchive.debugThrowInBucket = null; await LocalDb.pruneDecodedBeforeRecTs(cutoff, - archive: policy, hardFloorSec: cutoff); + archive: policy); expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); expect(await _live('decoded_rr', 'rec_ts < ?', [cutoff]), isEmpty); expect(await (await _db()).query('substrate_archive'), hasLength(4)); From 94084191227e2045f2fed38f34144d8c25cfc3c7 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Tue, 17 Nov 2026 18:33:54 -0700 Subject: [PATCH 3/6] data: clear archive failure counts when archiving is disabled --- lib/data/substrate_archive.dart | 7 +++++- test/substrate_archive_db_test.dart | 35 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index 7d5c552b3..e7a9355ff 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -270,7 +270,12 @@ abstract final class SubstrateArchive { 'payload_json': policy.toJson(), 'updated_at': DateTime.now().millisecondsSinceEpoch, }, conflictAlgorithm: ConflictAlgorithm.replace); - if (!policy.enabled) await txn.delete(table); + if (!policy.enabled) { + await txn.delete(table); + // A later flash backfill must get a fresh background attempt. + await txn.delete('compute_freshness', + where: 'key = ?', whereArgs: [failuresKey]); + } }); /// Restore/salvage onto a database with NO archive of its own: bring the diff --git a/test/substrate_archive_db_test.dart b/test/substrate_archive_db_test.dart index 83cd4600a..fd5f56763 100644 --- a/test/substrate_archive_db_test.dart +++ b/test/substrate_archive_db_test.dart @@ -4,6 +4,7 @@ // export day, restore/salvage, schema health, the prune guard). import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:math' as math; import 'dart:typed_data'; @@ -1497,6 +1498,40 @@ void main() { isNotEmpty, reason: 'the stuck bucket itself is still live'); }); + // WHY: after switching off, a flash backfill of the same old bucket must + // get a background attempt when archiving is enabled again. + // CLAIM(archive-before-delete); catches: stale counts surviving policy Off. + test('switching archiving off clears failures before it is enabled again', + () async { + await _seed(); + final db = await _db(); + SubstrateArchive.debugThrowInBucket = StateError('codec rejected bucket'); + for (var n = 0; n < 3; n++) { + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, maxArchiveBuckets: 1); + } + final counts = await db.query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]); + expect((jsonDecode(counts.single['payload_json'] as String) as Map).values, + [3], reason: 'the oldest bucket is parked before switching off'); + SubstrateArchive.debugThrowInBucket = null; + await SubstrateArchive.writePolicy(db, const SubstrateArchivePolicy(0)); + expect(await db.query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]), isEmpty); + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: const SubstrateArchivePolicy(0)); + expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); + // A later flash backfill re-delivers a second in that same UTC bucket. + await _rewriteLive(_d0 + 1); + await SubstrateArchive.writePolicy(db, policy); + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, maxArchiveBuckets: 1); + final archived = await db.query('substrate_archive'); + expect(archived.single['utc_day'], _d0 ~/ _day); + expect(archived.single['onehz_rows'], 1); + expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); + }); + // WHY: skipping is a background-only economy; a foreground pass has no // bucket cap, so it must retry the stuck bucket and clear its count. // CLAIM(archive-before-delete); catches: a bucket skipped forever. From be3106a6354bc90b7f793226662b140a4657967a Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:42:12 +0530 Subject: [PATCH 4/6] archive: stop foreground retries of codec-refused buckets until the next build, show failed buckets in issue reports --- lib/data/substrate_archive.dart | 49 +++++++++++++++++++++++------ lib/support/issue_report.dart | 2 ++ test/substrate_archive_db_test.dart | 32 +++++++++++++++++-- 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index e7a9355ff..f6e7e1d91 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -32,6 +32,8 @@ import 'dart:typed_data'; import 'package:flutter/foundation.dart' show visibleForTesting; import 'package:sqflite/sqflite.dart'; +import '../compute/derivation_engine.dart' show kAlgoVersion; + import 'substrate_archive_codec.dart'; /// How long archived substrate is kept. Stored in `compute_freshness` so a @@ -106,9 +108,19 @@ abstract final class SubstrateArchive { static const String failuresKey = 'substrate_archive_failures'; /// Failed passes after which a BOUNDED (background) pass stops attempting a - /// bucket — see [archiveAndDeleteBefore]. Foreground passes always retry. + /// bucket — see [archiveAndDeleteBefore]. Foreground passes keep retrying + /// unless the codec refused it, see [foregroundSkipAfterRefusals]. static const int backgroundSkipAfterFailures = 3; + /// Codec refusals ([StateError], [FormatException], [ArgumentError]) after + /// which a FOREGROUND pass stops retrying a bucket too. The same build + /// fails the same bytes the same way, so each retry only buys ~3 s of write + /// lock against the band's commit. Counted per [kAlgoVersion]: a new build + /// gets fresh attempts. + static const int foregroundSkipAfterRefusals = 3; + + static String get _refusalPrefix => 'refused$kAlgoVersion:'; + static const int _pageSize = 5000; /// Test seam: corrupts the next encodes INSIDE the verify step, so the @@ -373,8 +385,9 @@ abstract final class SubstrateArchive { /// day) and nothing behind it would ever archive. A bucket that has failed /// [backgroundSkipAfterFailures] passes is therefore skipped by bounded /// passes without using the budget. Unbounded (foreground) passes still - /// retry it every time, so a fix or a quieter moment can archive it; success - /// clears the count. + /// retry it, so a quieter moment can archive it, except after + /// [foregroundSkipAfterRefusals] codec refusals under this build; success + /// clears both counts. /// /// The `UNION` over both tables is required: gen4 history can carry /// beat-only seconds with no 1 Hz parent row. @@ -422,8 +435,10 @@ abstract final class SubstrateArchive { await debugBeforeBucket?.call(); if (maxBuckets != null && attempted >= maxBuckets) break; final failureKey = '$deviceId|$day'; - if (maxBuckets != null && - (failures[failureKey] ?? 0) >= backgroundSkipAfterFailures) { + final refusalKey = '$_refusalPrefix$failureKey'; + if (maxBuckets != null + ? (failures[failureKey] ?? 0) >= backgroundSkipAfterFailures + : (failures[refusalKey] ?? 0) >= foregroundSkipAfterRefusals) { continue; } attempted++; @@ -441,8 +456,12 @@ abstract final class SubstrateArchive { return n; }); if (failures.remove(failureKey) != null) failuresChanged = true; + if (failures.remove(refusalKey) != null) failuresChanged = true; } catch (e) { failures[failureKey] = (failures[failureKey] ?? 0) + 1; + if (e is StateError || e is FormatException || e is ArgumentError) { + failures[refusalKey] = (failures[refusalKey] ?? 0) + 1; + } failuresChanged = true; log?.call( 'substrate archive failed for $deviceId/$day ($e); kept live for ' @@ -475,7 +494,11 @@ abstract final class SubstrateArchive { final m = jsonDecode(rows.first['payload_json'] as String) as Map; return { for (final e in m.entries) - if (e.value is int) '${e.key}': e.value as int, + // Refusals counted by an older build are dropped. + if (e.value is int && + (!'${e.key}'.startsWith('refused') || + '${e.key}'.startsWith(_refusalPrefix))) + '${e.key}': e.value as int, }; } catch (_) { return {}; @@ -1056,8 +1079,10 @@ abstract final class SubstrateArchive { ); } - /// `{buckets, bytes, raw_bytes, oldest_from_ts, newest_to_ts}`; the two - /// timestamps are null on an empty archive. + /// `{buckets, bytes, raw_bytes, oldest_from_ts, newest_to_ts, + /// failed_buckets}`; the two timestamps are null on an empty archive. + /// `failed_buckets` counts buckets whose last archive attempt failed and + /// which therefore stay live. static Future> stats(DatabaseExecutor ex) async { final r = (await ex.rawQuery( 'SELECT COUNT(*) AS buckets, ' @@ -1066,6 +1091,12 @@ abstract final class SubstrateArchive { 'MIN(from_ts) AS oldest_from_ts, MAX(to_ts) AS newest_to_ts ' 'FROM $table', )).first; - return Map.from(r); + return { + ...r, + 'failed_buckets': (await _readFailures(ex)) + .keys + .where((k) => !k.startsWith('refused')) + .length, + }; } } diff --git a/lib/support/issue_report.dart b/lib/support/issue_report.dart index 2de999227..2e788deb0 100644 --- a/lib/support/issue_report.dart +++ b/lib/support/issue_report.dart @@ -334,6 +334,8 @@ Future<({File zip, Map meta})> collectReport( 'initError': app.initError, 'importRollupError': app.importRollupError, 'lastBackupError': app.lastBackupError, + 'archiveFailedBuckets': + (await LocalDb.substrateArchiveStats())['failed_buckets'], 'recentErrors': [ for (final r in records) if (r['kind'] == 'error' || r['kind'] == 'crash') r diff --git a/test/substrate_archive_db_test.dart b/test/substrate_archive_db_test.dart index fd5f56763..1b215bee8 100644 --- a/test/substrate_archive_db_test.dart +++ b/test/substrate_archive_db_test.dart @@ -1512,8 +1512,9 @@ void main() { } final counts = await db.query('compute_freshness', where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]); + // One failure count and one refusal count, both for the oldest bucket. expect((jsonDecode(counts.single['payload_json'] as String) as Map).values, - [3], reason: 'the oldest bucket is parked before switching off'); + [3, 3], reason: 'the oldest bucket is parked before switching off'); SubstrateArchive.debugThrowInBucket = null; await SubstrateArchive.writePolicy(db, const SubstrateArchivePolicy(0)); expect(await db.query('compute_freshness', @@ -1537,7 +1538,7 @@ void main() { // CLAIM(archive-before-delete); catches: a bucket skipped forever. test('a foreground pass retries a bucket the background skips', () async { await _seed(); - SubstrateArchive.debugThrowInBucket = StateError('codec rejected bucket'); + SubstrateArchive.debugThrowInBucket = Exception('database is locked'); for (var n = 0; n < 3; n++) { await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy, maxArchiveBuckets: 1); @@ -1551,6 +1552,33 @@ void main() { expect(counts.single['payload_json'], '{}'); }); + // WHY: a codec refusal fails the same way every time under one build, and + // each foreground retry holds the write lock the band's commit waits on. + // CLAIM(archive-before-delete); catches: unbounded foreground retries. + test('a foreground pass stops retrying a bucket the codec refused', () async { + await _seed(); + var calls = 0; + SubstrateArchive.debugBeforeBucket = () async => calls++; + SubstrateArchive.debugThrowInBucket = StateError('codec rejected bucket'); + final cap = SubstrateArchive.foregroundSkipAfterRefusals; + for (var n = 0; n < cap; n++) { + await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy); + } + expect((await LocalDb.substrateArchiveStats())['failed_buckets'], 4); + calls = 0; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy); + SubstrateArchive.debugBeforeBucket = null; + SubstrateArchive.debugThrowInBucket = null; + // Every bucket was visited but none attempted, and all stay live. + expect(calls, 4); + final counts = await (await _db()).query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]); + final m = jsonDecode(counts.single['payload_json'] as String) as Map; + expect(m.values.where((v) => v == cap), hasLength(8)); + expect(await (await _db()).query('substrate_archive'), isEmpty); + expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isNotEmpty); + }); + test( 'readBlob returns a blob larger than one chunk, byte for byte', () async { From 33a967803e5a907f21dd3c1d3487d7a1348c33fa Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:55:38 +0530 Subject: [PATCH 5/6] archive: drop failure counts for buckets with no live rows, never let archive stats break a report --- lib/data/substrate_archive.dart | 21 +++++++++++++++------ lib/support/issue_report.dart | 9 +++++++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index f6e7e1d91..25fb75f52 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -119,7 +119,8 @@ abstract final class SubstrateArchive { /// gets fresh attempts. static const int foregroundSkipAfterRefusals = 3; - static String get _refusalPrefix => 'refused$kAlgoVersion:'; + static const String _refused = '|refused'; + static String get _refusalSuffix => '$_refused$kAlgoVersion'; static const int _pageSize = 5000; @@ -423,7 +424,15 @@ abstract final class SubstrateArchive { ); var deleted = 0, attempted = 0; final failures = await _readFailures(db); - var failuresChanged = false; + // A count whose bucket has no live rows left below the cutoff (deleted, + // or archived by a pass that died before saving) is dropped, so it + // neither inflates [stats] nor parks rows that come back later. + final live = { + for (final b in buckets) '${b['device_id']}|${(b['d'] as num).toInt()}', + }; + final counted = failures.length; + failures.removeWhere((k, _) => !live.contains(k.split(_refused).first)); + var failuresChanged = failures.length != counted; // The counts are saved however the loop ends, so a pass that stops // early never loses the increments it already made. try { @@ -435,7 +444,7 @@ abstract final class SubstrateArchive { await debugBeforeBucket?.call(); if (maxBuckets != null && attempted >= maxBuckets) break; final failureKey = '$deviceId|$day'; - final refusalKey = '$_refusalPrefix$failureKey'; + final refusalKey = '$failureKey$_refusalSuffix'; if (maxBuckets != null ? (failures[failureKey] ?? 0) >= backgroundSkipAfterFailures : (failures[refusalKey] ?? 0) >= foregroundSkipAfterRefusals) { @@ -496,8 +505,8 @@ abstract final class SubstrateArchive { for (final e in m.entries) // Refusals counted by an older build are dropped. if (e.value is int && - (!'${e.key}'.startsWith('refused') || - '${e.key}'.startsWith(_refusalPrefix))) + (!'${e.key}'.contains(_refused) || + '${e.key}'.endsWith(_refusalSuffix))) '${e.key}': e.value as int, }; } catch (_) { @@ -1095,7 +1104,7 @@ abstract final class SubstrateArchive { ...r, 'failed_buckets': (await _readFailures(ex)) .keys - .where((k) => !k.startsWith('refused')) + .where((k) => !k.contains(_refused)) .length, }; } diff --git a/lib/support/issue_report.dart b/lib/support/issue_report.dart index 2e788deb0..07b33ab86 100644 --- a/lib/support/issue_report.dart +++ b/lib/support/issue_report.dart @@ -315,6 +315,12 @@ Future<({File zip, Map meta})> collectReport( for (final r in t.outboxSnapshot) if ((r['ts'] as int? ?? 0) >= weekAgo) r ]; + // Optional diagnostic: a database error must not cost the whole report. + Object? archiveFailedBuckets; + try { + archiveFailedBuckets = + (await LocalDb.substrateArchiveStats())['failed_buckets']; + } catch (_) {} final state = { 'syncStatus': app.status, 'syncingNow': app.syncingNow, @@ -334,8 +340,7 @@ Future<({File zip, Map meta})> collectReport( 'initError': app.initError, 'importRollupError': app.importRollupError, 'lastBackupError': app.lastBackupError, - 'archiveFailedBuckets': - (await LocalDb.substrateArchiveStats())['failed_buckets'], + 'archiveFailedBuckets': archiveFailedBuckets, 'recentErrors': [ for (final r in records) if (r['kind'] == 'error' || r['kind'] == 'crash') r From e47e09164bb5f517218d94ab38cf641c402203ac Mon Sep 17 00:00:00 2001 From: Mohammad Abdul Sahil <127765312+abdulsaheel@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:04:50 +0530 Subject: [PATCH 6/6] archive: don't write failure counts back after archiving was switched off --- lib/data/substrate_archive.dart | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index 25fb75f52..51dbda4f2 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -479,12 +479,17 @@ abstract final class SubstrateArchive { } } } finally { + // Checked in the write's own transaction: archiving switched off + // mid-pass cleared the counts, and this pass must not bring them back. if (failuresChanged) { - await db.insert('compute_freshness', { - 'key': failuresKey, - 'payload_json': jsonEncode(failures), - 'updated_at': DateTime.now().millisecondsSinceEpoch, - }, conflictAlgorithm: ConflictAlgorithm.replace); + await db.transaction((txn) async { + if (!(await readPolicy(txn)).enabled) return; + await txn.insert('compute_freshness', { + 'key': failuresKey, + 'payload_json': jsonEncode(failures), + 'updated_at': DateTime.now().millisecondsSinceEpoch, + }, conflictAlgorithm: ConflictAlgorithm.replace); + }); } } return deleted;