diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index d8af739ec..8133baa6c 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -9106,7 +9106,6 @@ class DerivationEngine { final deleted = await LocalDb.pruneDecodedBeforeRecTs( cutoffSec, cursorName: _prunedBeforeCursor, - hardFloorSec: dataNowSec - _maxRawHoldDays * 86400, log: _log, shouldYield: offloadActive, // A background engine runs on a short, throttled budget in its own diff --git a/lib/data/db.dart b/lib/data/db.dart index dc799032e..9cc34ec1e 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -16240,8 +16240,7 @@ class LocalDb { /// through [SubstrateArchive.archiveAndDeleteBefore] first: one transaction /// per (device_id, utc_day) bucket, deleted only once its compressed copy /// has been written and read back. A bucket that fails stays live for the - /// next pass unless it lies wholly behind [hardFloorSec] (the end of the - /// bounded raw hold), where it is deleted unarchived as before. Rows with + /// next pass, whatever its age. Rows with /// `rec_ts <= 0` cannot be placed on any day and are deleted unarchived. /// [log] receives the per-bucket failures (this file has no logger). /// [shouldYield] stops the archive step for this pass (see @@ -16255,7 +16254,6 @@ class LocalDb { int cutoffSec, { String? cursorName, SubstrateArchivePolicy? archive, - int? hardFloorSec, void Function(String message)? log, bool Function()? shouldYield, int? maxArchiveBuckets, @@ -16270,7 +16268,6 @@ class LocalDb { deleted += await SubstrateArchive.archiveAndDeleteBefore( db, cutoffSec, - hardFloorSec: hardFloorSec, log: log, shouldYield: shouldYield, maxBuckets: maxArchiveBuckets, diff --git a/lib/data/substrate_archive.dart b/lib/data/substrate_archive.dart index e8920bb03..51dbda4f2 100644 --- a/lib/data/substrate_archive.dart +++ b/lib/data/substrate_archive.dart @@ -32,6 +32,8 @@ import 'dart:typed_data'; import 'package:flutter/foundation.dart' show visibleForTesting; import 'package:sqflite/sqflite.dart'; +import '../compute/derivation_engine.dart' show kAlgoVersion; + import 'substrate_archive_codec.dart'; /// How long archived substrate is kept. Stored in `compute_freshness` so a @@ -102,12 +104,23 @@ abstract final class SubstrateArchive { static const String policyKey = 'substrate_archive_policy'; /// `compute_freshness` key counting, per `device|utc_day`, the passes on - /// which a bucket past the hold failed for a non-deterministic reason. + /// which a bucket failed to archive. static const String failuresKey = 'substrate_archive_failures'; - /// Failed passes past the hold after which ANY failure counts as - /// permanent — see [archiveAndDeleteBefore]. - static const int maxFailedPasses = 3; + /// Failed passes after which a BOUNDED (background) pass stops attempting a + /// bucket — see [archiveAndDeleteBefore]. Foreground passes keep retrying + /// unless the codec refused it, see [foregroundSkipAfterRefusals]. + static const int backgroundSkipAfterFailures = 3; + + /// Codec refusals ([StateError], [FormatException], [ArgumentError]) after + /// which a FOREGROUND pass stops retrying a bucket too. The same build + /// fails the same bytes the same way, so each retry only buys ~3 s of write + /// lock against the band's commit. Counted per [kAlgoVersion]: a new build + /// gets fresh attempts. + static const int foregroundSkipAfterRefusals = 3; + + static const String _refused = '|refused'; + static String get _refusalSuffix => '$_refused$kAlgoVersion'; static const int _pageSize = 5000; @@ -126,11 +139,6 @@ abstract final class SubstrateArchive { @visibleForTesting static Object? debugThrowInBucket; - /// Test seam: awaited inside the past-the-hold unarchived delete's - /// transaction, so a test can make that delete fail. - @visibleForTesting - static Future Function()? debugBeforeFallbackDelete; - @visibleForTesting static int get pageSizeForTest => _pageSize; @@ -275,7 +283,12 @@ abstract final class SubstrateArchive { 'payload_json': policy.toJson(), 'updated_at': DateTime.now().millisecondsSinceEpoch, }, conflictAlgorithm: ConflictAlgorithm.replace); - if (!policy.enabled) await txn.delete(table); + if (!policy.enabled) { + await txn.delete(table); + // A later flash backfill must get a fresh background attempt. + await txn.delete('compute_freshness', + where: 'key = ?', whereArgs: [failuresKey]); + } }); /// Restore/salvage onto a database with NO archive of its own: bring the @@ -358,21 +371,24 @@ abstract final class SubstrateArchive { /// the setting, see [writePolicy]) must not be followed by this prune /// writing a fresh bucket after it. /// - /// A bucket that fails STAYS LIVE and is retried on the next pass. The one - /// exception is a DETERMINISTIC failure — the codec refusing the data or - /// failing its own read-back, or an existing blob this build cannot read — - /// on a bucket entirely behind [hardFloorSec] (the end of the bounded - /// hold). That failure would repeat on every pass, so the bucket is deleted - /// unarchived exactly as before this table existed, and one bad bucket - /// cannot switch retention off. Anything else (SQLITE_BUSY, an isolate that - /// could not start, I/O) is treated as transient and retried — but a bucket - /// past the hold that has failed on [maxFailedPasses] passes (an encode that - /// runs out of memory on a huge day fails "transiently" forever) is treated - /// as permanent too. The count persists in `compute_freshness` - /// ([failuresKey]) and is cleared when the bucket archives. + /// CLAIM(archive-before-delete): a failed bucket stays live regardless of + /// age, error type or retry count [substrate_archive_db_test]. Neither a + /// corrupt encode nor repeated resource failures authorize data loss. The + /// failure counts persist in [failuresKey] and clear when a bucket archives. + /// A permanently failing bucket can therefore exceed the live retention + /// budget until archival succeeds or the user disables archiving. /// /// [maxBuckets] caps how many buckets one pass archives; the rest stay live /// for the next pass (background engines run on a short, throttled budget). + /// A bounded pass is also the BACKGROUND pass, and one bad bucket must not + /// starve it: buckets run oldest first, so a bucket that always fails would + /// spend the single attempt of every pass (up to ~3 s of write lock per full + /// day) and nothing behind it would ever archive. A bucket that has failed + /// [backgroundSkipAfterFailures] passes is therefore skipped by bounded + /// passes without using the budget. Unbounded (foreground) passes still + /// retry it, so a quieter moment can archive it, except after + /// [foregroundSkipAfterRefusals] codec refusals under this build; success + /// clears both counts. /// /// The `UNION` over both tables is required: gen4 history can carry /// beat-only seconds with no 1 Hz parent row. @@ -392,7 +408,6 @@ abstract final class SubstrateArchive { static Future archiveAndDeleteBefore( Database db, int cutoffSec, { - int? hardFloorSec, void Function(String message)? log, bool Function()? shouldYield, int? maxBuckets, @@ -409,7 +424,15 @@ abstract final class SubstrateArchive { ); var deleted = 0, attempted = 0; final failures = await _readFailures(db); - var failuresChanged = false; + // A count whose bucket has no live rows left below the cutoff (deleted, + // or archived by a pass that died before saving) is dropped, so it + // neither inflates [stats] nor parks rows that come back later. + final live = { + for (final b in buckets) '${b['device_id']}|${(b['d'] as num).toInt()}', + }; + final counted = failures.length; + failures.removeWhere((k, _) => !live.contains(k.split(_refused).first)); + var failuresChanged = failures.length != counted; // The counts are saved however the loop ends, so a pass that stops // early never loses the increments it already made. try { @@ -420,6 +443,13 @@ abstract final class SubstrateArchive { final hi = math.min((day + 1) * 86400, cutoffSec); await debugBeforeBucket?.call(); if (maxBuckets != null && attempted >= maxBuckets) break; + final failureKey = '$deviceId|$day'; + final refusalKey = '$failureKey$_refusalSuffix'; + if (maxBuckets != null + ? (failures[failureKey] ?? 0) >= backgroundSkipAfterFailures + : (failures[refusalKey] ?? 0) >= foregroundSkipAfterRefusals) { + continue; + } attempted++; if (shouldYield?.call() ?? false) { log?.call( @@ -428,7 +458,6 @@ abstract final class SubstrateArchive { ); break; } - final failureKey = '$deviceId|$day'; try { deleted += await db.transaction((txn) async { final n = await _archiveBucket(txn, deviceId, day, lo, hi); @@ -436,55 +465,31 @@ abstract final class SubstrateArchive { return n; }); if (failures.remove(failureKey) != null) failuresChanged = true; + if (failures.remove(refusalKey) != null) failuresChanged = true; } catch (e) { - final pastHold = hardFloorSec != null && hi <= hardFloorSec; - var permanent = - e is StateError || e is ArgumentError || e is FormatException; - if (pastHold && !permanent) { - final n = (failures[failureKey] ?? 0) + 1; - failures[failureKey] = n; - failuresChanged = true; - permanent = n >= maxFailedPasses; - } - if (permanent && pastHold) { - // Guarded: one bucket whose delete fails (a concurrent writer - // holding the lock) must not end the pass for every other - // bucket. It stays live and keeps its count, so the next pass - // tries again. - try { - deleted += await db.transaction((txn) async { - await debugBeforeFallbackDelete?.call(); - final n = await _deleteLive(txn, deviceId, lo, hi); - await onDeleted?.call(txn, hi); - return n; - }); - if (failures.remove(failureKey) != null) failuresChanged = true; - log?.call( - 'substrate archive failed for $deviceId/$day ($e); past the ' - 'hold, deleted unarchived', - ); - } catch (e2) { - log?.call( - 'substrate archive failed for $deviceId/$day ($e) and the ' - 'unarchived delete failed too ($e2); kept live for the next ' - 'pass', - ); - } - } else { - log?.call( - 'substrate archive failed for $deviceId/$day ($e); kept live for ' - 'the next pass', - ); + failures[failureKey] = (failures[failureKey] ?? 0) + 1; + if (e is StateError || e is FormatException || e is ArgumentError) { + failures[refusalKey] = (failures[refusalKey] ?? 0) + 1; } + failuresChanged = true; + log?.call( + 'substrate archive failed for $deviceId/$day ($e); kept live for ' + 'the next pass', + ); } } } finally { + // Checked in the write's own transaction: archiving switched off + // mid-pass cleared the counts, and this pass must not bring them back. if (failuresChanged) { - await db.insert('compute_freshness', { - 'key': failuresKey, - 'payload_json': jsonEncode(failures), - 'updated_at': DateTime.now().millisecondsSinceEpoch, - }, conflictAlgorithm: ConflictAlgorithm.replace); + await db.transaction((txn) async { + if (!(await readPolicy(txn)).enabled) return; + await txn.insert('compute_freshness', { + 'key': failuresKey, + 'payload_json': jsonEncode(failures), + 'updated_at': DateTime.now().millisecondsSinceEpoch, + }, conflictAlgorithm: ConflictAlgorithm.replace); + }); } } return deleted; @@ -503,7 +508,11 @@ abstract final class SubstrateArchive { final m = jsonDecode(rows.first['payload_json'] as String) as Map; return { for (final e in m.entries) - if (e.value is int) '${e.key}': e.value as int, + // Refusals counted by an older build are dropped. + if (e.value is int && + (!'${e.key}'.contains(_refused) || + '${e.key}'.endsWith(_refusalSuffix))) + '${e.key}': e.value as int, }; } catch (_) { return {}; @@ -1084,8 +1093,10 @@ abstract final class SubstrateArchive { ); } - /// `{buckets, bytes, raw_bytes, oldest_from_ts, newest_to_ts}`; the two - /// timestamps are null on an empty archive. + /// `{buckets, bytes, raw_bytes, oldest_from_ts, newest_to_ts, + /// failed_buckets}`; the two timestamps are null on an empty archive. + /// `failed_buckets` counts buckets whose last archive attempt failed and + /// which therefore stay live. static Future> stats(DatabaseExecutor ex) async { final r = (await ex.rawQuery( 'SELECT COUNT(*) AS buckets, ' @@ -1094,6 +1105,12 @@ abstract final class SubstrateArchive { 'MIN(from_ts) AS oldest_from_ts, MAX(to_ts) AS newest_to_ts ' 'FROM $table', )).first; - return Map.from(r); + return { + ...r, + 'failed_buckets': (await _readFailures(ex)) + .keys + .where((k) => !k.contains(_refused)) + .length, + }; } } diff --git a/lib/support/issue_report.dart b/lib/support/issue_report.dart index 2de999227..07b33ab86 100644 --- a/lib/support/issue_report.dart +++ b/lib/support/issue_report.dart @@ -315,6 +315,12 @@ Future<({File zip, Map meta})> collectReport( for (final r in t.outboxSnapshot) if ((r['ts'] as int? ?? 0) >= weekAgo) r ]; + // Optional diagnostic: a database error must not cost the whole report. + Object? archiveFailedBuckets; + try { + archiveFailedBuckets = + (await LocalDb.substrateArchiveStats())['failed_buckets']; + } catch (_) {} final state = { 'syncStatus': app.status, 'syncingNow': app.syncingNow, @@ -334,6 +340,7 @@ Future<({File zip, Map meta})> collectReport( 'initError': app.initError, 'importRollupError': app.importRollupError, 'lastBackupError': app.lastBackupError, + 'archiveFailedBuckets': archiveFailedBuckets, 'recentErrors': [ for (final r in records) if (r['kind'] == 'error' || r['kind'] == 'crash') r diff --git a/test/substrate_archive_db_test.dart b/test/substrate_archive_db_test.dart index 0fc3b18fc..1b215bee8 100644 --- a/test/substrate_archive_db_test.dart +++ b/test/substrate_archive_db_test.dart @@ -101,7 +101,6 @@ Future _clear() async { SubstrateArchive.debugCorruptEncode = false; SubstrateArchive.debugBeforeBucket = null; SubstrateArchive.debugThrowInBucket = null; - SubstrateArchive.debugBeforeFallbackDelete = null; LocalDb.debugFailExportStrip = false; } @@ -398,8 +397,9 @@ void main() { expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); }); - test('verify failure keeps rows live inside the hold window and deletes ' - 'past hardFloorSec', () async { + // WHY: the archive is the only durable copy once live rows are deleted. + // CLAIM(archive-before-delete); catches: hard-floor fallback deletion. + test('verify failure keeps rows live however often the prune runs', () async { await _seed(); final liveBefore = await _live( 'decoded_onehz', @@ -411,7 +411,6 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: _d0, // every bucket is still inside the hold log: logs.add, ); expect( @@ -422,14 +421,15 @@ void main() { expect(logs, hasLength(4)); expect(logs.first, contains('kept live')); - // Past the bounded hold the old behaviour applies: deleted unarchived. + // Age never authorizes deleting rows whose archive failed verification. await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: cutoff, ); - expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); - expect(await _live('decoded_rr', 'rec_ts < ?', [cutoff]), isEmpty); + expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + liveBefore); + expect(await _live('decoded_rr', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + isNotEmpty); expect(await (await _db()).query('substrate_archive'), isEmpty); }); @@ -1172,7 +1172,6 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: cutoff, log: logs.add, shouldYield: () => buckets > 1, // the offload begins after bucket 1 ); @@ -1191,7 +1190,6 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: cutoff, ); expect(await db.query('substrate_archive'), hasLength(4)); final (archived, _) = await _archived(); @@ -1443,7 +1441,6 @@ void main() { await LocalDb.pruneDecodedBeforeRecTs( cutoff, archive: policy, - hardFloorSec: cutoff, // every bucket is past the hold ); SubstrateArchive.debugThrowInBucket = null; expect( @@ -1469,6 +1466,119 @@ void main() { ); }); + // WHY: background passes archive ONE bucket (maxArchiveBuckets: 1), oldest + // first. A bucket that can never archive must not spend that one attempt on + // every pass, or background archiving would stall behind it for good. + // CLAIM(archive-before-delete); catches: a stuck oldest bucket starving the rest. + test('a bucket that keeps failing stops consuming the background budget', + () async { + await _seed(); + // Fails only the FIRST (oldest) bucket of every pass. The hook runs once + // per bucket, in order, before the pass decides whether to attempt it. + var seen = 0; + SubstrateArchive.debugBeforeBucket = () async { + SubstrateArchive.debugThrowInBucket = + seen++ == 0 ? StateError('codec rejected bucket') : null; + }; + Future pass() async { + seen = 0; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, maxArchiveBuckets: 1); + } + + for (var n = 1; n <= 3; n++) { + await pass(); + expect(await (await _db()).query('substrate_archive'), isEmpty, + reason: 'pass $n spends its one attempt on the failing bucket'); + } + await pass(); + expect(await (await _db()).query('substrate_archive'), hasLength(1), + reason: 'after three failures the next bucket gets the attempt'); + expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + isNotEmpty, reason: 'the stuck bucket itself is still live'); + }); + + // WHY: after switching off, a flash backfill of the same old bucket must + // get a background attempt when archiving is enabled again. + // CLAIM(archive-before-delete); catches: stale counts surviving policy Off. + test('switching archiving off clears failures before it is enabled again', + () async { + await _seed(); + final db = await _db(); + SubstrateArchive.debugThrowInBucket = StateError('codec rejected bucket'); + for (var n = 0; n < 3; n++) { + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, maxArchiveBuckets: 1); + } + final counts = await db.query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]); + // One failure count and one refusal count, both for the oldest bucket. + expect((jsonDecode(counts.single['payload_json'] as String) as Map).values, + [3, 3], reason: 'the oldest bucket is parked before switching off'); + SubstrateArchive.debugThrowInBucket = null; + await SubstrateArchive.writePolicy(db, const SubstrateArchivePolicy(0)); + expect(await db.query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]), isEmpty); + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: const SubstrateArchivePolicy(0)); + expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); + // A later flash backfill re-delivers a second in that same UTC bucket. + await _rewriteLive(_d0 + 1); + await SubstrateArchive.writePolicy(db, policy); + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, maxArchiveBuckets: 1); + final archived = await db.query('substrate_archive'); + expect(archived.single['utc_day'], _d0 ~/ _day); + expect(archived.single['onehz_rows'], 1); + expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); + }); + + // WHY: skipping is a background-only economy; a foreground pass has no + // bucket cap, so it must retry the stuck bucket and clear its count. + // CLAIM(archive-before-delete); catches: a bucket skipped forever. + test('a foreground pass retries a bucket the background skips', () async { + await _seed(); + SubstrateArchive.debugThrowInBucket = Exception('database is locked'); + for (var n = 0; n < 3; n++) { + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy, maxArchiveBuckets: 1); + } + SubstrateArchive.debugThrowInBucket = null; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy); + expect(await (await _db()).query('substrate_archive'), hasLength(4)); + expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); + final counts = await (await _db()).query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]); + expect(counts.single['payload_json'], '{}'); + }); + + // WHY: a codec refusal fails the same way every time under one build, and + // each foreground retry holds the write lock the band's commit waits on. + // CLAIM(archive-before-delete); catches: unbounded foreground retries. + test('a foreground pass stops retrying a bucket the codec refused', () async { + await _seed(); + var calls = 0; + SubstrateArchive.debugBeforeBucket = () async => calls++; + SubstrateArchive.debugThrowInBucket = StateError('codec rejected bucket'); + final cap = SubstrateArchive.foregroundSkipAfterRefusals; + for (var n = 0; n < cap; n++) { + await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy); + } + expect((await LocalDb.substrateArchiveStats())['failed_buckets'], 4); + calls = 0; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, archive: policy); + SubstrateArchive.debugBeforeBucket = null; + SubstrateArchive.debugThrowInBucket = null; + // Every bucket was visited but none attempted, and all stay live. + expect(calls, 4); + final counts = await (await _db()).query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]); + final m = jsonDecode(counts.single['payload_json'] as String) as Map; + expect(m.values.where((v) => v == cap), hasLength(8)); + expect(await (await _db()).query('substrate_archive'), isEmpty); + expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isNotEmpty); + }); + test( 'readBlob returns a blob larger than one chunk, byte for byte', () async { @@ -1546,91 +1656,54 @@ void main() { }, ); - test('a failed unarchived delete neither stops the pass nor loses the ' - 'failure counts', () async { + // WHY: a bad bucket must stay live while independent buckets still archive. + // CLAIM(archive-before-delete); catches: aborting the loop on first error. + test('a failed bucket stays live while later buckets archive', () async { await _seed(); - SubstrateArchive.debugThrowInBucket = Exception('out of memory'); - for (var pass = 1; pass < SubstrateArchive.maxFailedPasses; pass++) { - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); - } - // The pass that would delete unarchived: the FIRST bucket's delete fails - // (a concurrent writer holding the lock). var calls = 0; - SubstrateArchive.debugBeforeFallbackDelete = () async { - if (calls++ == 0) throw Exception('database is locked'); + SubstrateArchive.debugBeforeBucket = () async { + SubstrateArchive.debugThrowInBucket = calls++ == 0 + ? StateError('codec rejected bucket') : null; }; - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); - SubstrateArchive.debugBeforeFallbackDelete = null; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy); + expect(calls, 4); + expect(await (await _db()).query('substrate_archive'), hasLength(3)); + expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + isNotEmpty); + SubstrateArchive.debugBeforeBucket = null; SubstrateArchive.debugThrowInBucket = null; - expect(calls, greaterThan(1), reason: 'the later buckets still ran'); - final left = await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [ - cutoff, - ]); - expect(left, isNotEmpty, reason: 'the failed bucket stays live'); - final days = {for (final r in left) (r['rec_ts'] as int) ~/ 86400}; - expect(days, hasLength(1)); - // The rest of the prune ran: the undatable row is gone. - expect(await _live('decoded_onehz', 'rec_ts <= 0', const []), isEmpty); - // Every bucket's count was saved; only the failed one is still counted. - final row = await (await _db()).query( - 'compute_freshness', - where: 'key = ?', - whereArgs: [SubstrateArchive.failuresKey], - ); - final counts = jsonDecode(row.single['payload_json'] as String) as Map; - expect(counts.values.toSet(), {SubstrateArchive.maxFailedPasses}); - expect(counts.keys.every((k) => (k as String).endsWith('|${days.single}')), - isTrue); - // The next pass finishes the job. - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy); + expect(await (await _db()).query('substrate_archive'), hasLength(4)); expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); }); - test('a bucket that keeps failing past the hold is deleted after ' - '${SubstrateArchive.maxFailedPasses} passes', () async { + // WHY: repeated resource failures must never become permission to lose input. + // CLAIM(archive-before-delete); catches: deletion after three failed passes. + test('repeated failures past the hold retain every row until archival succeeds', () async { await _seed(); - final before = await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [ - cutoff, - ]); + final before = await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]); + final beats = await _live('decoded_rr', 'rec_ts > 0 AND rec_ts < ?', [cutoff]); SubstrateArchive.debugThrowInBucket = Exception('out of memory'); - for (var pass = 1; pass < SubstrateArchive.maxFailedPasses; pass++) { - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); - expect( - await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), - hasLength(before.length), - reason: 'pass $pass keeps it live', - ); + for (var pass = 1; pass <= 4; pass++) { + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy); + expect(await _live('decoded_onehz', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + before, reason: 'pass $pass'); + expect(await _live('decoded_rr', 'rec_ts > 0 AND rec_ts < ?', [cutoff]), + beats, reason: 'pass $pass'); + expect(await (await _db()).query('substrate_archive'), isEmpty); } - await LocalDb.pruneDecodedBeforeRecTs( - cutoff, - archive: policy, - hardFloorSec: cutoff, - ); SubstrateArchive.debugThrowInBucket = null; + await LocalDb.pruneDecodedBeforeRecTs(cutoff, + archive: policy); expect(await _live('decoded_onehz', 'rec_ts < ?', [cutoff]), isEmpty); - expect(await (await _db()).query('substrate_archive'), isEmpty); - // The counts are cleared with the buckets they counted. - final row = await (await _db()).query( - 'compute_freshness', - where: 'key = ?', - whereArgs: [SubstrateArchive.failuresKey], - ); - expect(row.single['payload_json'], '{}'); + expect(await _live('decoded_rr', 'rec_ts < ?', [cutoff]), isEmpty); + expect(await (await _db()).query('substrate_archive'), hasLength(4)); + final counts = await (await _db()).query('compute_freshness', + where: 'key = ?', whereArgs: [SubstrateArchive.failuresKey]); + expect(counts.single['payload_json'], '{}'); }); + }