From a5e51d28b8326435a1ff7d1432937b5dcf15325f Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 7 Oct 2026 06:59:13 -0700 Subject: [PATCH 1/4] fix: bound private Action recovery replay and state retention --- CHANGELOG.md | 2 + crates/warp-core/src/lib.rs | 7 + crates/warp-core/src/provenance_store.rs | 22 +- crates/warp-core/src/trusted_runtime_host.rs | 674 +++++++++++------- .../executable_operation_pipeline_tests.rs | 146 ++++ docs/topics/WAL.md | 4 + 6 files changed, 606 insertions(+), 249 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83a753c1b..bbad18904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ ### Fixed +- Action WAL recovery reuses ordered verified replay cursors instead of retaining a full state for each basis tick. It preserves basis, Tick, result, obstruction and conflict checks, including delayed stale bases. + - `xtask run-edict-operation` runs outside Git and resolves relative artifact and WAL paths from the caller directory. Repository maintenance commands retain their Git-root behavior. - `WorldlineState::state_root` documentation now states its reachable-state boundary. Detached create-if-absent writes remain bound by patch and commit identities. Hash bytes are unchanged. diff --git a/crates/warp-core/src/lib.rs b/crates/warp-core/src/lib.rs index c0cdb0f5c..84d240a50 100644 --- a/crates/warp-core/src/lib.rs +++ b/crates/warp-core/src/lib.rs @@ -486,6 +486,13 @@ pub use tick_patch::{ slice_worldline_indices, PortalInit, SlotId, TickCommitStatus, TickPatchError, WarpOp, WarpOpKey, WarpTickPatchV1, }; +#[cfg(all( + feature = "native_rule_bootstrap", + feature = "trusted_runtime", + any(test, feature = "host_test") +))] +pub use trusted_runtime_host::EchoOperationParentStateWorkForTest; + #[cfg(all(feature = "native_rule_bootstrap", feature = "trusted_runtime"))] pub use trusted_runtime_host::{ EvidenceCatalogPosture, RuntimeWalActivationGap, TrustedRuntimeApp, TrustedRuntimeHost, diff --git a/crates/warp-core/src/provenance_store.rs b/crates/warp-core/src/provenance_store.rs index 6d94cf3de..437d5d137 100644 --- a/crates/warp-core/src/provenance_store.rs +++ b/crates/warp-core/src/provenance_store.rs @@ -991,13 +991,14 @@ pub(crate) fn advance_replay_state( replayed: &mut WorldlineState, start_tick: WorldlineTick, target_tick: WorldlineTick, -) -> Result<(), ReplayError> { +) -> Result { if start_tick == target_tick { - return Ok(()); + return Ok(0); } let root = *replayed.root(); let mut last_entry = None; + let mut applied = 0; for raw_tick in start_tick.as_u64()..target_tick.as_u64() { let tick = WorldlineTick::from_raw(raw_tick); let entry = provenance.entry(worldline_id, tick)?; @@ -1044,10 +1045,11 @@ pub(crate) fn advance_replay_state( .tick_history .push((snapshot, receipt, replay_patch)); last_entry = Some(entry); + applied += 1; } finalize_replay_metadata(replayed, target_tick, last_entry.as_ref()); - Ok(()) + Ok(applied) } pub(crate) fn replay_worldline_state_at_from_provenance( @@ -1056,6 +1058,16 @@ pub(crate) fn replay_worldline_state_at_from_provenance( base_state: &WorldlineState, target_tick: WorldlineTick, ) -> Result { + replay_worldline_state_at_with_work(provenance, worldline_id, base_state, target_tick) + .map(|(state, _)| state) +} + +pub(crate) fn replay_worldline_state_at_with_work( + provenance: &P, + worldline_id: WorldlineId, + base_state: &WorldlineState, + target_tick: WorldlineTick, +) -> Result<(WorldlineState, usize), ReplayError> { let history_len = WorldlineTick::from_raw(provenance.len(worldline_id)?); if target_tick > history_len { return Err(ReplayError::History(HistoryError::HistoryUnavailable { @@ -1066,14 +1078,14 @@ pub(crate) fn replay_worldline_state_at_from_provenance( validate_replay_base(provenance, worldline_id, base_state)?; let (mut replayed, start_tick) = restore_replay_base(provenance, worldline_id, base_state, target_tick)?; - advance_replay_state( + let applied = advance_replay_state( provenance, worldline_id, &mut replayed, start_tick, target_tick, )?; - Ok(replayed) + Ok((replayed, applied)) } #[derive(Debug, Clone)] diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 3acc60ce1..ef5119923 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -633,6 +633,23 @@ impl TrustedRuntimeWalRecovery { provenance: &ProvenanceService, ) -> Result { validate_recovered_echo_operation_parent_states(runtime, provenance, self) + .map(|work| work.replay_count) + } + + /// Measures actual replay applications and peak private retained data for tests. + #[cfg(any(test, feature = "host_test"))] + pub fn echo_operation_parent_state_work_for_test( + &self, + runtime: &WorldlineRuntime, + provenance: &ProvenanceService, + ) -> Result { + validate_recovered_echo_operation_parent_states(runtime, provenance, self).map(|work| { + EchoOperationParentStateWorkForTest { + replay_count: work.replay_count, + replayed_patches: work.replayed_patches, + ..work.retained + } + }) } /// Recomputes the certificate's canonical index root from recovered evidence. @@ -4193,36 +4210,131 @@ fn operation_application_basis_matches_scope_v1( } } +/// Test-only work and retained-data counters for one Action parent-state validation. +#[cfg(any(test, feature = "host_test"))] +#[derive(Clone, Copy, Debug, Default)] +pub struct EchoOperationParentStateWorkForTest { + /// Complete prefix initializations, preserving the legacy replay-count meaning. + pub replay_count: usize, + /// Successfully applied and verified provenance replay patches. + pub replayed_patches: usize, + /// Peak private replay states, including one transient Tick simulation. + pub peak_retained_states: usize, + /// Peak retained graph node records across those states. + pub peak_retained_node_records: usize, + /// Peak retained Tick-history records across those states. + pub peak_retained_tick_records: usize, + /// Peak logical atom bytes across those states, not a process-RSS measurement. + pub peak_retained_atom_bytes: usize, +} + +#[derive(Default)] +struct ParentStateValidationWork { + replay_count: usize, + replayed_patches: usize, + #[cfg(any(test, feature = "host_test"))] + retained: EchoOperationParentStateWorkForTest, +} + +impl ParentStateValidationWork { + #[cfg(any(test, feature = "host_test"))] + fn note_states<'a>(&mut self, states: impl Iterator) { + #[cfg(any(test, feature = "host_test"))] + { + let mut state_count = 0; + let mut nodes = 0; + let mut ticks = 0; + let mut atom_bytes = 0; + for state in states { + state_count += 1; + ticks += state.tick_history().len(); + for store in state.warp_state().stores.values() { + nodes += store.nodes.len(); + for attachment in store + .node_attachments + .values() + .chain(store.edge_attachments.values()) + { + if let crate::AttachmentValue::Atom(atom) = attachment { + atom_bytes += atom.bytes.len(); + } + } + } + } + self.retained.peak_retained_states = + self.retained.peak_retained_states.max(state_count); + self.retained.peak_retained_node_records = + self.retained.peak_retained_node_records.max(nodes); + self.retained.peak_retained_tick_records = + self.retained.peak_retained_tick_records.max(ticks); + self.retained.peak_retained_atom_bytes = + self.retained.peak_retained_atom_bytes.max(atom_bytes); + } + } +} + fn recovered_worldline_state_at<'a>( - cache: &'a mut BTreeMap<(crate::WorldlineId, crate::WorldlineTick), crate::WorldlineState>, + cache: &'a mut BTreeMap, recovered_provenance: &ProvenanceService, worldline_id: crate::WorldlineId, frontier_state: &crate::WorldlineState, worldline_tick: crate::WorldlineTick, failure_detail: &'static str, + work: &mut ParentStateValidationWork, ) -> Result<&'a crate::WorldlineState, TrustedRuntimeWalError> { - let coordinate = (worldline_id, worldline_tick); - if let std::collections::btree_map::Entry::Vacant(entry) = cache.entry(coordinate) { - let state = recovered_provenance - .replay_worldline_state_at(worldline_id, frontier_state, worldline_tick) - .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { + let initialize = cache + .get(&worldline_id) + .is_none_or(|(tick, _)| *tick > worldline_tick); + if initialize { + // Drop the old cursor before constructing a replacement prefix. + drop(cache.remove(&worldline_id)); + let (state, applied) = crate::provenance_store::replay_worldline_state_at_with_work( + recovered_provenance, + worldline_id, + frontier_state, + worldline_tick, + ) + .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: failure_detail, + })?; + cache.insert(worldline_id, (worldline_tick, state)); + work.replay_count += 1; + work.replayed_patches += applied; + } else { + let (tick, state) = cache.get_mut(&worldline_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: failure_detail, - })?; - entry.insert(state); + }, + )?; + let applied = crate::provenance_store::advance_replay_state( + recovered_provenance, + worldline_id, + state, + *tick, + worldline_tick, + ) + .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: failure_detail, + })?; + *tick = worldline_tick; + work.replayed_patches += applied; } - cache - .get(&coordinate) - .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + #[cfg(any(test, feature = "host_test"))] + work.note_states(cache.values().map(|(_, state)| state)); + cache.get(&worldline_id).map(|(_, state)| state).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: failure_detail, - }) + }, + ) } fn validate_recovered_echo_operation_parent_states( runtime: &WorldlineRuntime, recovered_provenance: &ProvenanceService, recovery: &TrustedRuntimeWalRecovery, -) -> Result { +) -> Result { let mut recovered_states = BTreeMap::new(); + let mut work = ParentStateValidationWork::default(); let installations = recovery .installed_echo_operations .iter() @@ -4234,60 +4346,6 @@ fn validate_recovered_echo_operation_parent_states( .map(|entry| ((entry.worldline_id, entry.worldline_tick), entry)) .collect::>(); - for receipt in &recovery.echo_operation_receipts { - let basis = receipt.evaluation_basis(); - let worldline_id = basis.writer_head().worldline_id; - let frontier = runtime.worldlines().get(&worldline_id).ok_or( - TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "operation receipt names an unavailable recovery worldline", - }, - )?; - let parent_state = recovered_worldline_state_at( - &mut recovered_states, - recovered_provenance, - worldline_id, - frontier.state(), - basis.worldline_tick(), - "operation receipt parent state cannot be reconstructed", - )?; - let entry = entries.get(&(worldline_id, basis.worldline_tick())).ok_or( - TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "operation receipt has no recovered state transition", - }, - )?; - let installed = installations.get(&receipt.package_id()).ok_or( - TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "operation receipt has no recovered installation", - }, - )?; - let exact_parent_state_binding = entry - .patch - .as_ref() - .zip(entry.tick_receipt.as_ref()) - .is_some_and(|(patch, tick_receipt)| { - let exact_scope = operation_tick_scope_in_parent_state_v1( - tick_receipt, - patch, - receipt.installed_operation_id().as_hash(), - installed.program(), - parent_state, - ); - let application_basis_matches = exact_scope.is_some_and(|node| { - operation_application_basis_matches_scope_v1( - installed.program(), - node, - basis.application_basis(), - ) - }); - parent_state.state_root() == basis.state_root() && application_basis_matches - }); - if !exact_parent_state_binding { - return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "operation receipt patch disagrees with its reconstructed parent state", - }); - } - } - let submissions = recovery .witnessed_submissions .records() @@ -4316,190 +4374,306 @@ fn validate_recovered_echo_operation_parent_states( ), Vec<(Hash, Hash)>, >::new(); - for (submission_id, ingress_id, outcome) in &recovery.echo_operation_action_outcomes { + #[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] + enum BasisObligation { + Receipt(usize), + Action(usize), + } + let mut basis_obligations = Vec::new(); + for (index, receipt) in recovery.echo_operation_receipts.iter().enumerate() { + let basis = receipt.evaluation_basis(); + basis_obligations.push(( + basis.writer_head().worldline_id, + basis.worldline_tick(), + BasisObligation::Receipt(index), + )); + } + for (index, (submission_id, _, _)) in recovery.echo_operation_action_outcomes.iter().enumerate() + { let submission = submissions.get(submission_id).ok_or( TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "Action outcome has no retained submission", }, )?; - let correlation = correlations.get(submission_id).ok_or( + let bytes = echo_operation_action_invocation_bytes_v1(&submission.envelope).ok_or( TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action outcome has no retained receipt correlation", + detail: "Action outcome has no canonical invocation", }, )?; - let invocation_bytes = echo_operation_action_invocation_bytes_v1(&submission.envelope) - .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action outcome has no canonical invocation", - })?; - let invocation = inspect_action_invocation_v1(invocation_bytes).map_err(|_| { + let invocation = inspect_action_invocation_v1(bytes).map_err(|_| { TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "Action outcome invocation cannot be inspected", } })?; - action_scopes.insert(*submission_id, invocation.scope); - let installed = installations.get(&invocation.package_id).ok_or( - TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action outcome has no recovered installation", - }, - )?; let basis = invocation.evaluation_basis; - let basis_worldline_id = basis.writer_head().worldline_id; - let frontier = runtime.worldlines().get(&basis_worldline_id).ok_or( - TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action basis names an unavailable recovery worldline", - }, - )?; - let basis_state = recovered_worldline_state_at( - &mut recovered_states, - recovered_provenance, - basis_worldline_id, - frontier.state(), + basis_obligations.push(( + basis.writer_head().worldline_id, basis.worldline_tick(), - "Action basis state cannot be reconstructed", - )?; - if basis_state.state_root() != basis.state_root() - || !action_application_basis_matches_state_v1(installed, invocation_bytes, basis_state) - .unwrap_or(false) - || !evaluation_basis_matches_recovered_coordinate(basis, &entries) - { - return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action basis disagrees with reconstructed causal state", - }); - } - let tick_before = correlation - .worldline_tick_after - .as_u64() - .checked_sub(1) - .map(crate::WorldlineTick::from_raw) - .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action outcome has an impossible Tick coordinate", - })?; - let basis_is_current = - basis.writer_head() == correlation.head_key && basis.worldline_tick() == tick_before; - let basis_changed = matches!( - outcome, - EchoOperationActionOutcomeV1::Obstructed(obstruction) - if obstruction.kind() == crate::EchoOperationObstructionKindV1::BasisChanged - ); - if basis_changed == basis_is_current { - return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action basis posture disagrees with the scheduler Tick parent", - }); - } - let transition = entries - .get(&(correlation.head_key.worldline_id, tick_before)) - .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action outcome has no recovered scheduler transition", - })?; - let policy_id = transition - .patch - .as_ref() - .map(crate::WorldlineTickPatchV1::policy_id) - .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action outcome scheduler transition has no retained patch", - })?; - let reconstructed = match outcome { - EchoOperationActionOutcomeV1::Committed(receipt) => { - let prepared = reconstruct_action_preparation_v1( - installed, - invocation_bytes, - receipt.retained_invocation_admission_maximum_budget(), - receipt.retained_invocation_admission_policy_id(), - receipt.invocation_admission_id(), - basis_state, - policy_id, - ) - .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "committed Action preparation cannot be reconstructed", - })?; - if prepared.package_id() != receipt.package_id() - || prepared.installed_operation_id() != receipt.installed_operation_id() - || prepared.invocation_id() != receipt.invocation_id() - || prepared.evaluation_basis().identity() != receipt.evaluation_basis_id() - || prepared.private_evaluation_id() != receipt.private_evaluation_id() - || prepared.prepared_patch_digest() != receipt.prepared_patch_digest() - || prepared.result_id() != receipt.prepared_result_id() - || prepared.actual_footprint_digest() != receipt.actual_footprint_digest() - || prepared.preparation_id() != receipt.preparation_id() - { + BasisObligation::Action(index), + )); + } + // Sort verification references; retained WAL/decision order remains unchanged. + basis_obligations.sort_unstable(); + for (_, _, obligation) in basis_obligations { + match obligation { + BasisObligation::Receipt(index) => { + let receipt = &recovery.echo_operation_receipts[index]; + let basis = receipt.evaluation_basis(); + let worldline_id = basis.writer_head().worldline_id; + let frontier = runtime.worldlines().get(&worldline_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "operation receipt names an unavailable recovery worldline", + }, + )?; + let parent_state = recovered_worldline_state_at( + &mut recovered_states, + recovered_provenance, + worldline_id, + frontier.state(), + basis.worldline_tick(), + "operation receipt parent state cannot be reconstructed", + &mut work, + )?; + let entry = entries.get(&(worldline_id, basis.worldline_tick())).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "operation receipt has no recovered state transition", + }, + )?; + let installed = installations.get(&receipt.package_id()).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "operation receipt has no recovered installation", + }, + )?; + let exact_parent_state_binding = entry + .patch + .as_ref() + .zip(entry.tick_receipt.as_ref()) + .is_some_and(|(patch, tick_receipt)| { + let exact_scope = operation_tick_scope_in_parent_state_v1( + tick_receipt, + patch, + receipt.installed_operation_id().as_hash(), + installed.program(), + parent_state, + ); + let application_basis_matches = exact_scope.is_some_and(|node| { + operation_application_basis_matches_scope_v1( + installed.program(), + node, + basis.application_basis(), + ) + }); + parent_state.state_root() == basis.state_root() && application_basis_matches + }); + if !exact_parent_state_binding { return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: - "committed Action evidence disagrees with reconstructed preparation", + "operation receipt patch disagrees with its reconstructed parent state", }); } - reconstructed_evaluations.insert( - *submission_id, - EchoOperationPreparationV1::Prepared(prepared.clone()), - ); - Some(prepared) } - EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { - let prepared = reconstruct_action_preparation_v1( - installed, - invocation_bytes, - conflict.invocation_admission_maximum_budget, - conflict.invocation_admission_policy_id, - conflict.invocation_admission_id, - basis_state, - policy_id, + BasisObligation::Action(index) => { + let (submission_id, ingress_id, outcome) = + &recovery.echo_operation_action_outcomes[index]; + let submission = submissions.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no retained submission", + }, + )?; + let correlation = correlations.get(submission_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no retained receipt correlation", + }, + )?; + let invocation_bytes = echo_operation_action_invocation_bytes_v1( + &submission.envelope, ) .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "conflicting Action preparation cannot be reconstructed", + detail: "Action outcome has no canonical invocation", })?; - if prepared.package_id() != conflict.package_id - || prepared.installed_operation_id() != conflict.installed_operation_id - || prepared.invocation_id() != conflict.invocation_id - || prepared.evaluation_basis().identity() != conflict.evaluation_basis_id - || prepared.private_evaluation_id() != conflict.private_evaluation_id - || prepared.prepared_patch_digest() != conflict.prepared_patch_digest - || prepared.result_id() != conflict.prepared_result_id - || prepared.actual_footprint_digest() != conflict.actual_footprint_digest - || prepared.preparation_id() != conflict.preparation_id + let invocation = inspect_action_invocation_v1(invocation_bytes).map_err(|_| { + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome invocation cannot be inspected", + } + })?; + action_scopes.insert(*submission_id, invocation.scope); + let installed = installations.get(&invocation.package_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no recovered installation", + }, + )?; + let basis = invocation.evaluation_basis; + let basis_worldline_id = basis.writer_head().worldline_id; + let frontier = runtime.worldlines().get(&basis_worldline_id).ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action basis names an unavailable recovery worldline", + }, + )?; + let basis_state = recovered_worldline_state_at( + &mut recovered_states, + recovered_provenance, + basis_worldline_id, + frontier.state(), + basis.worldline_tick(), + "Action basis state cannot be reconstructed", + &mut work, + )?; + if basis_state.state_root() != basis.state_root() + || !action_application_basis_matches_state_v1( + installed, + invocation_bytes, + basis_state, + ) + .unwrap_or(false) + || !evaluation_basis_matches_recovered_coordinate(basis, &entries) { return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "conflict evidence disagrees with reconstructed preparation", + detail: "Action basis disagrees with reconstructed causal state", }); } - reconstructed_evaluations.insert( - *submission_id, - EchoOperationPreparationV1::Prepared(prepared.clone()), + let tick_before = correlation + .worldline_tick_after + .as_u64() + .checked_sub(1) + .map(crate::WorldlineTick::from_raw) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has an impossible Tick coordinate", + })?; + let basis_is_current = basis.writer_head() == correlation.head_key + && basis.worldline_tick() == tick_before; + let basis_changed = matches!( + outcome, + EchoOperationActionOutcomeV1::Obstructed(obstruction) + if obstruction.kind() == crate::EchoOperationObstructionKindV1::BasisChanged ); - Some(prepared) - } - EchoOperationActionOutcomeV1::Obstructed(obstruction) => { - if basis_changed { - if !action_admission_evidence_matches_v1( - installed, - invocation_bytes, - obstruction.invocation_admission_maximum_budget(), - obstruction.invocation_admission_policy_id(), - obstruction.invocation_admission_id(), - ) { - return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + if basis_changed == basis_is_current { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action basis posture disagrees with the scheduler Tick parent", + }); + } + let transition = entries + .get(&(correlation.head_key.worldline_id, tick_before)) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome has no recovered scheduler transition", + })?; + let policy_id = transition + .patch + .as_ref() + .map(crate::WorldlineTickPatchV1::policy_id) + .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "Action outcome scheduler transition has no retained patch", + })?; + let reconstructed = match outcome { + EchoOperationActionOutcomeV1::Committed(receipt) => { + let prepared = reconstruct_action_preparation_v1( + installed, + invocation_bytes, + receipt.retained_invocation_admission_maximum_budget(), + receipt.retained_invocation_admission_policy_id(), + receipt.invocation_admission_id(), + basis_state, + policy_id, + ) + .ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "committed Action preparation cannot be reconstructed", + }, + )?; + if prepared.package_id() != receipt.package_id() + || prepared.installed_operation_id() != receipt.installed_operation_id() + || prepared.invocation_id() != receipt.invocation_id() + || prepared.evaluation_basis().identity() + != receipt.evaluation_basis_id() + || prepared.private_evaluation_id() != receipt.private_evaluation_id() + || prepared.prepared_patch_digest() != receipt.prepared_patch_digest() + || prepared.result_id() != receipt.prepared_result_id() + || prepared.actual_footprint_digest() + != receipt.actual_footprint_digest() + || prepared.preparation_id() != receipt.preparation_id() + { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: + "committed Action evidence disagrees with reconstructed preparation", + }); + } + reconstructed_evaluations.insert( + *submission_id, + EchoOperationPreparationV1::Prepared(prepared.clone()), + ); + Some(prepared) + } + EchoOperationActionOutcomeV1::RejectedFootprintConflict(conflict) => { + let prepared = reconstruct_action_preparation_v1( + installed, + invocation_bytes, + conflict.invocation_admission_maximum_budget, + conflict.invocation_admission_policy_id, + conflict.invocation_admission_id, + basis_state, + policy_id, + ) + .ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "conflicting Action preparation cannot be reconstructed", + }, + )?; + if prepared.package_id() != conflict.package_id + || prepared.installed_operation_id() != conflict.installed_operation_id + || prepared.invocation_id() != conflict.invocation_id + || prepared.evaluation_basis().identity() + != conflict.evaluation_basis_id + || prepared.private_evaluation_id() != conflict.private_evaluation_id + || prepared.prepared_patch_digest() != conflict.prepared_patch_digest + || prepared.result_id() != conflict.prepared_result_id + || prepared.actual_footprint_digest() + != conflict.actual_footprint_digest + || prepared.preparation_id() != conflict.preparation_id + { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: + "conflict evidence disagrees with reconstructed preparation", + }); + } + reconstructed_evaluations.insert( + *submission_id, + EchoOperationPreparationV1::Prepared(prepared.clone()), + ); + Some(prepared) + } + EchoOperationActionOutcomeV1::Obstructed(obstruction) => { + if basis_changed { + if !action_admission_evidence_matches_v1( + installed, + invocation_bytes, + obstruction.invocation_admission_maximum_budget(), + obstruction.invocation_admission_policy_id(), + obstruction.invocation_admission_id(), + ) { + return Err(TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "basis obstruction has invalid invocation-admission evidence", }); - } - reconstructed_evaluations.insert( - *submission_id, - EchoOperationPreparationV1::Obstructed(obstruction.clone()), - ); - None - } else { - let evaluation = reconstruct_action_evaluation_v1( - installed, - invocation_bytes, - obstruction.invocation_admission_maximum_budget(), - obstruction.invocation_admission_policy_id(), - obstruction.invocation_admission_id(), - basis_state, - policy_id, - ) - .ok_or( - TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "obstructed Action evaluation cannot be reconstructed", - }, - )?; - match &evaluation { + } + reconstructed_evaluations.insert( + *submission_id, + EchoOperationPreparationV1::Obstructed(obstruction.clone()), + ); + None + } else { + let evaluation = reconstruct_action_evaluation_v1( + installed, + invocation_bytes, + obstruction.invocation_admission_maximum_budget(), + obstruction.invocation_admission_policy_id(), + obstruction.invocation_admission_id(), + basis_state, + policy_id, + ) + .ok_or( + TrustedRuntimeWalError::EchoOperationExecutionMismatch { + detail: "obstructed Action evaluation cannot be reconstructed", + }, + )?; + match &evaluation { EchoOperationPreparationV1::Obstructed(reconstructed) if reconstructed.has_same_predecision_evidence(obstruction) => {} EchoOperationPreparationV1::Prepared(_) @@ -4512,31 +4686,35 @@ fn validate_recovered_echo_operation_parent_states( }) } } - let prepared = match &evaluation { - EchoOperationPreparationV1::Prepared(prepared) => Some(prepared.clone()), - EchoOperationPreparationV1::Obstructed(_) => None, - }; - reconstructed_evaluations.insert(*submission_id, evaluation); - prepared + let prepared = match &evaluation { + EchoOperationPreparationV1::Prepared(prepared) => { + Some(prepared.clone()) + } + EchoOperationPreparationV1::Obstructed(_) => None, + }; + reconstructed_evaluations.insert(*submission_id, evaluation); + prepared + } + } + }; + if let Some(prepared) = reconstructed { + reconstructed_preparations.insert(*submission_id, prepared); } + action_tick_members + .entry(( + correlation.head_key, + correlation.worldline_tick_after, + correlation.commit_global_tick, + correlation.commit_hash, + )) + .or_default() + .push((*ingress_id, *submission_id)); } - }; - if let Some(prepared) = reconstructed { - reconstructed_preparations.insert(*submission_id, prepared); } - action_tick_members - .entry(( - correlation.head_key, - correlation.worldline_tick_after, - correlation.commit_global_tick, - correlation.commit_hash, - )) - .or_default() - .push((*ingress_id, *submission_id)); } - for ((head_key, worldline_tick_after, commit_global_tick, _), mut members) in - action_tick_members - { + let mut ordered_ticks = action_tick_members.into_iter().collect::>(); + ordered_ticks.sort_by_key(|((head, tick, _, _), _)| (head.worldline_id, *tick, head.head_id)); + for ((head_key, worldline_tick_after, commit_global_tick, _), mut members) in ordered_ticks { members.sort_by_key(|(ingress_id, _)| *ingress_id); let tick_before = worldline_tick_after .as_u64() @@ -4569,8 +4747,16 @@ fn validate_recovered_echo_operation_parent_states( frontier.state(), tick_before, "Action Tick parent state cannot be reconstructed", + &mut work, )? .clone(); + #[cfg(any(test, feature = "host_test"))] + work.note_states( + recovered_states + .values() + .map(|(_, state)| state) + .chain(std::iter::once(&reconstructed_state)), + ); let candidates = members .iter() .map( @@ -4715,7 +4901,7 @@ fn validate_recovered_echo_operation_parent_states( } } } - Ok(recovered_states.len()) + Ok(work) } fn evaluation_basis_matches_recovered_coordinate( diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 4aa30dbcb..14f43fae3 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3301,6 +3301,152 @@ fn wal_enabled_host_rejects_action_submission_without_durable_ack() { assert!(recovery.echo_operation_action_outcomes.is_empty()); } +#[cfg(feature = "host_test")] +#[test] +fn action_recovery_bounds_replay_work_and_retained_data_with_stale_bases() { + for (create_count, retain_stale) in [(8_usize, false), (16, true)] { + let wal_dir = TempWalDir::new(); + let mut stale_invocation = None; + let mut created_nodes = Vec::new(); + let head_key; + { + let (mut host, head, root) = fixture_host(); + head_key = head; + host.enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("the growth fixture WAL opens"); + let installed = install_fixture_creation_operation(&mut host); + host.install_echo_operation_action_admission_policy_v1(invocation_policy()); + for index in 0..create_count { + let node = NodeKey { + warp_id: root.warp_id, + local_id: make_node_id(&format!("recovery-created-{index}")), + }; + let basis = host + .echo_operation_evaluation_basis_v1( + head, + warp_core::echo_operation_anchored_node_absent_application_basis_v1(node), + ) + .expect("the exact absent-node basis resolves"); + let invocation = + EchoOperationInvocationV1::anchored_node_attachment_create_if_absent( + installed.package_id(), + installed.operation_coordinate(), + basis, + digest("fixture-authority-grant"), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + node, + b"created".to_vec(), + ) + .to_canonical_bytes() + .expect("growth invocation encodes"); + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head }, + invocation, + ) + .expect("growth Action encodes"); + let id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("growth Action is durable") + .submission_id; + host.tick_once().expect("growth Action reaches its Tick"); + assert!(matches!( + host.echo_operation_action_outcome_v1(&id), + Some(EchoOperationActionOutcomeV1::Committed(_)) + )); + created_nodes.push(node); + if retain_stale && index == 1 { + let late_node = NodeKey { + warp_id: root.warp_id, + local_id: make_node_id("delayed-absent"), + }; + let basis = host + .echo_operation_evaluation_basis_v1( + head, + warp_core::echo_operation_anchored_node_absent_application_basis_v1( + late_node, + ), + ) + .expect("the delayed basis resolves"); + stale_invocation = Some( + EchoOperationInvocationV1::anchored_node_attachment_create_if_absent( + installed.package_id(), + installed.operation_coordinate(), + basis, + digest("fixture-authority-grant"), + EchoOperationBudgetV1::new(16, 4_096, 4_096), + late_node, + b"late".to_vec(), + ) + .to_canonical_bytes() + .expect("delayed invocation encodes"), + ); + } + } + if let Some(invocation) = stale_invocation { + let envelope = warp_core::echo_operation_action_envelope_v1( + IngressTarget::ExactHead { key: head }, + invocation, + ) + .expect("delayed Action encodes"); + let id = host + .app() + .submit_intent_with_runtime_wal_ack(envelope) + .expect("delayed Action is durable") + .submission_id; + host.tick_once() + .expect("delayed Action receives a decision"); + assert!(matches!(host.echo_operation_action_outcome_v1(&id), + Some(EchoOperationActionOutcomeV1::Obstructed(o)) + if o.kind() == EchoOperationObstructionKindV1::BasisChanged)); + } + } + let (mut reopened, _, _) = fixture_host(); + reopened + .enable_runtime_wal(TrustedRuntimeWalConfig::filesystem(wal_dir.path())) + .expect("fresh host recovers every honest decision"); + let recovery = reopened + .runtime_wal() + .expect("recovered WAL exists") + .recover_read_only() + .expect("recovered evidence is inspectable"); + let work = recovery + .echo_operation_parent_state_work_for_test(reopened.runtime(), reopened.provenance()) + .expect("parent-state validation remains lawful"); + let ticks = create_count + usize::from(retain_stale); + assert!( + work.replayed_patches <= 2 * ticks, + "replay must not reapply every prefix: {work:?}" + ); + assert!( + work.peak_retained_states <= 2, + "one worldline needs one cursor plus one transient simulation: {work:?}" + ); + assert!( + work.peak_retained_node_records <= 2 * (create_count + 1), + "{work:?}" + ); + assert!(work.peak_retained_tick_records <= 2 * ticks, "{work:?}"); + assert!( + work.peak_retained_atom_bytes + <= 2 * (b"before".len() + create_count * b"created".len()), + "{work:?}" + ); + let state = reopened + .runtime() + .worldlines() + .get(&head_key.worldline_id) + .expect("recovered worldline exists") + .state(); + for node in created_nodes { + assert!( + matches!(state.store(&node.warp_id).and_then(|store| store.node_attachment(&node.local_id)), + Some(AttachmentValue::Atom(atom)) if atom.bytes.as_ref() == b"created") + ); + } + } +} + #[cfg(feature = "host_test")] #[test] fn scheduler_action_hot_path_uses_bounded_indexes() { diff --git a/docs/topics/WAL.md b/docs/topics/WAL.md index ad07dc020..293c39dd7 100644 --- a/docs/topics/WAL.md +++ b/docs/topics/WAL.md @@ -247,6 +247,10 @@ later recorded any decided receipt under host-owned WAL authority. State roots cover reachable WARP state. A detached create-if-absent cell can leave the root unchanged. The retained patch and commit bind that write. Root equality alone is not a complete recovery or no-mutation witness. +Action parent-state validation indexes verification obligations by exact worldline and tick. It checks basis obligations before reconstructing composite Tick decisions. Ordered cursors reuse verified prefixes within each sweep; a second sweep checks Tick parents, including delayed and cross-worldline bases. Raw retained decision order remains authoritative. Initial-boundary, checkpoint, patch, commit, result, obstruction, and conflict checks remain required. + +Each validation call retains one replay cursor per needed worldline and one transient Tick simulation. Replay-patch applications stay within two history sweeps. This bounds repeated prefix work and private snapshot retention; it does not claim linear elapsed time. State hashing and Tick simulation still depend on graph size. Test-only counters measure applied patches and logical retained data without becoming causal authority. + ## Recovery Postures The useful postures are: From ea4bee72f959683cae84bed35557f8976976fa38 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 7 Oct 2026 07:13:46 -0700 Subject: [PATCH 2/4] fix: place recovery obligation type before statements --- crates/warp-core/src/trusted_runtime_host.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index ef5119923..82e2cf7f6 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -4333,6 +4333,11 @@ fn validate_recovered_echo_operation_parent_states( recovered_provenance: &ProvenanceService, recovery: &TrustedRuntimeWalRecovery, ) -> Result { + #[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] + enum BasisObligation { + Receipt(usize), + Action(usize), + } let mut recovered_states = BTreeMap::new(); let mut work = ParentStateValidationWork::default(); let installations = recovery @@ -4374,11 +4379,6 @@ fn validate_recovered_echo_operation_parent_states( ), Vec<(Hash, Hash)>, >::new(); - #[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] - enum BasisObligation { - Receipt(usize), - Action(usize), - } let mut basis_obligations = Vec::new(); for (index, receipt) in recovery.echo_operation_receipts.iter().enumerate() { let basis = receipt.evaluation_basis(); From 6d01408079bf77e376846084ad234c6132b1acce Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 7 Oct 2026 08:49:18 -0700 Subject: [PATCH 3/4] fix(wal): reuse inspected Action invocations during recovery --- crates/warp-core/src/trusted_runtime_host.rs | 19 +++---------------- docs/topics/WAL.md | 2 +- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index 82e2cf7f6..f4e4caaae 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -4388,6 +4388,7 @@ fn validate_recovered_echo_operation_parent_states( BasisObligation::Receipt(index), )); } + let mut action_invocations = Vec::with_capacity(recovery.echo_operation_action_outcomes.len()); for (index, (submission_id, _, _)) in recovery.echo_operation_action_outcomes.iter().enumerate() { let submission = submissions.get(submission_id).ok_or( @@ -4411,6 +4412,7 @@ fn validate_recovered_echo_operation_parent_states( basis.worldline_tick(), BasisObligation::Action(index), )); + action_invocations.push((bytes, invocation)); } // Sort verification references; retained WAL/decision order remains unchanged. basis_obligations.sort_unstable(); @@ -4475,27 +4477,12 @@ fn validate_recovered_echo_operation_parent_states( BasisObligation::Action(index) => { let (submission_id, ingress_id, outcome) = &recovery.echo_operation_action_outcomes[index]; - let submission = submissions.get(submission_id).ok_or( - TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action outcome has no retained submission", - }, - )?; let correlation = correlations.get(submission_id).ok_or( TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "Action outcome has no retained receipt correlation", }, )?; - let invocation_bytes = echo_operation_action_invocation_bytes_v1( - &submission.envelope, - ) - .ok_or(TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action outcome has no canonical invocation", - })?; - let invocation = inspect_action_invocation_v1(invocation_bytes).map_err(|_| { - TrustedRuntimeWalError::EchoOperationExecutionMismatch { - detail: "Action outcome invocation cannot be inspected", - } - })?; + let (invocation_bytes, invocation) = &action_invocations[index]; action_scopes.insert(*submission_id, invocation.scope); let installed = installations.get(&invocation.package_id).ok_or( TrustedRuntimeWalError::EchoOperationExecutionMismatch { diff --git a/docs/topics/WAL.md b/docs/topics/WAL.md index 293c39dd7..689ad2616 100644 --- a/docs/topics/WAL.md +++ b/docs/topics/WAL.md @@ -247,7 +247,7 @@ later recorded any decided receipt under host-owned WAL authority. State roots cover reachable WARP state. A detached create-if-absent cell can leave the root unchanged. The retained patch and commit bind that write. Root equality alone is not a complete recovery or no-mutation witness. -Action parent-state validation indexes verification obligations by exact worldline and tick. It checks basis obligations before reconstructing composite Tick decisions. Ordered cursors reuse verified prefixes within each sweep; a second sweep checks Tick parents, including delayed and cross-worldline bases. Raw retained decision order remains authoritative. Initial-boundary, checkpoint, patch, commit, result, obstruction, and conflict checks remain required. +Action parent-state validation indexes verification obligations by exact worldline and tick. A first sweep checks receipt and Action basis obligations, including delayed and cross-worldline bases. A second sweep reconstructs composite Tick decisions from their scheduler Tick parents. Ordered cursors reuse verified prefixes within each sweep. Raw retained decision order remains authoritative. Initial-boundary, checkpoint, patch, commit, result, obstruction, and conflict checks remain required. Each validation call retains one replay cursor per needed worldline and one transient Tick simulation. Replay-patch applications stay within two history sweeps. This bounds repeated prefix work and private snapshot retention; it does not claim linear elapsed time. State hashing and Tick simulation still depend on graph size. Test-only counters measure applied patches and logical retained data without becoming causal authority. From ba3bc2ed16a808e6c3f2c327e0a18c53184d26bf Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 7 Oct 2026 08:53:51 -0700 Subject: [PATCH 4/4] test(wal): measure retained state after Tick composition --- crates/warp-core/src/trusted_runtime_host.rs | 7 +++++++ .../executable_operation_pipeline_tests.rs | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/crates/warp-core/src/trusted_runtime_host.rs b/crates/warp-core/src/trusted_runtime_host.rs index f4e4caaae..0f796264e 100644 --- a/crates/warp-core/src/trusted_runtime_host.rs +++ b/crates/warp-core/src/trusted_runtime_host.rs @@ -4793,6 +4793,13 @@ fn validate_recovered_echo_operation_parent_states( .map_err(|_| TrustedRuntimeWalError::EchoOperationExecutionMismatch { detail: "Action Tick composition cannot be reconstructed", })?; + #[cfg(any(test, feature = "host_test"))] + work.note_states( + recovered_states + .values() + .map(|(_, state)| state) + .chain(std::iter::once(&reconstructed_state)), + ); let reconstructed_outcomes = reconstructed_batch .outcomes .into_iter() diff --git a/crates/warp-core/tests/executable_operation_pipeline_tests.rs b/crates/warp-core/tests/executable_operation_pipeline_tests.rs index 14f43fae3..8c6bfc601 100644 --- a/crates/warp-core/tests/executable_operation_pipeline_tests.rs +++ b/crates/warp-core/tests/executable_operation_pipeline_tests.rs @@ -3432,6 +3432,24 @@ fn action_recovery_bounds_replay_work_and_retained_data_with_stale_bases() { <= 2 * (b"before".len() + create_count * b"created".len()), "{work:?}" ); + if !retain_stale { + // The last composition retains its parent cursor and the larger child. + assert_eq!( + work.peak_retained_node_records, + 2 * create_count + 1, + "{work:?}" + ); + assert_eq!( + work.peak_retained_tick_records, + 2 * create_count - 1, + "{work:?}" + ); + assert_eq!( + work.peak_retained_atom_bytes, + 2 * b"before".len() + (2 * create_count - 1) * b"created".len(), + "{work:?}" + ); + } let state = reopened .runtime() .worldlines()