From b3221b8f8d0bf4de130fcd2af0701afa5247a062 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:46:09 +0100 Subject: [PATCH 1/6] Bound operation-conflict projection refolds without full replay agent-identity: agent/compoundingtech/st-conversations agent-persona: generalist agent-supervisor: unavailable agent-tool: unknown agent-runtime: unknown agent-model: unknown --- crates/fractal-store/src/store.rs | 232 +++++++++++++++--- .../src/store/operation_conflict_tests.rs | 186 ++++++++++++++ crates/fractal-store/src/store/tests.rs | 26 +- crates/smallclaims/src/store.rs | 90 ++++++- crates/smallclaims/src/store/runtime.rs | 5 + crates/smallclaims/src/windows/holds.rs | 31 ++- docs/st3/large-value-sites.json | 10 +- docs/st3/large-values.md | 17 ++ docs/st3/replication.md | 15 ++ ...tion-conflict-bounded-refold-20261111.json | 11 + 10 files changed, 557 insertions(+), 66 deletions(-) create mode 100644 crates/fractal-store/src/store/operation_conflict_tests.rs create mode 100644 release-notes/operation-conflict-bounded-refold-20261111.json diff --git a/crates/fractal-store/src/store.rs b/crates/fractal-store/src/store.rs index 0d59fbf9673..ae505419f7e 100644 --- a/crates/fractal-store/src/store.rs +++ b/crates/fractal-store/src/store.rs @@ -81,6 +81,8 @@ pub mod mailbox_changes; pub mod mailbox_lease; #[cfg(test)] pub mod message_send_tests; +#[cfg(test)] +mod operation_conflict_tests; // Qualification only: no production caller or Source registration. #[cfg(test)] pub mod message_point_qualification; @@ -11007,7 +11009,9 @@ pub fn rebuild_planning_tx(transaction: &Transaction<'_>) -> Result<()> { fn project_planning_claims_tx(transaction: &Transaction<'_>, claim_ids: Vec) -> Result<()> { holds::rebuilt(claim_ids.len()); - for id in claim_ids { + holds::fold_progress(0, claim_ids.len()); + let total = claim_ids.len(); + for (position, id) in claim_ids.into_iter().enumerate() { let claim = claim_by_id_tx(transaction, &id)? .context("a planning claim disappeared during replay")?; // A claim admission accepted but this projection cannot read, such as one from a faulty @@ -11017,6 +11021,7 @@ fn project_planning_claims_tx(transaction: &Transaction<'_>, claim_ids: Vec, session: &str) )?; let claim_ids = statement .query_map([format!("planning-session/{session}")], |row| row.get::<_, String>(0))? + .take(if CONFLICT_REFOLD.get() { OPERATION_CONFLICT_REFOLD_CLAIMS + 1 } else { usize::MAX }) .collect::, _>>()?; + check_conflict_refold_size(claim_ids.len()).map_err(anyhow::Error::new)?; drop(statement); // Only this session's claims are decided again, so only their quarantine is forgotten. for id in &claim_ids { @@ -16930,6 +16937,7 @@ fn claims_in_replay_order_tx( for subject in subjects { for claim in by_subject.query_map([subject], row).map_err(internal)? { found.push(claim.map_err(internal)?); + check_conflict_refold_size(found.len())?; } } let mut by_range = transaction @@ -16944,6 +16952,7 @@ fn claims_in_replay_order_tx( .map_err(internal)? { found.push(claim.map_err(internal)?); + check_conflict_refold_size(found.len())?; } } found.sort_by(|left, right| left.1.cmp(&right.1)); @@ -17061,8 +17070,10 @@ fn rebuild_runs_tx(transaction: &Transaction<'_>, runs: &BTreeSet) -> Re .map_err(internal)? .execute(/* sqlite-large-value: allow-for-now lv-h090e8ca62948 owner=fractal_store::store TODO=trace-statement-and-value-representation */ [String::from_utf8(claim_ids).map_err(internal)?]) .map_err(internal)?; + holds::fold_progress(0, claims.len() * 3); for pass in 0..3 { - for claim in &claims { + for (position, claim) in claims.iter().enumerate() { + holds::fold_progress(pass * claims.len() + position + 1, claims.len() * 3); let included = match pass { 0 => claim.kind == "mission-run.created", 1 => { @@ -17134,7 +17145,8 @@ fn rebuild_base_aggregate_tx( Some(&[kind]), )?; holds::rebuilt(claims.len()); - for claim in &claims { + holds::fold_progress(0, claims.len()); + for (position, claim) in claims.iter().enumerate() { project_claim_isolated_tx(transaction, "projection:base", claim, || { match claim.kind.as_str() { "intent.desired" => { @@ -17146,6 +17158,7 @@ fn rebuild_base_aggregate_tx( _ => select_replicated_mission(transaction, claim, claim.store_index), } })?; + holds::fold_progress(position + 1, claims.len()); } if let Aggregate::Desired(subject) = aggregate { retirements::project_subject(transaction, subject)?; @@ -17172,6 +17185,56 @@ fn incremental_claim_error(error: FractalError, claim: &ClaimRecord, stage: &'st None => error, } } +// A conflict receipt may repair one small subject, never silently borrow the writer for an +// arbitrarily large history. 4096 covers sixteen normal projection pages; larger histories +// retain their previous projection and an explicit degraded diagnostic. +const OPERATION_CONFLICT_REFOLD_CLAIMS: usize = 4096; +thread_local! { + static CONFLICT_REFOLD: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +fn check_conflict_refold_size(claims: usize) -> Result<(), FractalError> { + if claims > OPERATION_CONFLICT_REFOLD_CLAIMS && CONFLICT_REFOLD.get() { + return Err(FractalError::new("operation-conflict-refold-limit", + "affected projection exceeds the bounded operation-conflict refold")); + } + Ok(()) +} + +fn refold_after_operation_conflict( + transaction: &Transaction<'_>, + diagnostics: &mut [FractalError], + affected: &[usize], + project: impl FnOnce() -> Result<(), FractalError>, +) -> Result<(), FractalError> { + if affected.is_empty() { return project(); } + transaction.execute_batch(/* sqlite-large-value: allow-for-now lv-h58880ffb5371 owner=fractal_store::store TODO=trace-statement-and-value-representation */ "SAVEPOINT operation_conflict_refold").map_err(internal)?; + struct Reset(bool); + impl Drop for Reset { + fn drop(&mut self) { CONFLICT_REFOLD.set(self.0); } + } + let _reset = Reset(CONFLICT_REFOLD.replace(true)); + match project() { + Ok(()) => transaction.execute_batch(/* sqlite-large-value: allow-for-now lv-h53bf80630f0f owner=fractal_store::store TODO=trace-statement-and-value-representation */ "RELEASE operation_conflict_refold").map_err(internal), + Err(error) => { + transaction.execute_batch( + /* sqlite-large-value: allow-for-now lv-hc33c6f16bf1d owner=fractal_store::store TODO=trace-statement-and-value-representation */ "ROLLBACK TO operation_conflict_refold; RELEASE operation_conflict_refold", + ).map_err(internal)?; + for &index in affected { + let diagnostic = &mut diagnostics[index]; + diagnostic.details.insert("projection_degraded".into(), json!(true)); + diagnostic.details.insert("refold_error_code".into(), json!(error.code)); + transaction.execute( + /* sqlite-large-value: allow-for-now lv-h3b9353d94be0 owner=fractal_store::store TODO=bound-or-reference-values-and-reclamation */ "UPDATE projection_health SET status='stale', error_code=?1, error_message=?2 + WHERE aggregate=?3", + params![error.code, json!({"conflict":diagnostic.details,"repair_error":error.message}).to_string(), + format!("projection:operation-conflict:{}", diagnostic.details["projection_operation_id"].as_str().unwrap_or(""))], + ).map_err(internal)?; + } + Ok(()) + } + } +} /// Retain the guard reason for the always-on log at the actual fallback boundary. fn replay_needed(reason: &'static str) -> IncrementalProjection { @@ -17189,7 +17252,7 @@ pub fn try_project_all_simple_replication_tx( origin, current_index_tx(transaction).map_err(internal)?, ) - .map(|decision| decision == IncrementalProjection::Projected) + .map(|decision| matches!(decision, IncrementalProjection::Projected | IncrementalProjection::ProjectedWithContext { .. })) } pub fn try_project_simple_replication_tx( @@ -17321,6 +17384,7 @@ pub fn try_project_simple_replication_tx( mark_dirty(&mut dirty, part, "local claim"); } } + let mut unsupported_conflict_subjects = BTreeSet::new(); for claim in &claims { let has_operation = claim.body.get("_operation").is_some(); // Supported work transitions already share canonical ordering and affected-tree @@ -17375,6 +17439,22 @@ pub fn try_project_simple_replication_tx( } else { "non-incremental-kind" }; + if reason == "non-incremental-kind" + && let Some((operation_id, digest)) = operation_parts(&claim.body) { + let conflict: bool = transaction.query_row( + "SELECT EXISTS(SELECT 1 FROM operations WHERE id=?1 + AND (request_digest<>?2 OR state='conflict')) + OR EXISTS(SELECT 1 FROM claims INDEXED BY claims_operation_index + WHERE json_extract(body,'$._operation.id')=?1 + AND json_extract(body,'$._operation.request_digest')<>?2 + AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=claims.id))", + params![operation_id, digest], |row| row.get(0), + ).map_err(internal)?; + if conflict { + unsupported_conflict_subjects.insert(claim.subject.clone()); + continue; + } + } crate::profile::note(&format!("replay: {reason}")); return Ok(IncrementalProjection::ReplayWithContext { reason, @@ -17556,11 +17636,11 @@ pub fn try_project_simple_replication_tx( } } } - // Event-only claims do not change the structural graph. Their operation - // registry can advance with the frontier. Message sends reconcile an ambiguous - // receipt from that operation's indexed history, using the full replay's reducer. - // Other kinds retain the full replay guard for ambiguous IDs. - let mut reconciled_message_operations = BTreeSet::new(); + // Registry conflicts use the same keyed reducer as replay. Graph effects still fold all + // claims in canonical order, independently of which request the registry names. + let mut conflict_diagnostics = Vec::new(); + let mut aggregate_conflicts = BTreeMap::>::new(); + let mut planning_conflicts = BTreeMap::>::new(); for claim in &claims { let Some((operation_id, request_digest)) = operation_parts(&claim.body) else { continue; @@ -17577,31 +17657,76 @@ pub fn try_project_simple_replication_tx( if repaired { continue; } - if claim.kind == "message.sent" && reconciled_message_operations.contains(operation_id) { - continue; + let mut current = operation_tx(transaction, operation_id).map_err(|error| { + incremental_claim_error(internal(error), claim, "operation-registry") + })?; + if current.is_none() { + register_operation_tx(transaction, claim).map_err(internal)?; + current = operation_tx(transaction, operation_id).map_err(internal)?; + if current.as_ref().is_none_or(|(_, _, state)| state == "active") { + continue; + } + // A checkpoint-only receipt supplies the conflicting predecessor, not the + // newly retained claim the registry now necessarily names. + let previous: String = transaction.query_row( + "SELECT id FROM checkpoint_claims WHERE operation_id=?1 AND request_digest<>?2 + AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=checkpoint_claims.id) + ORDER BY request_digest,id LIMIT 1", params![operation_id, request_digest], |row| row.get(0), + ).map_err(internal)?; + if let Some((_, canonical, _)) = &mut current { *canonical = previous; } } - if let Some((stored_digest, _, state)) = - operation_tx(transaction, operation_id).map_err(|error| { - incremental_claim_error(internal(error), claim, "operation-registry") - })? + if let Some((stored_digest, previous_claim_id, state)) = current && (stored_digest != request_digest || state != "active") { - if claim.kind == "message.sent" { - repair_operations_tx(transaction, &[operation_id.to_owned()]).map_err(|error| { - incremental_claim_error(internal(error), claim, "operation-registry") - })?; - reconciled_message_operations.insert(operation_id.to_owned()); - continue; + holds::stage("operation-conflict", || -> anyhow::Result<()> { + let replaced: bool = transaction.query_row( + "SELECT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=?1)", + [&previous_claim_id], |row| row.get(0), + )?; + if replaced { + // Explicit record repair can remove the row's summarized receipt. + repair_operations_tx(transaction, &[operation_id.to_owned()])?; + } else { + register_operation_tx(transaction, claim)?; + } + Ok(()) + }).map_err(|error| incremental_claim_error(internal(error), claim, "operation-registry"))?; + let canonical_claim_id = operation_tx(transaction, operation_id) + .map_err(internal)?.map(|(_, id, _)| id); + let count: u64 = transaction.query_row( + "SELECT COUNT(*) FROM operations WHERE state='conflict'", [], |row| row.get(0), + ).map_err(internal)?; + let mut diagnostic = incremental_claim_error( + FractalError::new("operation-conflict", "conflicting operation reconciled canonically without replay"), + claim, "operation-registry", + ).with_detail("previous_claim_id", previous_claim_id) + .with_detail("canonical_claim_id", canonical_claim_id) + .with_detail("operation_conflict_count", count); + if unsupported_conflict_subjects.contains(&claim.subject) { + diagnostic.details.insert("projection_degraded".into(), json!(true)); + diagnostic.details.insert("refold_error_code".into(), json!("operation-conflict-no-bounded-refold")); } - crate::profile::note("replay: operation-conflict"); - return Ok(IncrementalProjection::ReplayWithContext { - reason: "operation-conflict", - details: incremental_claim_error(FractalError::new("operation-conflict", - "operation digest differs or registry state is not active"), claim, - "operation-registry").with_detail("projection_error", format!( - "operation registry refuses incoming claim: digest_matches={}; stored state {}", stored_digest == request_digest, state.chars().take(32).collect::() - )).details, - }); + transaction.execute( + /* sqlite-large-value: allow-for-now lv-h00461836c5f4 owner=fractal_store::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) + VALUES (?1,?2,?3,?4,?5) + ON CONFLICT(aggregate) DO UPDATE SET status=excluded.status,error_code=excluded.error_code, + error_message=excluded.error_message,updated_at_unix_ms=excluded.updated_at_unix_ms", + params![format!("projection:operation-conflict:{operation_id}"), + if unsupported_conflict_subjects.contains(&claim.subject) { "stale" } else { "indeterminate" }, + if unsupported_conflict_subjects.contains(&claim.subject) { "operation-conflict-no-bounded-refold" } else { "operation-conflict" }, + serde_json::to_string(&diagnostic.details).map_err(internal)?, now_ms().to_string()], + ).map_err(internal)?; + let index = conflict_diagnostics.len(); + if let Some(session) = claim.subject.strip_prefix("planning-session/") { + planning_conflicts.entry(session.to_owned()).or_default().push(index); + } else if replay_projects_kind(&claim.kind) + && !unsupported_conflict_subjects.contains(&claim.subject) + && let Some(aggregate) = aggregate_of_tx(transaction, claim)? { + aggregate_conflicts.entry(aggregate.clone()).or_default().push(index); + mark_dirty(&mut dirty, aggregate, "operation conflict canonical refold"); + } + conflict_diagnostics.push(diagnostic); + continue; } register_operation_tx(transaction, claim).map_err(|error| { incremental_claim_error(internal(error), claim, "operation-registry") @@ -17659,6 +17784,9 @@ pub fn try_project_simple_replication_tx( // A claim is left to its aggregate's rebuild when that aggregate is dirty, and left waiting // when it is about a run tree that nothing has created yet. let mut in_dirty = |claim: &ClaimRecord| -> Result { + if unsupported_conflict_subjects.contains(&claim.subject) { + return Ok(true); + } if Store::simple_replication_kind(&claim.kind) && !claim.kind.starts_with("work.") { return Ok(false); } @@ -17747,7 +17875,10 @@ pub fn try_project_simple_replication_tx( .partition(|aggregate| matches!(aggregate, Aggregate::RunTree(_) | Aggregate::RunPart(_))); for aggregate in bases { holds::stage("rebuild-base", || { - rebuild_base_aggregate_tx(transaction, aggregate) + refold_after_operation_conflict(transaction, &mut conflict_diagnostics, + aggregate_conflicts.get(aggregate).map_or(&[], Vec::as_slice), || { + rebuild_base_aggregate_tx(transaction, aggregate) + }) }) .map_err(|error| { let subject = match aggregate { @@ -17764,11 +17895,17 @@ pub fn try_project_simple_replication_tx( for aggregate in trees { let (rebuilt, root) = match aggregate { Aggregate::RunTree(root) => ( - holds::stage("rebuild-tree", || rebuild_run_tree_tx(transaction, root)), + holds::stage("rebuild-tree", || refold_after_operation_conflict( + transaction, &mut conflict_diagnostics, + aggregate_conflicts.get(aggregate).map_or(&[], Vec::as_slice), + || rebuild_run_tree_tx(transaction, root))), root, ), Aggregate::RunPart(top) => ( - holds::stage("rebuild-part", || rebuild_run_part_tx(transaction, top)), + holds::stage("rebuild-part", || refold_after_operation_conflict( + transaction, &mut conflict_diagnostics, + aggregate_conflicts.get(aggregate).map_or(&[], Vec::as_slice), + || rebuild_run_part_tx(transaction, top))), top, ), _ => continue, @@ -17782,8 +17919,16 @@ pub fn try_project_simple_replication_tx( match planning_sessions { Some(sessions) => { for session in sessions { + if unsupported_conflict_subjects.contains(&format!("planning-session/{session}")) { + continue; + } holds::stage("rebuild-planning-session", || { - rebuild_planning_session_tx(transaction, session) + refold_after_operation_conflict(transaction, &mut conflict_diagnostics, + planning_conflicts.get(session).map_or(&[], Vec::as_slice), || { + rebuild_planning_session_tx(transaction, session).map_err(|error| { + error.downcast::().unwrap_or_else(internal) + }) + }) }) .map_err(|error| { internal(error) @@ -17792,12 +17937,25 @@ pub fn try_project_simple_replication_tx( })?; } } + None if !unsupported_conflict_subjects.is_empty() => {}, None => holds::stage("rebuild-planning", || rebuild_planning_tx(transaction)) .map_err(|error| internal(error).with_detail("projection_stage", "planning-rebuild"))?, } - owned_sets::project_tx(transaction) - .map_err(|error| error.with_detail("projection_stage", "owned-set-project"))?; - Ok(IncrementalProjection::Projected) + if unsupported_conflict_subjects.is_empty() { + owned_sets::project_tx(transaction) + .map_err(|error| error.with_detail("projection_stage", "owned-set-project"))?; + } + // Prefer a degraded subject's identifiers over an unrelated later event-only conflict. + let diagnostic = conflict_diagnostics.iter().rposition(|error| + error.details.get("projection_degraded").and_then(Value::as_bool) == Some(true)) + .or_else(|| conflict_diagnostics.len().checked_sub(1)); + Ok(match diagnostic { + Some(index) => { + let diagnostic = conflict_diagnostics.swap_remove(index); + IncrementalProjection::ProjectedWithContext { code: diagnostic.code, details: diagnostic.details } + }, + None => IncrementalProjection::Projected, + }) } /// The graph tables a replay derives from claims alone, children before the tables their diff --git a/crates/fractal-store/src/store/operation_conflict_tests.rs b/crates/fractal-store/src/store/operation_conflict_tests.rs new file mode 100644 index 00000000000..eeb11651090 --- /dev/null +++ b/crates/fractal-store/src/store/operation_conflict_tests.rs @@ -0,0 +1,186 @@ +use super::*; +use super::message_send_tests::operation_rows; +use super::tests_shared::{TEST_FLEET, exchange_from}; +use super::tests_shared_seats::graph_digest_of; + +#[test] +fn operation_conflict_receipts_match_full_replay_in_opposite_orders() { + for kind in ["custom.test.observed", "intent.desired"] { + let sources = [Store::open_memory("early").unwrap(), Store::open_memory("late").unwrap()]; + let mut ids = Vec::new(); + for (number, source) in sources.iter().enumerate() { + let mut connection = source.connection.write(); + let tx = connection.transaction().unwrap(); + let mut body = if kind == "intent.desired" { + json!({"subject":"exec/conflict","kind":"exec","desired":{"number":number}}) + } else { json!({"fields":{"number":number}}) }; + body["_operation"] = json!({"id":"op/conflict-parity", + "request_digest":if number == 0 { "digest-z" } else { "digest-a" }}); + ids.push(append_claim_record_tx(&tx, &source.origin, "exec/conflict", kind, + None, &body, &[], None).unwrap().id); + tx.commit().unwrap(); + } + let mut answers = Vec::new(); + for reverse in [false, true] { + let target = Store::open_memory(if reverse { "receiver-b" } else { "receiver-a" }).unwrap(); + target.bind_fleet(TEST_FLEET).unwrap(); + target.project_replication_backlog().unwrap(); + let mut logs = Vec::new(); + FULL_REPLAYS.with(|count| count.set(0)); + for index in if reverse { [1, 0] } else { [0, 1] } { + let source = &sources[index]; + let exchange = exchange_from(source, &target.replication_inventory().unwrap()); + target.receive_replication_exchange(&source.origin, TEST_FLEET, &exchange).unwrap(); + target.validate_replication_backlog().unwrap(); + target.project_replication_backlog_with_log("conflict-test", |line| logs.push(line.to_owned())).unwrap(); + } + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0, "{kind}/{reverse}"); + let incremental = (operation_rows(&target.readers.get()), graph_digest_of(&target)); + target.replay_replication_graph().unwrap(); + assert_eq!(incremental, (operation_rows(&target.readers.get()), graph_digest_of(&target))); + let health = target.replication_status(false, None, &[]).unwrap(); + let conflict = health.unhealthy.iter().find(|row| row.error_code.as_deref() == Some("operation-conflict")).unwrap(); + let detail: Value = serde_json::from_str(conflict.error_message.as_ref().unwrap()).unwrap(); + assert_eq!(detail["operation_conflict_count"], 1); + assert_eq!(detail["projection_operation_id"], "op/conflict-parity"); + assert!(ids.iter().any(|id| detail["projection_claim_id"] == *id)); + let log = logs.iter().find_map(|line| line.strip_prefix("fractal: projection failure detail ")).unwrap(); + let log: Value = serde_json::from_str(log).unwrap(); + assert_eq!(log["code"], "operation-conflict"); + assert_eq!(log["operation_conflict_count"], 1); + assert_eq!(log["operation_id"], "op/conflict-parity"); + assert!(ids.iter().any(|id| log["previous_claim_id"] == *id)); + answers.push(incremental); + } + assert_eq!(answers[0], answers[1], "{kind}: arrival order changed the projection"); + } +} + +#[test] +fn operation_conflict_above_refold_cap_preserves_graph_and_reports_degraded() { + let store = Store::open_memory("fixture").unwrap(); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + for number in 0..=OPERATION_CONFLICT_REFOLD_CLAIMS { + let mut body = json!({"subject":"exec/large-conflict","kind":"exec","desired":{"number":number}}); + if number == 0 { + body["_operation"] = json!({"id":"op/large-conflict","request_digest":"digest-z"}); + } + append_claim_record_tx(&tx, "peer", "exec/large-conflict", "intent.desired", None, + &body, &[], None).unwrap(); + } + append_claim_record_tx(&tx, "peer", "custom/later-conflict", "custom.test.observed", None, + &json!({"fields":{},"_operation":{"id":"op/later-event","request_digest":"a"}}), &[], None).unwrap(); + tx.commit().unwrap(); + } + store.replay_replication_graph().unwrap(); + let desired = || store.readers.get().query_row( + "SELECT claim_id,body FROM desired WHERE subject='exec/large-conflict'", + [], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ).unwrap(); + let before = desired(); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + append_claim_record_tx(&tx, "peer", "exec/large-conflict", "intent.desired", None, + &json!({"subject":"exec/large-conflict","kind":"exec","desired":{"number":"new"}, + "_operation":{"id":"op/large-conflict","request_digest":"digest-a"}}), &[], None).unwrap(); + append_claim_record_tx(&tx, "peer", "custom/later-conflict", "custom.test.observed", None, + &json!({"fields":{},"_operation":{"id":"op/later-event","request_digest":"b"}}), &[], None).unwrap(); + tx.commit().unwrap(); + } + FULL_REPLAYS.with(|count| count.set(0)); + let mut logs = Vec::new(); + store.project_replication_backlog_with_log("large-conflict", |line| logs.push(line.to_owned())).unwrap(); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + assert_eq!(before, desired()); + let health = store.replication_status(false, None, &[]).unwrap(); + assert!(health.unhealthy.iter().any(|row| row.status == "stale" + && row.error_code.as_deref() == Some("operation-conflict-refold-limit") + && row.error_message.as_deref().unwrap().contains("op/large-conflict"))); + assert!(health.unhealthy.iter().any(|row| row.aggregate.ends_with("op/later-event") + && row.status == "indeterminate" && row.error_code.as_deref() == Some("operation-conflict"))); + let detail = logs.iter().find_map(|line| line.strip_prefix("fractal: projection failure detail ")).unwrap(); + let detail: Value = serde_json::from_str(detail).unwrap(); + assert_eq!(detail["projection_degraded"], true); + assert_eq!(detail["refold_error_code"], "operation-conflict-refold-limit"); + store.project_replication_backlog().unwrap(); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); +} + +#[test] +fn operation_conflict_refuses_a_guard_requested_full_replay() { + for checkpoint_only in [false, true] { + let store = Store::open_memory("fixture").unwrap(); + store.project_replication_backlog().unwrap(); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + if checkpoint_only { + tx.execute(/* sqlite-large-value: allow-for-now lv-hc0649a7fe6c7 owner=fractal_store::store::operation_conflict_tests TODO=stage-generation-and-keyed-reclamation */ "INSERT INTO checkpoint_claims(id,writer,sequence,envelope_hash,subject,kind, + actor,predecessors,operation_id,request_digest,accepted_at_unix_ms,checkpoint) + VALUES ('tombstone','peer',1,'envelope','custom/conflict','custom.test.observed',NULL,'[]', + 'op/guard-conflict','a',0,'checkpoint/2026-10-01')", []).unwrap(); + } else { + append_claim_record_tx(&tx, "peer", "custom/conflict", "custom.test.observed", + None, &json!({"fields":{},"_operation":{"id":"op/guard-conflict","request_digest":"a"}}), + &[], None).unwrap(); + } + append_claim_record_tx(&tx, "peer", "custom/conflict", "custom.test.observed", + None, &json!({"fields":{},"_operation":{"id":"op/guard-conflict","request_digest":"b"}}), + &[], None).unwrap(); + tx.execute(/* sqlite-large-value: allow-for-now lv-hf161f602d783 owner=fractal_store::store::operation_conflict_tests TODO=bound-or-reference-values-and-reclamation */ "UPDATE projection_health SET status='stale' WHERE aggregate='graph'", []).unwrap(); + tx.commit().unwrap(); + } + FULL_REPLAYS.with(|count| count.set(0)); + assert!(!store.project_replication_backlog().unwrap()); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + assert!(store.replication_status(false, None, &[]).unwrap().unhealthy.iter().any(|row| + row.error_code.as_deref() == Some("operation-conflict-projection-degraded"))); + } +} + +#[test] +fn operation_conflict_github_event_observers_never_replay_during_receipt() { + let subject = "github-event/github/acme/garden/pull-request/7/checks/abc/fail"; + let sources = [Store::open_memory("dev5").unwrap(), Store::open_memory("dev3").unwrap()]; + let mut claims = Vec::new(); + for (number, source) in sources.iter().enumerate() { + claims.push(source.append_claim(&ClaimInput { + subject: subject.into(), kind: fractal_schema::github_plugin::EVENT.into(), actor: None, + fields: BTreeMap::from([ + ("version".into(), json!(fractal_schema::github_plugin::VERSION)), + ("item".into(), json!("resource/github/acme/garden/pull-request/7")), + ("event".into(), json!("checks-failed")), + ("observer".into(), json!(format!("observer/github/{}", source.origin))), + ("at".into(), json!(format!("2026-11-10T23:0{number}:00.000Z"))), + ("detail".into(), json!(format!("observed by {}", source.origin))), + ("head_sha".into(), json!(format!("head-{number}"))), + ]), + evidence: Vec::new(), expected_subject: None, + idempotency_key: Some(format!("github-event:{subject}")), + }).unwrap()); + } + assert_eq!(claims[0].operation_id, claims[1].operation_id); + let mut answers = Vec::new(); + for reverse in [false, true] { + let target = Store::open_memory("receiver").unwrap(); + target.bind_fleet(TEST_FLEET).unwrap(); + target.project_replication_backlog().unwrap(); + FULL_REPLAYS.with(|count| count.set(0)); + for index in if reverse { [1, 0] } else { [0, 1] } { + let source = &sources[index]; + let exchange = exchange_from(source, &target.replication_inventory().unwrap()); + target.receive_replication_exchange(&source.origin, TEST_FLEET, &exchange).unwrap(); + target.validate_replication_backlog().unwrap(); + target.project_replication_backlog().unwrap(); + } + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + let before = (operation_rows(&target.readers.get()), graph_digest_of(&target)); + target.replay_replication_graph().unwrap(); + assert_eq!(before, (operation_rows(&target.readers.get()), graph_digest_of(&target))); + answers.push(before); + } + assert_eq!(answers[0], answers[1]); +} diff --git a/crates/fractal-store/src/store/tests.rs b/crates/fractal-store/src/store/tests.rs index 83c2bace5eb..1c800446535 100644 --- a/crates/fractal-store/src/store/tests.rs +++ b/crates/fractal-store/src/store/tests.rs @@ -2294,7 +2294,7 @@ fn planning_replication_keeps_malformed_operation_and_unknown_kind_guards() { | IncrementalProjection::ReplayWithContext { reason, .. } => { assert_eq!(reason, expected) } - IncrementalProjection::Projected => panic!("{kind} bypassed {expected}"), + IncrementalProjection::Projected | IncrementalProjection::ProjectedWithContext { .. } => panic!("{kind} bypassed {expected}"), } transaction.rollback().unwrap(); } @@ -2308,7 +2308,6 @@ fn full_replay_logs_every_guard_and_incremental_error_once() { "frontier-ahead", "non-incremental-kind", "malformed-operation", - "operation-conflict", "incremental-error:internal", ] { FULL_REPLAYS.with(|count| count.set(0)); @@ -2320,22 +2319,17 @@ fn full_replay_logs_every_guard_and_incremental_error_once() { { let mut connection = store.connection.lock().unwrap(); let transaction = connection.transaction().unwrap(); - let operation = json!({"id":"op/replay-test", "request_digest":"digest-a"}); let (kind, body) = match reason { "non-incremental-kind" => ("work.unknown", json!({"fields":{}})), "malformed-operation" => ( "harness.observed", json!({"fields":{"state":"ready"}, "_operation":{"id":"op/malformed"}}), ), - "operation-conflict" => ( - "harness.observed", - json!({"fields":{"state":"ready"}, "_operation":operation}), - ), "incremental-error:internal" => ("intent.desired", json!({})), _ => ("harness.observed", json!({"fields":{"state":"ready"}})), }; // Seed historical input without pre-projecting it or requiring a current schema. - let claim = append_claim_record_tx( + append_claim_record_tx( &transaction, "node", "agent/node.test", @@ -2346,9 +2340,6 @@ fn full_replay_logs_every_guard_and_incremental_error_once() { None, ) .unwrap(); - if reason == "operation-conflict" { - transaction.execute(/* sqlite-large-value: allow-for-now lv-h5c80b3d61773 owner=fractal_store::store::tests TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO operations(id, request_digest, canonical_claim_id, state) VALUES ('op/replay-test', 'digest-b', ?1, 'active')", [&claim.id]).unwrap(); - } match reason { "missing-health" => { transaction @@ -2406,7 +2397,7 @@ fn full_replay_logs_every_guard_and_incremental_error_once() { .filter_map(|line| line.strip_prefix("fractal: projection failure detail ")) .map(|line| serde_json::from_str::(line).unwrap()) .collect::>(); - if matches!(reason, "incremental-error:internal" | "operation-conflict" + if matches!(reason, "incremental-error:internal" | "malformed-operation" | "non-incremental-kind") { assert_eq!(details.len(), 1, "{reason}"); assert_eq!(details[0]["subject"], "agent/node.test"); @@ -2416,10 +2407,7 @@ fn full_replay_logs_every_guard_and_incremental_error_once() { .is_some_and(|id| id.len() == 64) ); assert_eq!(details[0]["error_truncated"], false); - if reason == "operation-conflict" { - assert_eq!(details[0]["operation_id"], "op/replay-test"); - assert!(details[0]["error"].as_str().unwrap().contains("digest_matches=false")); - } else if reason == "incremental-error:internal" { + if reason == "incremental-error:internal" { assert!( details[0]["error"] .as_str() @@ -2543,7 +2531,7 @@ fn simple_replication_registers_unambiguous_event_operations() { None, ) .unwrap(); - assert!(!try_project_all_simple_replication_tx(&transaction, &store.origin).unwrap()); + assert!(try_project_all_simple_replication_tx(&transaction, &store.origin).unwrap()); transaction.commit().unwrap(); } assert!(store.project_replication_backlog().unwrap()); @@ -4426,6 +4414,10 @@ fn work_nudged_incremental_guards_and_rollback_preserve_the_committed_frontier() assert_eq!(refusal, "none"); assert_eq!(operation_tx(&transaction, "op/nudge-guard").unwrap().unwrap().2, "active"); } + IncrementalProjection::ProjectedWithContext { code, .. } => { + assert_eq!(refusal, "operation-conflict"); + assert_eq!(code, "operation-conflict"); + } IncrementalProjection::Replay(reason) | IncrementalProjection::ReplayWithContext { reason, .. } => assert_eq!(reason, refusal), } diff --git a/crates/smallclaims/src/store.rs b/crates/smallclaims/src/store.rs index bf6c6751c00..ff0868469d1 100644 --- a/crates/smallclaims/src/store.rs +++ b/crates/smallclaims/src/store.rs @@ -284,6 +284,11 @@ fn projection_failure_log_line( "stage": field("projection_stage"), "claim_id": field("projection_claim_id"), "subject": field("projection_subject"), "operation_id": field("projection_operation_id"), "claim_kind": field("projection_claim_kind"), + "operation_conflict_count": details.get("operation_conflict_count"), + "canonical_claim_id": field("canonical_claim_id"), + "previous_claim_id": field("previous_claim_id"), + "projection_degraded": details.get("projection_degraded"), + "refold_error_code": field("refold_error_code"), "frontier": if details.get("projection_frontier_unknown").and_then(Value::as_bool).unwrap_or(false) { None } else { Some(frontier) }, "target": target, "suppressed_errors": emission.suppressed, "rate_bucket_overflow": emission.overflow, "context_truncated": details.get("projection_context_truncated").and_then(Value::as_bool).unwrap_or(false) @@ -8892,6 +8897,17 @@ impl Store { .map_err(internal)?; match projected { IncrementalProjection::Projected => None, + IncrementalProjection::ProjectedWithContext { code, details } => { + let error = FractalError { + code, + message: "operation conflict reconciled without full replay".into(), + details, + }; + self.log_projection_failure( + &error, phase, frontier, target, &mut log_staged, + ); + None + } IncrementalProjection::Replay(reason) => Some(reason.to_owned()), IncrementalProjection::ReplayWithContext { reason, details } => { let message = details @@ -8933,6 +8949,48 @@ impl Store { drop(incremental); let projected = fallback_reason.is_none(); if let Some(reason) = fallback_reason { + // A failed bounded repair must never turn into a whole-log fold merely + // because another guard (or a projector error) requested fallback first. + let conflict: Option<(String, String, Option)> = transaction.query_row( + "SELECT incoming.id,json_extract(incoming.body,'$._operation.id'), + COALESCE(operations.canonical_claim_id,(SELECT id FROM checkpoint_claims + WHERE operation_id=json_extract(incoming.body,'$._operation.id') + AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=checkpoint_claims.id) + ORDER BY request_digest,id LIMIT 1)) + FROM claims incoming LEFT JOIN operations + ON operations.id=json_extract(incoming.body,'$._operation.id') + WHERE incoming.store_index>?1 AND incoming.store_index<=?2 + AND (operations.state='conflict' + OR operations.request_digest<>json_extract(incoming.body,'$._operation.request_digest') + OR EXISTS(SELECT 1 FROM claims other INDEXED BY claims_operation_index + WHERE json_extract(other.body,'$._operation.id')=json_extract(incoming.body,'$._operation.id') + AND json_extract(other.body,'$._operation.request_digest')<>json_extract(incoming.body,'$._operation.request_digest') + AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=other.id)) + OR EXISTS(SELECT 1 FROM checkpoint_claims + WHERE operation_id=json_extract(incoming.body,'$._operation.id') + AND request_digest<>json_extract(incoming.body,'$._operation.request_digest') + AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=checkpoint_claims.id))) + AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=incoming.id) + ORDER BY incoming.store_index DESC LIMIT 1", + params![frontier, through], |row| Ok((row.get(0)?,row.get(1)?,row.get(2)?)), + ).optional().map_err(internal)?; + if let Some((claim_id, operation_id, previous_claim_id)) = conflict { + let count: u64 = transaction.query_row( + "SELECT COUNT(*) + NOT EXISTS(SELECT 1 FROM operations WHERE id=?1 AND state='conflict') + FROM operations WHERE state='conflict'", [&operation_id], |row| row.get(0), + ).map_err(internal)?; + let error = FractalError::new("operation-conflict-projection-degraded", + json!({"operation_id":operation_id,"claim_id":claim_id, + "previous_claim_id":previous_claim_id,"operation_conflict_count":count, + "fallback_refused":reason}).to_string()) + .with_detail("projection_stage", "operation-conflict-no-replay") + .with_detail("projection_operation_id", operation_id) + .with_detail("projection_claim_id", claim_id) + .with_detail("previous_claim_id", previous_claim_id) + .with_detail("operation_conflict_count", count); + self.log_projection_failure(&error, phase, frontier, target, &mut log_staged); + return Err(error); + } log_staged(&full_replay_log_line(phase, &reason, frontier, target)); stages.enter("replay"); #[cfg(any(test, feature = "test-support"))] @@ -10730,21 +10788,37 @@ pub fn register_operation_tx(transaction: &Transaction<'_>, claim: &ClaimRecord) )?; Ok(dropped.is_empty()) } - Some((stored_digest, canonical, state)) if stored_digest == request_digest => { + Some((stored_digest, canonical, state)) if stored_digest == request_digest && state == "active" => { if claim.id < canonical { transaction.execute( /* sqlite-large-value: allow-for-now lv-h6f9c71946a9a owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "UPDATE operations SET canonical_claim_id=?2 WHERE id=?1", params![operation_id, claim.id], )?; } - Ok(state == "active" && canonical == claim.id) - } - Some(_) => { - transaction.execute( - /* sqlite-large-value: allow-for-now lv-hc9d2f03397dd owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "UPDATE operations SET state='conflict' WHERE id=?1", - [operation_id], + Ok(canonical == claim.id) + } + Some((stored_digest, canonical, state)) => { + // The row summarizes all earlier receipts. Its minimum digest may survive only + // as a tombstone, so include the retained canonical claim's actual digest too. + // This is the full replay's reducer, with constant work per new receipt. + let canonical_digest: String = transaction.query_row( + "SELECT json_extract(body,'$._operation.request_digest') FROM claims WHERE id=?1", + [&canonical], |row| row.get(0), )?; - Ok(false) + let mut metadata = OperationMetadata::default(); + metadata.include(stored_digest.clone(), String::new(), false); + metadata.include(canonical_digest, canonical.clone(), true); + metadata.include(request_digest.to_owned(), claim.id.clone(), true); + let (digest, canonical_claim, mut next_state) = metadata.row().expect("a retained receipt"); + if state == "conflict" { next_state = state.clone(); } + if (digest.as_str(), canonical_claim.as_str(), next_state.as_str()) + != (stored_digest.as_str(), canonical.as_str(), state.as_str()) { + transaction.execute( + /* sqlite-large-value: allow-for-now lv-h60fdf4c7a583 owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "UPDATE operations SET request_digest=?2,canonical_claim_id=?3,state=?4 WHERE id=?1", + params![operation_id, digest, canonical_claim, next_state], + )?; + } + Ok(state == "active" && stored_digest == request_digest && canonical == claim.id) } } } diff --git a/crates/smallclaims/src/store/runtime.rs b/crates/smallclaims/src/store/runtime.rs index 1d66d0e19e3..6298808910b 100644 --- a/crates/smallclaims/src/store/runtime.rs +++ b/crates/smallclaims/src/store/runtime.rs @@ -31,6 +31,11 @@ pub type LegacyDigestTable = ( #[derive(Debug, PartialEq, Eq)] pub enum IncrementalProjection { Projected, + /// The frontier advanced, retaining a visible diagnostic without requesting replay. + ProjectedWithContext { + code: &'static str, + details: serde_json::Map, + }, Replay(&'static str), /// The same replay decision, with identifiers from the guard that requested it. ReplayWithContext { diff --git a/crates/smallclaims/src/windows/holds.rs b/crates/smallclaims/src/windows/holds.rs index a206f1b9d2c..ea8fcae1070 100644 --- a/crates/smallclaims/src/windows/holds.rs +++ b/crates/smallclaims/src/windows/holds.rs @@ -87,11 +87,17 @@ impl Holder { } fn json(&self) -> Value { - json!({ + let mut row = json!({ "holder": &*self.0.label, "client": self.0.client.as_deref(), "at": format!("{}:{}", self.0.at.file(), self.0.at.line()), - }) + }); + if let Some(clock) = &self.0.stages + && let Some((processed, total)) = clock.lock().unwrap_or_else(PoisonError::into_inner).progress { + row["processed"] = json!(processed); + row["total"] = json!(total); + } + row } } @@ -593,6 +599,7 @@ struct StageClock { spent: Vec<(&'static str, Duration)>, rebuilt: Vec<(&'static str, Rebuilt)>, closed: bool, + progress: Option<(u64, u64)>, } impl StageClock { @@ -739,6 +746,16 @@ pub fn rebuilt(claims: usize) { } } +/// Publish the current fold's progress on its live and completed writer-hold row. +pub fn fold_progress(processed: usize, total: usize) { + if let Some(clock) = STAGES.with(|stages| stages.borrow().clone()) { + let mut clock = clock.lock().unwrap_or_else(PoisonError::into_inner); + if !clock.closed { + clock.progress = Some((processed as u64, total as u64)); + } + } +} + /// A staged lend's stages and rebuilt aggregates, as the writer closed them. #[cfg(any(test, feature = "test-support"))] pub type StagedLend = (Vec<(&'static str, Duration)>, Vec<(&'static str, Rebuilt)>); @@ -1225,4 +1242,14 @@ mod tests { rebuilt(9); assert_eq!(stages.clock.lock().unwrap().rebuilt.len(), 2); } + + #[test] + fn fold_progress_is_visible_on_the_writer_hold_row() { + let _stages = Stages::begin(); + let holder = Holder::current(); + fold_progress(17, 42); + let row = holder.json(); + assert_eq!(row["processed"], 17); + assert_eq!(row["total"], 42); + } } diff --git a/docs/st3/large-value-sites.json b/docs/st3/large-value-sites.json index 6adaa294d2e..243fbe37c6a 100644 --- a/docs/st3/large-value-sites.json +++ b/docs/st3/large-value-sites.json @@ -832,6 +832,7 @@ {"id": "lv-h002dc3085322", "path": "crates/smallclaims/src/sqlite/read_lifetime/tests.rs", "kind": "sql-template", "fingerprint": "d8be2362a0af57f12843d781e1c80c8a8dd05891916daeba63b747bc2989bf79", "owner": "smallclaims::sqlite::read_lifetime::tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 170, "excerpt": "UPDATE payload SET value=2"}, {"id": "lv-h003472555c90", "path": "crates/fractal-store/src/store/checkpoint_capture_epoch_tests.rs", "kind": "sql-template", "fingerprint": "61fb77404b67fa172fc2866f25d7f3c62dae9f37da2a36c7cf0a808b4947d814", "owner": "fractal_store::store::checkpoint_capture_epoch_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 137, "excerpt": "INSERT INTO documents(name,hash,created_index,binding_claim_id) VALUES('newer','protection-blob',1,?1)"}, {"id": "lv-h003a82076c71", "path": "crates/fractal-store/src/store/attention_budget.rs", "kind": "unresolved-sql-sink", "fingerprint": "b293d9ae7302a1380124bc42f10a45cfc392045b485ec39514c358d9fe671132", "owner": "fractal_store::store::attention_budget", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 86, "excerpt": "ANCESTRY"}, + {"id": "lv-h00461836c5f4", "path": "crates/fractal-store/src/store.rs", "kind": "sql-template", "fingerprint": "594ff99df9fb0f74bf4a928cb45388948cfd3063a0cd906ebcb97762c002e04c", "owner": "fractal_store::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 17710, "excerpt": "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) VALUES (?1,?2,?3,?4,?5) ON CONFLICT(aggregate) DO UPDATE SET status=excluded.status,error_code=excluded.erro"}, {"id": "lv-h004d34afdcce", "path": "crates/fractal-store/src/store/resources_preparation_tests.rs", "kind": "sql-template", "fingerprint": "35766f6ffd0ec40bdb6a9e8f34fd2903059b1876392462f852f4ed5adff7c382", "owner": "fractal_store::store::resources_preparation_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 628, "excerpt": "INSERT INTO operations(id,request_digest,canonical_claim_id,state) VALUES('operation/example-preparation','example-digest',?1,'active')"}, {"id": "lv-h007497d73c53", "path": "crates/fractal-store/src/store/tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "9c20b409f669351299b9007fcd2272d94726d494a3b3bd13f9be9dd02657c935", "owner": "fractal_store::store::tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 7311, "excerpt": "params ! [ \"agent/grove/cedar\" , i64 : : MAX , \"current\" ] , | row | row . get ( 0 ) ,"}, {"id": "lv-h007aa9af14d7", "path": "crates/fractal-store/src/store/custom/tests_shared_delivery.rs", "kind": "included-source", "fingerprint": "5d604bf3342897bb94342ddbdbe8296d87ee778849bee124dd73f3c81b8f6a26", "owner": "fractal_store::store::custom::tests_shared_delivery", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 5, "excerpt": "include_str ! ( \"../../../../../examples/fractal/custom-review.json\" )"}, @@ -1805,6 +1806,7 @@ {"id": "lv-h3b5efa6a0e33", "path": "crates/fractal-delivery/src/store/unread_mail.rs", "kind": "sql-template", "fingerprint": "134987d473c7704be290adcf6d9a5a81e59fd04965f86c670e037ba5a3b2e3e5", "owner": "fractal_delivery::store::unread_mail", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 22, "excerpt": "DELETE FROM unread_mail WHERE subject=?1"}, {"id": "lv-h3b7ad1f6ed2e", "path": "crates/fractal-seats/src/store/reconcile_work_context/tests.rs", "kind": "sql-template", "fingerprint": "cb38c898cec3b14bce43b8bf47c7542985d1ff9eb881fdd6c785e1cabc42cecf", "owner": "fractal_seats::store::reconcile_work_context::tests", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 317, "excerpt": "UPDATE claims SET body=?2 WHERE id=?1"}, {"id": "lv-h3b83ae471055", "path": "crates/fractal-missions/src/store/mission_deadline_tests.rs", "kind": "sql-template", "fingerprint": "71a54dcd87bab81093c06b73a1c016316d7866397b96d7cda53913108a077003", "owner": "fractal_missions::store::mission_deadline_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 437, "excerpt": "INSERT INTO mission_run_deadlines VALUES(?1,?2,?3)"}, + {"id": "lv-h3b9353d94be0", "path": "crates/fractal-store/src/store.rs", "kind": "sql-template", "fingerprint": "99064acdd84ee1bc64ad788455f87f29dc1d2798b5ca4aabc5403bd8492b866d", "owner": "fractal_store::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 17228, "excerpt": "UPDATE projection_health SET status='stale', error_code=?1, error_message=?2 WHERE aggregate=?3"}, {"id": "lv-h3b9ab67a68ef", "path": "crates/fractal-store/src/store.rs", "kind": "sql-template", "fingerprint": "26455523b683785aabfebafad0c6909158ab0913d73dd979565de18772835b35", "owner": "fractal_store::store", "todo": "chunk-or-reference-exact-wire-values", "disposition": "allow-for-now", "line_hint": 4195, "excerpt": "UPDATE replica_records SET state='repaired', replacement_claim_id=?2, error_code=NULL, error_message=NULL, updated_at_unix_ms=?3 WHERE record_ref=?1 AND state IN ('invalid','unknown','repaired')"}, {"id": "lv-h3bb2a6750188", "path": "crates/fractal-store/src/store/directive_notes.rs", "kind": "sql-template", "fingerprint": "3471a6d5f6abf84c78e12cb83c44ef70d68c9e86851e280a246815946dab9e37", "owner": "fractal_store::store::directive_notes", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 127, "excerpt": "CREATE TRIGGER IF NOT EXISTS {name}"}, {"id": "lv-h3bb68cc79270", "path": "crates/fractal-store/src/store/client_messages.rs", "kind": "unresolved-sql-sink", "fingerprint": "37f485b5dcd90e8473f26d724b538b35aaf6cc186138fed0852f3155cb856e6b", "owner": "fractal_store::store::client_messages", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 598, "excerpt": "& format ! ( \"SELECT subject FROM local_client_message_selectors_v1 INDEXED BY {winner_index} WHERE born_index>=0 AND retired_index IS NULL AND {flag}=1 AND reminder IS NOT NULL AND reminder=?1 {scope"}, @@ -2226,6 +2228,7 @@ {"id": "lv-h539605b8dc87", "path": "crates/fractal-seats/src/store/reconcile_work_context/tests.rs", "kind": "sql-template", "fingerprint": "b7093ef52ec236587ad92b3ec6b9bace117767da78b39099d57b71ee47728c18", "owner": "fractal_seats::store::reconcile_work_context::tests", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 736, "excerpt": "UPDATE claims SET accepted_at_unix_ms=?2 WHERE subject=?1 AND kind='work.progress' AND json_extract(body,'$.fields.handoff_acknowledged') IS NOT NULL"}, {"id": "lv-h5397dfc4c7e7", "path": "crates/fractal-store/src/store/person_work.rs", "kind": "sql-template", "fingerprint": "51408d83b4a10d968e9fa11cf571cd379ee9b56c6590ded5502ec9c1f097a11f", "owner": "fractal_store::store::person_work", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 864, "excerpt": "UPDATE mission_definitions SET revision=?2, claim_id=?3 WHERE mission_id=?1"}, {"id": "lv-h53a01c6a0750", "path": "crates/fractal-missions/src/store/missions.rs", "kind": "sql-template", "fingerprint": "87d1a5981f0f202e915e07ebc4ae1b385e116780a56c4c8745c8cc56f69605bc", "owner": "fractal_missions::store::missions", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 6493, "excerpt": "UPDATE step_runs SET status='pending', attempt=?2, goals=?3, worker_reported=0, lease_owner=NULL, lease_incarnation=NULL, lease_expires_at_unix_ms=NULL, blocked_reason=?4, not_before_unix_ms=NULL, act"}, + {"id": "lv-h53bf80630f0f", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "622b078c8cbb989b29fbd26eed0b9011af73e795f18529a45a830928d37769dc", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 17218, "excerpt": "\"RELEASE operation_conflict_refold\""}, {"id": "lv-h53c8fb7ca9fb", "path": "crates/fractal-store/src/store/publication_facts.rs", "kind": "unresolved-sql-sink", "fingerprint": "8edb630b01a1f78d4abc5ee08b714ce7a8578b35d2269aec45684bf19ac1e4fc", "owner": "fractal_store::store::publication_facts", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 240, "excerpt": "& query"}, {"id": "lv-h53cc4b4e8c5e", "path": "crates/smallclaims/src/store.rs", "kind": "sql-template", "fingerprint": "0d946e3360d7776f1595f83a545408ac9800ccd2985bba17242a18ec3e0b35d0", "owner": "smallclaims::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 7898, "excerpt": "INSERT INTO meta(key,value) VALUES(?1,?2) ON CONFLICT(key) DO UPDATE SET value=excluded.value WHERE meta.value IS NOT excluded.value"}, {"id": "lv-h53f8f1472a60", "path": "crates/fractal-store/src/store/checkpoint_identity_reuse_tests.rs", "kind": "sql-template", "fingerprint": "2127b63e2ba0c0063034a2c5fed0a5c05426207b57aa10673d29283c2bc92112", "owner": "fractal_store::store::checkpoint_identity_reuse_tests", "todo": "chunk-or-reference-exact-wire-values", "disposition": "allow-for-now", "line_hint": 343, "excerpt": "UPDATE replica_records SET state=?1 WHERE claim_id=?2"}, @@ -2308,6 +2311,7 @@ {"id": "lv-h5880045ca4a3", "path": "crates/fractal-seats/src/store/stop_deferral.rs", "kind": "sql-template", "fingerprint": "c016a9e29e34161afb705f3f6d8b193d611b76ba836bfac5148aaf82e627c9ab", "owner": "fractal_seats::store::stop_deferral", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 106, "excerpt": "UPDATE local_stop_deferral_backfill SET complete=1 WHERE singleton=1"}, {"id": "lv-h58850df68114", "path": "crates/fractal-missions/src/store/operations.rs", "kind": "sql-template", "fingerprint": "60b9537b32e1a110f494ed8cdacd4f700cb1e1cb15fdcf0a7d860f3428d6e3bb", "owner": "fractal_missions::store::operations", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 48, "excerpt": "WITH RECURSIVE seq(n) AS (VALUES(0) UNION ALL SELECT n+1 FROM seq WHERE n<249999) INSERT INTO claims(id,batch_id,subject,kind,origin,body,predecessors,accepted_at_unix_ms) SELECT 'fixture-heartbeat-' "}, {"id": "lv-h588570f9435e", "path": "crates/fractal-missions/src/store/tests.rs", "kind": "sql-template", "fingerprint": "086466319522c51cebce6be6b3915dc9f3f246e6bad15787e405512a1dd6d05b", "owner": "fractal_missions::store::tests", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 6766, "excerpt": "UPDATE claims SET body=json_extract(body,'$.fields') WHERE id=?1"}, + {"id": "lv-h58880ffb5371", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "cd2e5712521901eb7ae9a4d0f938e3555e200c458530a66074cde71b0dfe7704", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 17211, "excerpt": "\"SAVEPOINT operation_conflict_refold\""}, {"id": "lv-h58e04850fe62", "path": "crates/fractal-store/src/store/agent_messages.rs", "kind": "sql-template", "fingerprint": "f0adec3eb7b6155c2fdc3047146c1149614d9ff74a3c9bfa727acb3e73978516", "owner": "fractal_store::store::agent_messages", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 298, "excerpt": "INSERT OR IGNORE INTO local_agent_message_pending SELECT subject FROM local_stream_sent"}, {"id": "lv-h58eafaf28baf", "path": "crates/smallclaims/src/store/events.rs", "kind": "unresolved-sql-sink", "fingerprint": "28baebc3d12621e39673279d6837b21441f1b563cd5b9ceee9a46a8215be5538", "owner": "smallclaims::store::events", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 42, "excerpt": "VIEW"}, {"id": "lv-h59110ab2cf1d", "path": "crates/fractal-store/src/store/completion_state_tests.rs", "kind": "sql-template", "fingerprint": "22ecb8d34339a970f64897ea8804fe073c159b4a8ea13a18dae192585bbfa098", "owner": "fractal_store::store::completion_state_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 110, "excerpt": "DELETE FROM projection_digest_repaired_claims WHERE id=?1"}, @@ -2364,7 +2368,6 @@ {"id": "lv-h5c7073d4c441", "path": "crates/fractal-store/src/store/observer_actuals_tests.rs", "kind": "sql-template", "fingerprint": "9f199b1dfb8ae2c647387724d6729bbb96f7b0f611c404fe6762312d498facf3", "owner": "fractal_store::store::observer_actuals_tests", "todo": "chunk-or-reference-exact-wire-values", "disposition": "allow-for-now", "line_hint": 296, "excerpt": "INSERT INTO replica_records(record_ref,writer,sequence,envelope_hash,position,raw,state, claim_id,updated_at_unix_ms) VALUES('record/late-rank','fixture',1,'fixture',-1,X'','valid',?1,'0')"}, {"id": "lv-h5c79b1985e4a", "path": "crates/fractal-store/src/store/checkpoint_rules/staged.rs", "kind": "unresolved-sql-sink", "fingerprint": "c4c37bd5d9efb6d50f3813021dd53d55975a93bfbf96898ee5c434158e2a9b9c", "owner": "fractal_store::store::checkpoint_rules::staged", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 30, "excerpt": "& next , [ & after ] , | row | row . get ( 0 )"}, {"id": "lv-h5c7fa1bc38b3", "path": "crates/smallclaims/src/store/principals/judging.rs", "kind": "unresolved-sql-sink", "fingerprint": "55e502c7aacacc6446d3cc57d1e0e110132c95b60cc9ed4b6c676a145dea96a6", "owner": "smallclaims::store::principals::judging", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 77, "excerpt": "[ claim_id ] , | row | Ok ( ( row . get : : < _ , u64 > ( 0 ) ? , row . get : : < _ , String > ( 1 ) ? , row . get : : < _ , String > ( 2 ) ? , row . get : : < _ , Option < String > > ( 3 ) ? , row . "}, - {"id": "lv-h5c80b3d61773", "path": "crates/fractal-store/src/store/tests.rs", "kind": "sql-template", "fingerprint": "ad4facccd3aafe4f73026861ec3d9a7f55d21a296acfaa4b42bfaf101cbf13f8", "owner": "fractal_store::store::tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 2350, "excerpt": "INSERT INTO operations(id, request_digest, canonical_claim_id, state) VALUES ('op/replay-test', 'digest-b', ?1, 'active')"}, {"id": "lv-h5c86cd14f5b2", "path": "crates/fractal-store/src/store/checkpoint_tests.rs", "kind": "sql-template", "fingerprint": "d01d7270da8345cd1738861a99b451321d6ff4ef533c972b8dadab69de6ba3c0", "owner": "fractal_store::store::checkpoint_tests", "todo": "chunk-or-reference-variable-columns", "disposition": "allow-for-now", "line_hint": 666, "excerpt": "CREATE TABLE capture_registration_count(value INTEGER NOT NULL); INSERT INTO capture_registration_count VALUES (0); CREATE TRIGGER capture_registration_count AFTER UPDATE OF envelope_frontier, cut_uni"}, {"id": "lv-h5c8d7cd8a53b", "path": "crates/fractal-store/src/store/resources_preparation_tests.rs", "kind": "sql-template", "fingerprint": "8a4b30e68ee9ba464010f0df0f92e06c5b4e2b5520e9421ab1bf6ac06d94c66e", "owner": "fractal_store::store::resources_preparation_tests", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 206, "excerpt": "UPDATE claims SET subject=?2 WHERE id=?1"}, {"id": "lv-h5c9af66a6a3a", "path": "crates/smallclaims/src/store/checkpoint.rs", "kind": "unresolved-sql-sink", "fingerprint": "f91e379a207f4660271b94841c74ddaed726836889e811f67c10fc095f8ba837", "owner": "smallclaims::store::checkpoint", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 448, "excerpt": "params ! [ envelope . writer , envelope . sequence , envelope . envelope_hash ] , | row | row . get ( 0 ) ,"}, @@ -2444,6 +2447,7 @@ {"id": "lv-h60c64430d1d8", "path": "crates/smallclaims/src/store/repair_operation_tests.rs", "kind": "sql-template", "fingerprint": "8fbbbb1c9c620953ba488dc5d91a55c59378f1cfad0869c4b95861f316e8b6b1", "owner": "smallclaims::store::repair_operation_tests", "todo": "stage-generation-and-keyed-reclamation", "disposition": "allow-for-now", "line_hint": 1156, "excerpt": "WITH RECURSIVE n(value) AS (VALUES(0) UNION ALL SELECT value+1 FROM n WHERE value+1 Date: Wed, 11 Nov 2026 00:56:54 +0100 Subject: [PATCH 2/6] Use synthetic node names in operation conflict fixture agent-identity: agent/compoundingtech/st-conversations agent-persona: generalist agent-supervisor: unavailable agent-tool: unknown agent-runtime: unknown agent-model: unknown --- .../src/store/operation_conflict_tests.rs | 2 +- .../operation-conflict-synthetic-nodes-20261111.json | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 release-notes/operation-conflict-synthetic-nodes-20261111.json diff --git a/crates/fractal-store/src/store/operation_conflict_tests.rs b/crates/fractal-store/src/store/operation_conflict_tests.rs index eeb11651090..05efb9d1770 100644 --- a/crates/fractal-store/src/store/operation_conflict_tests.rs +++ b/crates/fractal-store/src/store/operation_conflict_tests.rs @@ -144,7 +144,7 @@ fn operation_conflict_refuses_a_guard_requested_full_replay() { #[test] fn operation_conflict_github_event_observers_never_replay_during_receipt() { let subject = "github-event/github/acme/garden/pull-request/7/checks/abc/fail"; - let sources = [Store::open_memory("dev5").unwrap(), Store::open_memory("dev3").unwrap()]; + let sources = [Store::open_memory("node-a").unwrap(), Store::open_memory("node-b").unwrap()]; let mut claims = Vec::new(); for (number, source) in sources.iter().enumerate() { claims.push(source.append_claim(&ClaimInput { diff --git a/release-notes/operation-conflict-synthetic-nodes-20261111.json b/release-notes/operation-conflict-synthetic-nodes-20261111.json new file mode 100644 index 00000000000..17cd199adc9 --- /dev/null +++ b/release-notes/operation-conflict-synthetic-nodes-20261111.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "summary": "Use synthetic node names in the operation-conflict regression fixture.", + "replay": {"status": "none", "detail": "Fixture names only; projection behavior is unchanged."}, + "schema": {"status": "none", "detail": "No schema changes."}, + "rules": {"status": "none", "detail": "No rule changes."}, + "compatibility": {"status": "none", "detail": "No runtime or wire changes."}, + "manual": {"status": "none", "detail": "No manual steps."}, + "recovery": {"status": "none", "detail": "No recovery changes."}, + "downtime": {"status": "none", "detail": "No runtime changes."} +} From 7d09f8173d51e855414fff502f0acede0f930b38 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:12:39 +0100 Subject: [PATCH 3/6] fix(store): preserve atomic subject-scoped conflict degradation agent-identity: unknown agent-persona: unknown agent-supervisor: unavailable agent-tool: unknown agent-runtime: unknown agent-model: unknown --- crates/fractal-store/src/store.rs | 98 ++++++++++++---- .../src/store/operation_conflict_tests.rs | 110 +++++++++++++++++- crates/fractal-store/src/store/owned_sets.rs | 10 +- crates/smallclaims/src/store.rs | 15 +-- docs/st3/large-value-sites.json | 5 +- docs/st3/large-values.md | 8 +- ...-conflict-atomic-degradation-20261111.json | 11 ++ 7 files changed, 213 insertions(+), 44 deletions(-) create mode 100644 release-notes/operation-conflict-atomic-degradation-20261111.json diff --git a/crates/fractal-store/src/store.rs b/crates/fractal-store/src/store.rs index cb20412e136..00b23d5c39b 100644 --- a/crates/fractal-store/src/store.rs +++ b/crates/fractal-store/src/store.rs @@ -17203,9 +17203,15 @@ fn check_conflict_refold_size(claims: usize) -> Result<(), FractalError> { Ok(()) } +struct OperationConflictDiagnostic { + error: FractalError, + // Full operation and subject identities; never derive a durable key from display context. + health_key: String, +} + fn refold_after_operation_conflict( transaction: &Transaction<'_>, - diagnostics: &mut [FractalError], + diagnostics: &mut [OperationConflictDiagnostic], affected: &[usize], project: impl FnOnce() -> Result<(), FractalError>, ) -> Result<(), FractalError> { @@ -17217,20 +17223,30 @@ fn refold_after_operation_conflict( } let _reset = Reset(CONFLICT_REFOLD.replace(true)); match project() { - Ok(()) => transaction.execute_batch(/* sqlite-large-value: allow-for-now lv-h53bf80630f0f owner=fractal_store::store TODO=trace-statement-and-value-representation */ "RELEASE operation_conflict_refold").map_err(internal), + Ok(()) => { + transaction.execute_batch(/* sqlite-large-value: allow-for-now lv-h53bf80630f0f owner=fractal_store::store TODO=trace-statement-and-value-representation */ "RELEASE operation_conflict_refold").map_err(internal)?; + for &index in affected { + transaction.execute( + /* sqlite-large-value: allow-for-now lv-he57da1ef4545 owner=fractal_store::store TODO=bound-or-reference-values-and-reclamation */ "UPDATE projection_health SET status='indeterminate',error_code='operation-conflict',error_message=?1 WHERE aggregate=?2", + params![serde_json::to_string(&diagnostics[index].error.details).map_err(internal)?, + diagnostics[index].health_key], + ).map_err(internal)?; + } + Ok(()) + }, Err(error) => { transaction.execute_batch( /* sqlite-large-value: allow-for-now lv-hc33c6f16bf1d owner=fractal_store::store TODO=trace-statement-and-value-representation */ "ROLLBACK TO operation_conflict_refold; RELEASE operation_conflict_refold", ).map_err(internal)?; for &index in affected { let diagnostic = &mut diagnostics[index]; - diagnostic.details.insert("projection_degraded".into(), json!(true)); - diagnostic.details.insert("refold_error_code".into(), json!(error.code)); + diagnostic.error.details.insert("projection_degraded".into(), json!(true)); + diagnostic.error.details.insert("refold_error_code".into(), json!(error.code)); transaction.execute( /* sqlite-large-value: allow-for-now lv-h3b9353d94be0 owner=fractal_store::store TODO=bound-or-reference-values-and-reclamation */ "UPDATE projection_health SET status='stale', error_code=?1, error_message=?2 WHERE aggregate=?3", - params![error.code, json!({"conflict":diagnostic.details,"repair_error":error.message}).to_string(), - format!("projection:operation-conflict:{}", diagnostic.details["projection_operation_id"].as_str().unwrap_or(""))], + params![error.code, json!({"conflict":diagnostic.error.details,"repair_error":error.message}).to_string(), + diagnostic.health_key], ).map_err(internal)?; } Ok(()) @@ -17257,6 +17273,31 @@ pub fn try_project_all_simple_replication_tx( .map(|decision| matches!(decision, IncrementalProjection::Projected | IncrementalProjection::ProjectedWithContext { .. })) } +// A whole-log replay is unsafe while this log contains conflicting operations. Keep the +// independent replay requirement visible, but let bounded receipt move past it. Repair is +// explicit: receipt must not repeatedly strand unrelated claims behind an unhealthy graph. +fn defer_replay_for_operation_conflict(tx: &Transaction<'_>, reason: &str) -> Result { + let conflict: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM operations WHERE state='conflict') + OR EXISTS(SELECT 1 FROM claims incoming INDEXED BY claims_operation_index + JOIN claims other INDEXED BY claims_operation_index + ON json_extract(other.body,'$._operation.id')=json_extract(incoming.body,'$._operation.id') + WHERE json_extract(incoming.body,'$._operation.id') IS NOT NULL + AND json_extract(other.body,'$._operation.request_digest')<>json_extract(incoming.body,'$._operation.request_digest') + AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=incoming.id OR id=other.id))", + [], |row| row.get(0), + ).map_err(internal)?; + if conflict { + tx.execute( + /* sqlite-large-value: allow-for-now lv-ha8b524108446 owner=fractal_store::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) + VALUES ('projection:deferred-replay','stale','operation-conflict-replay-deferred',?1,?2) + ON CONFLICT(aggregate) DO UPDATE SET error_message=excluded.error_message,updated_at_unix_ms=excluded.updated_at_unix_ms", + params![reason, now_ms().to_string()], + ).map_err(internal)?; + } + Ok(conflict) +} + pub fn try_project_simple_replication_tx( transaction: &Transaction<'_>, origin: &str, @@ -17270,10 +17311,13 @@ pub fn try_project_simple_replication_tx( ) .optional() .map_err(internal)?; - let Some((status, frontier)) = health else { - return Ok(replay_needed("missing-health")); + let (status, frontier) = match health { + Some(health) => health, + None if defer_replay_for_operation_conflict(transaction, "missing-health")? => ("healthy".to_owned(), 0), + None => return Ok(replay_needed("missing-health")), }; - if status != "healthy" { + if status != "healthy" + && !defer_replay_for_operation_conflict(transaction, "unhealthy-projection")? { return Ok(replay_needed("unhealthy-projection")); } if frontier > current_index_tx(transaction).map_err(internal)? { @@ -17457,6 +17501,10 @@ pub fn try_project_simple_replication_tx( continue; } } + if defer_replay_for_operation_conflict(transaction, reason)? { + unsupported_conflict_subjects.insert(claim.subject.clone()); + continue; + } crate::profile::note(&format!("replay: {reason}")); return Ok(IncrementalProjection::ReplayWithContext { reason, @@ -17695,25 +17743,23 @@ pub fn try_project_simple_replication_tx( }).map_err(|error| incremental_claim_error(internal(error), claim, "operation-registry"))?; let canonical_claim_id = operation_tx(transaction, operation_id) .map_err(internal)?.map(|(_, id, _)| id); - let count: u64 = transaction.query_row( - "SELECT COUNT(*) FROM operations WHERE state='conflict'", [], |row| row.get(0), - ).map_err(internal)?; let mut diagnostic = incremental_claim_error( FractalError::new("operation-conflict", "conflicting operation reconciled canonically without replay"), claim, "operation-registry", ).with_detail("previous_claim_id", previous_claim_id) - .with_detail("canonical_claim_id", canonical_claim_id) - .with_detail("operation_conflict_count", count); + .with_detail("canonical_claim_id", canonical_claim_id); if unsupported_conflict_subjects.contains(&claim.subject) { diagnostic.details.insert("projection_degraded".into(), json!(true)); diagnostic.details.insert("refold_error_code".into(), json!("operation-conflict-no-bounded-refold")); } + let health_key = format!("projection:operation-conflict:{}", json!([operation_id, claim.subject])); transaction.execute( - /* sqlite-large-value: allow-for-now lv-h00461836c5f4 owner=fractal_store::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) + /* sqlite-large-value: allow-for-now lv-hdacd188e0574 owner=fractal_store::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) VALUES (?1,?2,?3,?4,?5) ON CONFLICT(aggregate) DO UPDATE SET status=excluded.status,error_code=excluded.error_code, - error_message=excluded.error_message,updated_at_unix_ms=excluded.updated_at_unix_ms", - params![format!("projection:operation-conflict:{operation_id}"), + error_message=excluded.error_message,updated_at_unix_ms=excluded.updated_at_unix_ms + WHERE projection_health.status<>'stale'", + params![health_key, if unsupported_conflict_subjects.contains(&claim.subject) { "stale" } else { "indeterminate" }, if unsupported_conflict_subjects.contains(&claim.subject) { "operation-conflict-no-bounded-refold" } else { "operation-conflict" }, serde_json::to_string(&diagnostic.details).map_err(internal)?, now_ms().to_string()], @@ -17727,7 +17773,7 @@ pub fn try_project_simple_replication_tx( aggregate_conflicts.entry(aggregate.clone()).or_default().push(index); mark_dirty(&mut dirty, aggregate, "operation conflict canonical refold"); } - conflict_diagnostics.push(diagnostic); + conflict_diagnostics.push(OperationConflictDiagnostic { error: diagnostic, health_key }); continue; } register_operation_tx(transaction, claim).map_err(|error| { @@ -17943,17 +17989,15 @@ pub fn try_project_simple_replication_tx( None => holds::stage("rebuild-planning", || rebuild_planning_tx(transaction)) .map_err(|error| internal(error).with_detail("projection_stage", "planning-rebuild"))?, } - if unsupported_conflict_subjects.is_empty() { - owned_sets::project_tx(transaction) - .map_err(|error| error.with_detail("projection_stage", "owned-set-project"))?; - } + owned_sets::project_excluding_tx(transaction, &unsupported_conflict_subjects) + .map_err(|error| error.with_detail("projection_stage", "owned-set-project"))?; // Prefer a degraded subject's identifiers over an unrelated later event-only conflict. - let diagnostic = conflict_diagnostics.iter().rposition(|error| - error.details.get("projection_degraded").and_then(Value::as_bool) == Some(true)) + let diagnostic = conflict_diagnostics.iter().rposition(|diagnostic| + diagnostic.error.details.get("projection_degraded").and_then(Value::as_bool) == Some(true)) .or_else(|| conflict_diagnostics.len().checked_sub(1)); Ok(match diagnostic { Some(index) => { - let diagnostic = conflict_diagnostics.swap_remove(index); + let diagnostic = conflict_diagnostics.swap_remove(index).error; IncrementalProjection::ProjectedWithContext { code: diagnostic.code, details: diagnostic.details } }, None => IncrementalProjection::Projected, @@ -18167,6 +18211,10 @@ pub fn project_claim_isolated_tx( transaction .execute_batch(/* sqlite-large-value: allow-for-now lv-h0dbb3a5ec63d owner=fractal_store::store TODO=trace-statement-and-value-representation */ "ROLLBACK TO project_claim; RELEASE project_claim") .map_err(internal)?; + // A bounded aggregate refold must be atomic, not a partial replay with holes. + if CONFLICT_REFOLD.get() { + return Err(error); + } transaction .execute( /* sqlite-large-value: allow-for-now lv-ha5591425e84e owner=fractal_store::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate, status, error_code, error_message, updated_at_unix_ms) diff --git a/crates/fractal-store/src/store/operation_conflict_tests.rs b/crates/fractal-store/src/store/operation_conflict_tests.rs index 05efb9d1770..0514d9491f2 100644 --- a/crates/fractal-store/src/store/operation_conflict_tests.rs +++ b/crates/fractal-store/src/store/operation_conflict_tests.rs @@ -41,13 +41,12 @@ fn operation_conflict_receipts_match_full_replay_in_opposite_orders() { let health = target.replication_status(false, None, &[]).unwrap(); let conflict = health.unhealthy.iter().find(|row| row.error_code.as_deref() == Some("operation-conflict")).unwrap(); let detail: Value = serde_json::from_str(conflict.error_message.as_ref().unwrap()).unwrap(); - assert_eq!(detail["operation_conflict_count"], 1); + assert_eq!(target.idempotency_conflicts(1).unwrap().0, 1); assert_eq!(detail["projection_operation_id"], "op/conflict-parity"); assert!(ids.iter().any(|id| detail["projection_claim_id"] == *id)); let log = logs.iter().find_map(|line| line.strip_prefix("fractal: projection failure detail ")).unwrap(); let log: Value = serde_json::from_str(log).unwrap(); assert_eq!(log["code"], "operation-conflict"); - assert_eq!(log["operation_conflict_count"], 1); assert_eq!(log["operation_id"], "op/conflict-parity"); assert!(ids.iter().any(|id| log["previous_claim_id"] == *id)); answers.push(incremental); @@ -59,13 +58,14 @@ fn operation_conflict_receipts_match_full_replay_in_opposite_orders() { #[test] fn operation_conflict_above_refold_cap_preserves_graph_and_reports_degraded() { let store = Store::open_memory("fixture").unwrap(); + let operation_id = format!("op/large-conflict-{}", "x".repeat(300)); { let mut connection = store.connection.write(); let tx = connection.transaction().unwrap(); for number in 0..=OPERATION_CONFLICT_REFOLD_CLAIMS { let mut body = json!({"subject":"exec/large-conflict","kind":"exec","desired":{"number":number}}); if number == 0 { - body["_operation"] = json!({"id":"op/large-conflict","request_digest":"digest-z"}); + body["_operation"] = json!({"id":operation_id,"request_digest":"digest-z"}); } append_claim_record_tx(&tx, "peer", "exec/large-conflict", "intent.desired", None, &body, &[], None).unwrap(); @@ -85,7 +85,7 @@ fn operation_conflict_above_refold_cap_preserves_graph_and_reports_degraded() { let tx = connection.transaction().unwrap(); append_claim_record_tx(&tx, "peer", "exec/large-conflict", "intent.desired", None, &json!({"subject":"exec/large-conflict","kind":"exec","desired":{"number":"new"}, - "_operation":{"id":"op/large-conflict","request_digest":"digest-a"}}), &[], None).unwrap(); + "_operation":{"id":operation_id,"request_digest":"digest-a"}}), &[], None).unwrap(); append_claim_record_tx(&tx, "peer", "custom/later-conflict", "custom.test.observed", None, &json!({"fields":{},"_operation":{"id":"op/later-event","request_digest":"b"}}), &[], None).unwrap(); tx.commit().unwrap(); @@ -98,8 +98,8 @@ fn operation_conflict_above_refold_cap_preserves_graph_and_reports_degraded() { let health = store.replication_status(false, None, &[]).unwrap(); assert!(health.unhealthy.iter().any(|row| row.status == "stale" && row.error_code.as_deref() == Some("operation-conflict-refold-limit") - && row.error_message.as_deref().unwrap().contains("op/large-conflict"))); - assert!(health.unhealthy.iter().any(|row| row.aggregate.ends_with("op/later-event") + && row.aggregate.contains(&operation_id))); + assert!(health.unhealthy.iter().any(|row| row.aggregate.contains("op/later-event") && row.status == "indeterminate" && row.error_code.as_deref() == Some("operation-conflict"))); let detail = logs.iter().find_map(|line| line.strip_prefix("fractal: projection failure detail ")).unwrap(); let detail: Value = serde_json::from_str(detail).unwrap(); @@ -107,6 +107,22 @@ fn operation_conflict_above_refold_cap_preserves_graph_and_reports_degraded() { assert_eq!(detail["refold_error_code"], "operation-conflict-refold-limit"); store.project_replication_backlog().unwrap(); assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + let stale_key = health.unhealthy.iter().find(|row| row.status == "stale").unwrap().aggregate.clone(); + let stale_message = health.unhealthy.iter().find(|row| row.aggregate == stale_key).unwrap().error_message.clone(); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + append_claim_record_tx(&tx, "peer", "exec/another-subject", "intent.desired", None, + &json!({"subject":"exec/another-subject","kind":"exec","desired":{}, + "_operation":{"id":operation_id,"request_digest":"digest-b"}}), &[], None).unwrap(); + tx.commit().unwrap(); + } + store.project_replication_backlog().unwrap(); + let health = store.replication_status(false, None, &[]).unwrap(); + let original = health.unhealthy.iter().find(|row| row.aggregate == stale_key).unwrap(); + assert_eq!(original.status, "stale"); + assert_eq!(original.error_message, stale_message); + assert_eq!(before, desired()); } #[test] @@ -184,3 +200,85 @@ fn operation_conflict_github_event_observers_never_replay_during_receipt() { } assert_eq!(answers[0], answers[1]); } + +#[test] +fn operation_conflict_in_second_receipt_chunk_fences_first_chunk_fallback() { + let source = Store::open_memory("source").unwrap(); + { + let mut connection = source.connection.write(); + let tx = connection.transaction().unwrap(); + for number in 0..smallclaims::store::PROJECTION_CHUNK_CLAIMS { + append_claim_record_tx(&tx, "source", "custom/multi-chunk", "custom.test.observed", + None, &json!({"fields":{"number":number}}), &[], None).unwrap(); + } + for digest in ["a", "b"] { + append_claim_record_tx(&tx, "source", "custom/multi-chunk-conflict", "custom.test.observed", + None, &json!({"fields":{},"_operation":{"id":"op/second-chunk","request_digest":digest}}), + &[], None).unwrap(); + } + tx.commit().unwrap(); + } + let target = Store::open_memory("target").unwrap(); + target.bind_fleet(TEST_FLEET).unwrap(); + let exchange = exchange_from(&source, &target.replication_inventory().unwrap()); + target.receive_replication_exchange("source", TEST_FLEET, &exchange).unwrap(); + target.validate_replication_backlog().unwrap(); + assert!(target.index().unwrap() > smallclaims::store::PROJECTION_CHUNK_CLAIMS as u64); + target.connection.write().execute( + /* sqlite-large-value: allow-for-now lv-hf0bbe1f00bc2 owner=fractal_store::store::operation_conflict_tests TODO=bound-or-reference-values-and-reclamation */ "UPDATE projection_health SET status='stale' WHERE aggregate='graph'", [], + ).unwrap(); + FULL_REPLAYS.with(|count| count.set(0)); + target.project_replication_backlog().unwrap(); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + assert!(target.replication_status(false, None, &[]).unwrap().unhealthy.iter().any(|row| + row.error_code.as_deref() == Some("operation-conflict-replay-deferred"))); + { + let mut connection = target.connection.write(); + let tx = connection.transaction().unwrap(); + append_claim_record_tx(&tx, "peer", "exec/after-deferred", "intent.desired", None, + &json!({"subject":"exec/after-deferred","kind":"exec","desired":{"value":"projected"}}), + &[], None).unwrap(); + tx.commit().unwrap(); + } + target.project_replication_backlog().unwrap(); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + assert!(target.readers.get().query_row( + "SELECT EXISTS(SELECT 1 FROM desired WHERE subject='exec/after-deferred')", + [], |row| row.get::<_, bool>(0), + ).unwrap()); +} + +#[test] +fn operation_conflict_refold_failure_restores_every_prior_desired_column() { + let store = Store::open_memory("fixture").unwrap(); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + append_claim_record_tx(&tx, "peer", "exec/refold-failure", "intent.desired", None, + &json!({"subject":"exec/refold-failure","kind":"exec","desired":{"value":"prior"}, + "_operation":{"id":"op/refold-failure","request_digest":"z"}}), &[], None).unwrap(); + append_claim_record_tx(&tx, "peer", "exec/refold-failure", "intent.desired", + None, &json!({}), &[], None).unwrap(); + tx.commit().unwrap(); + } + store.replay_replication_graph().unwrap(); + let bytes = || store.readers.get().query_row( + "SELECT json_array(subject,kind,revision,claim_id,body,member,owner_run,owner_generation,owner_step) + FROM desired WHERE subject='exec/refold-failure'", [], |row| row.get::<_, String>(0), + ).unwrap().into_bytes(); + let before = bytes(); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + append_claim_record_tx(&tx, "peer", "exec/refold-failure", "intent.desired", None, + &json!({"subject":"exec/refold-failure","kind":"exec","desired":{"value":"new"}, + "_operation":{"id":"op/refold-failure","request_digest":"a"}}), &[], None).unwrap(); + tx.commit().unwrap(); + } + FULL_REPLAYS.with(|count| count.set(0)); + store.project_replication_backlog().unwrap(); + assert_eq!(before, bytes()); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + assert!(store.replication_status(false, None, &[]).unwrap().unhealthy.iter().any(|row| + row.status == "stale" && row.aggregate.contains("op/refold-failure"))); +} diff --git a/crates/fractal-store/src/store/owned_sets.rs b/crates/fractal-store/src/store/owned_sets.rs index 9dc8daa09cd..20ee8b595dc 100644 --- a/crates/fractal-store/src/store/owned_sets.rs +++ b/crates/fractal-store/src/store/owned_sets.rs @@ -1208,8 +1208,16 @@ pub fn commit_tx( /// Rebuild the existing desired and mission projections through exact set member references. pub fn project_tx(transaction: &Transaction<'_>) -> Result<(), FractalError> { + project_excluding_tx(transaction, &BTreeSet::new()) +} + +/// Leave only explicitly degraded set subjects untouched while projecting unrelated sets. +pub fn project_excluding_tx( + transaction: &Transaction<'_>, + excluded_subjects: &BTreeSet, +) -> Result<(), FractalError> { for view in selected(transaction, None)? { - if !view.blockers.is_empty() { + if excluded_subjects.contains(&view.id) || !view.blockers.is_empty() { continue; } // Existing runtimes hold; callers fence actions below. for (s, (m, retired, implicit)) in effective_members(transaction, &view, None)? { diff --git a/crates/smallclaims/src/store.rs b/crates/smallclaims/src/store.rs index ff0868469d1..bbeffef0c55 100644 --- a/crates/smallclaims/src/store.rs +++ b/crates/smallclaims/src/store.rs @@ -8957,9 +8957,9 @@ impl Store { WHERE operation_id=json_extract(incoming.body,'$._operation.id') AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=checkpoint_claims.id) ORDER BY request_digest,id LIMIT 1)) - FROM claims incoming LEFT JOIN operations + FROM claims incoming INDEXED BY claims_operation_index LEFT JOIN operations ON operations.id=json_extract(incoming.body,'$._operation.id') - WHERE incoming.store_index>?1 AND incoming.store_index<=?2 + WHERE json_extract(incoming.body,'$._operation.id') IS NOT NULL AND (operations.state='conflict' OR operations.request_digest<>json_extract(incoming.body,'$._operation.request_digest') OR EXISTS(SELECT 1 FROM claims other INDEXED BY claims_operation_index @@ -8972,22 +8972,17 @@ impl Store { AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=checkpoint_claims.id))) AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=incoming.id) ORDER BY incoming.store_index DESC LIMIT 1", - params![frontier, through], |row| Ok((row.get(0)?,row.get(1)?,row.get(2)?)), + [], |row| Ok((row.get(0)?,row.get(1)?,row.get(2)?)), ).optional().map_err(internal)?; if let Some((claim_id, operation_id, previous_claim_id)) = conflict { - let count: u64 = transaction.query_row( - "SELECT COUNT(*) + NOT EXISTS(SELECT 1 FROM operations WHERE id=?1 AND state='conflict') - FROM operations WHERE state='conflict'", [&operation_id], |row| row.get(0), - ).map_err(internal)?; let error = FractalError::new("operation-conflict-projection-degraded", json!({"operation_id":operation_id,"claim_id":claim_id, - "previous_claim_id":previous_claim_id,"operation_conflict_count":count, + "previous_claim_id":previous_claim_id, "fallback_refused":reason}).to_string()) .with_detail("projection_stage", "operation-conflict-no-replay") .with_detail("projection_operation_id", operation_id) .with_detail("projection_claim_id", claim_id) - .with_detail("previous_claim_id", previous_claim_id) - .with_detail("operation_conflict_count", count); + .with_detail("previous_claim_id", previous_claim_id); self.log_projection_failure(&error, phase, frontier, target, &mut log_staged); return Err(error); } diff --git a/docs/st3/large-value-sites.json b/docs/st3/large-value-sites.json index 243fbe37c6a..0e9f52a2675 100644 --- a/docs/st3/large-value-sites.json +++ b/docs/st3/large-value-sites.json @@ -832,7 +832,6 @@ {"id": "lv-h002dc3085322", "path": "crates/smallclaims/src/sqlite/read_lifetime/tests.rs", "kind": "sql-template", "fingerprint": "d8be2362a0af57f12843d781e1c80c8a8dd05891916daeba63b747bc2989bf79", "owner": "smallclaims::sqlite::read_lifetime::tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 170, "excerpt": "UPDATE payload SET value=2"}, {"id": "lv-h003472555c90", "path": "crates/fractal-store/src/store/checkpoint_capture_epoch_tests.rs", "kind": "sql-template", "fingerprint": "61fb77404b67fa172fc2866f25d7f3c62dae9f37da2a36c7cf0a808b4947d814", "owner": "fractal_store::store::checkpoint_capture_epoch_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 137, "excerpt": "INSERT INTO documents(name,hash,created_index,binding_claim_id) VALUES('newer','protection-blob',1,?1)"}, {"id": "lv-h003a82076c71", "path": "crates/fractal-store/src/store/attention_budget.rs", "kind": "unresolved-sql-sink", "fingerprint": "b293d9ae7302a1380124bc42f10a45cfc392045b485ec39514c358d9fe671132", "owner": "fractal_store::store::attention_budget", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 86, "excerpt": "ANCESTRY"}, - {"id": "lv-h00461836c5f4", "path": "crates/fractal-store/src/store.rs", "kind": "sql-template", "fingerprint": "594ff99df9fb0f74bf4a928cb45388948cfd3063a0cd906ebcb97762c002e04c", "owner": "fractal_store::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 17710, "excerpt": "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) VALUES (?1,?2,?3,?4,?5) ON CONFLICT(aggregate) DO UPDATE SET status=excluded.status,error_code=excluded.erro"}, {"id": "lv-h004d34afdcce", "path": "crates/fractal-store/src/store/resources_preparation_tests.rs", "kind": "sql-template", "fingerprint": "35766f6ffd0ec40bdb6a9e8f34fd2903059b1876392462f852f4ed5adff7c382", "owner": "fractal_store::store::resources_preparation_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 628, "excerpt": "INSERT INTO operations(id,request_digest,canonical_claim_id,state) VALUES('operation/example-preparation','example-digest',?1,'active')"}, {"id": "lv-h007497d73c53", "path": "crates/fractal-store/src/store/tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "9c20b409f669351299b9007fcd2272d94726d494a3b3bd13f9be9dd02657c935", "owner": "fractal_store::store::tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 7311, "excerpt": "params ! [ \"agent/grove/cedar\" , i64 : : MAX , \"current\" ] , | row | row . get ( 0 ) ,"}, {"id": "lv-h007aa9af14d7", "path": "crates/fractal-store/src/store/custom/tests_shared_delivery.rs", "kind": "included-source", "fingerprint": "5d604bf3342897bb94342ddbdbe8296d87ee778849bee124dd73f3c81b8f6a26", "owner": "fractal_store::store::custom::tests_shared_delivery", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 5, "excerpt": "include_str ! ( \"../../../../../examples/fractal/custom-review.json\" )"}, @@ -3674,6 +3673,7 @@ {"id": "lv-ha879de4a84fc", "path": "crates/smallclaims/src/ivm.rs", "kind": "sql-template", "fingerprint": "e68ffead6145ce73d1af5369c111d2d00c0071122a333e1a268f7d54cc123970", "owner": "smallclaims::ivm", "todo": "chunk-or-reference-variable-columns", "disposition": "allow-for-now", "line_hint": 34, "excerpt": " CREATE TABLE IF NOT EXISTS ivm_source ( singleton INTEGER PRIMARY KEY CHECK(singleton=1), epoch INTEGER NOT NULL, admitted INTEGER NOT NULL, projected INTEGER NOT NULL, local_generation INTEGER NOT N"}, {"id": "lv-ha8b24ee58218", "path": "crates/fractal-api-core/src/api/client_messages_tests.rs", "kind": "sql-template", "fingerprint": "ac8e61bdb8ccbefab4c0f5bb21d9c3dd5d1a24bf62c97206a634159d3827200e", "owner": "fractal_api_core::api::client_messages_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 58, "excerpt": "DELETE FROM meta WHERE key IN ('client_message_selectors_v1_cut','client_message_selectors_v1_backfill')"}, {"id": "lv-ha8b271924da3", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "f67407213e838e5f2e13a1f1d64ffd1860b7693cd608733f031068e6a8f93109", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 12434, "excerpt": "params ! [ fence . subject , i64 : : MAX ] , | row | row . get ( 0 )"}, + {"id": "lv-ha8b524108446", "path": "crates/fractal-store/src/store.rs", "kind": "sql-template", "fingerprint": "294cfb32e3532acfcac05184513371d2aa77bf63b27a8ab38cf55a000bcae6ad", "owner": "fractal_store::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 17292, "excerpt": "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) VALUES ('projection:deferred-replay','stale','operation-conflict-replay-deferred',?1,?2) ON CONFLICT(aggrega"}, {"id": "lv-ha8e0f7b45414", "path": "crates/smallclaims/src/store/retired.rs", "kind": "sql-template", "fingerprint": "ffaca3cdb7720f9c68cd484e31b749d5d672c4f1362eaa6eaed133fcb28d6f5b", "owner": "smallclaims::store::retired", "todo": "chunk-or-reference-variable-columns", "disposition": "allow-for-now", "line_hint": 46, "excerpt": "CREATE TABLE IF NOT EXISTS retired_claim_ids (id BLOB NOT NULL PRIMARY KEY) WITHOUT ROWID; CREATE TABLE IF NOT EXISTS retired_envelopes ( writer TEXT NOT NULL, sequence INTEGER NOT NULL, envelope_hash"}, {"id": "lv-ha8edfbce8177", "path": "crates/fractal-store/src/store/attention_snapshot.rs", "kind": "unresolved-sql-sink", "fingerprint": "35ce0a9d145e7d2aeeaece1b87fb0fa8da30aed6b2c0320b5b9eaab689f36651", "owner": "fractal_store::store::attention_snapshot", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 126, "excerpt": "& canonical_sql ( \"SELECT claims.subject, claims.accepted_at_unix_ms FROM json_each(?1) requested JOIN claims ON claims.id=( SELECT claims.id FROM claims WHERE claims.subject=requested.value AND claim"}, {"id": "lv-ha8f290219962", "path": "crates/fractal-store/src/store/ordered_membership.rs", "kind": "sql-template", "fingerprint": "ef1529b31ddf4c4e2612d27ad794bea5a25b5ba85af764bd3d0208e97d59b66d", "owner": "fractal_store::store::ordered_membership", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 143, "excerpt": "INSERT INTO ordered_membership_lifecycle(subject,visible,revision) VALUES(?1,?2,?3) ON CONFLICT(subject) DO UPDATE SET visible=excluded.visible,revision=excluded.revision WHERE visible!=excluded.visib"}, @@ -4527,6 +4527,7 @@ {"id": "lv-hdab4da12bdd0", "path": "crates/fractal-store/src/store/backup_ledgers.rs", "kind": "unresolved-sql-sink", "fingerprint": "faa8085f1d6cc4397f9aca72ee2a5f42c6ed01d7fd0e0541367d188cdfbe3bc3", "owner": "fractal_store::store::backup_ledgers", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 236, "excerpt": "& page_query ( ledger , after . is_none ( ) )"}, {"id": "lv-hdacaab2c061c", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "73e9726766615edb1d3ffd4ae267633878f2752cea488a98caa6a111454943be", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 5610, "excerpt": "status_query"}, {"id": "lv-hdacd067611e1", "path": "crates/fractal-store/src/store/resources.rs", "kind": "sql-template", "fingerprint": "00cd35dacaffbd752a0f9a51ace18fbf4a5ec79846017c4962c570d55e4a0cd7", "owner": "fractal_store::store::resources", "todo": "chunk-or-reference-variable-columns", "disposition": "allow-for-now", "line_hint": 23, "excerpt": " CREATE TABLE IF NOT EXISTS resource_observations ( subject TEXT PRIMARY KEY, claim_id TEXT NOT NULL, kind TEXT NOT NULL, facts TEXT NOT NULL, observed_at TEXT NOT NULL, opened_by TEXT, opened_by_run "}, + {"id": "lv-hdacd188e0574", "path": "crates/fractal-store/src/store.rs", "kind": "sql-template", "fingerprint": "84e3f6dd8d320ef5ac0bb66dd71dfc19b5ec3f3abb87c1f2749dd51c5762147c", "owner": "fractal_store::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 17747, "excerpt": "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) VALUES (?1,?2,?3,?4,?5) ON CONFLICT(aggregate) DO UPDATE SET status=excluded.status,error_code=excluded.erro"}, {"id": "lv-hdad79852ce9a", "path": "crates/fractal-store/src/store/client_messages.rs", "kind": "sql-template", "fingerprint": "874136574d2c0c5d8497585bad8a70de4912c5f9e7783c43dbe867c29a405ec9", "owner": "fractal_store::store::client_messages", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 847, "excerpt": "UPDATE meta SET value=?2 WHERE key=?1 AND ?2 IS NOT NULL"}, {"id": "lv-hdade9b126123", "path": "crates/smallclaims/src/store/writer_heads.rs", "kind": "sql-template", "fingerprint": "f0d04717e2bf0edda759fb9cfef21299f14d5b67bc36be915ab8b47efbb65632", "owner": "smallclaims::store::writer_heads", "todo": "chunk-or-reference-variable-columns", "disposition": "allow-for-now", "line_hint": 12, "excerpt": " CREATE TABLE IF NOT EXISTS writer_heads ( writer TEXT NOT NULL PRIMARY KEY, sequence INTEGER NOT NULL, hash TEXT NOT NULL ) WITHOUT ROWID; CREATE TRIGGER IF NOT EXISTS writer_heads_batch_insert_v1 AF"}, {"id": "lv-hdae0d65c679a", "path": "crates/smallclaims/src/ivm/asynchronous.rs", "kind": "sql-template", "fingerprint": "fd047af90bba6f886f65e9b5dc860a15e3beec046cc2397270df306c47524f5c", "owner": "smallclaims::ivm::asynchronous", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 499, "excerpt": "UPDATE ivm_async_state SET draining=1 WHERE singleton=1"}, @@ -4713,6 +4714,7 @@ {"id": "lv-he57a4ce28014", "path": "crates/fractal/src/main_tests.rs", "kind": "included-source", "fingerprint": "e449b05971f992322e4786dc6cfdcfbd178929a158827d3f9e71f6847ea6fdf3", "owner": "fractal::main_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 3389, "excerpt": "include_str ! ( \"driver_cli.rs\" )"}, {"id": "lv-he57b8c61e68c", "path": "crates/fractal-delivery/src/store/unread_mail.rs", "kind": "unresolved-sql-sink", "fingerprint": "4a02535b13973c4e087573f1ccce6e0054e559e8370f41eaff2d996544fdc511", "owner": "fractal_delivery::store::unread_mail", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 76, "excerpt": "[ before ] , | row | row . get ( 0 )"}, {"id": "lv-he57c3e1c3fb2", "path": "crates/smallclaims/src/store/checkpoint_trim/capture_lifetime_tests.rs", "kind": "sql-template", "fingerprint": "a903570fb3e92313427f3657d79b1c5399a0e395dd49ccbb6ad0e56c2d20fc83", "owner": "smallclaims::store::checkpoint_trim::capture_lifetime_tests", "todo": "stage-generation-and-keyed-reclamation", "disposition": "allow-for-now", "line_hint": 163, "excerpt": "INSERT INTO checkpoints(id, cut_unix_ms, state, seal_rowid, updated_at_unix_ms) VALUES (?1, ?2, 'verified', ?3, ?2)"}, + {"id": "lv-he57da1ef4545", "path": "crates/fractal-store/src/store.rs", "kind": "sql-template", "fingerprint": "ef4f2b7be00c4aa1c0e24d9b90402b589f761f06f372bbf32b0c98e02e4be8a1", "owner": "fractal_store::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 17230, "excerpt": "UPDATE projection_health SET status='indeterminate',error_code='operation-conflict',error_message=?1 WHERE aggregate=?2"}, {"id": "lv-he5957c758461", "path": "crates/smallclaims/src/sqlite/writer_queue/tests.rs", "kind": "sql-template", "fingerprint": "71f5d5423100ec2c92b15b2e0ab52a4b5fdff952a65c8ae40607bcf2bd19ee10", "owner": "smallclaims::sqlite::writer_queue::tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 166, "excerpt": "INSERT INTO values_ VALUES(1)"}, {"id": "lv-he59d5e1a74a6", "path": "crates/fractal-store/src/store/run_eta_tests.rs", "kind": "sql-template", "fingerprint": "ed87fe0214a6390f40888018d31b3a933ee28176617daf768468decb71590168", "owner": "fractal_store::store::run_eta_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 84, "excerpt": "INSERT INTO mission_runs(id,current_generation_id,phase,status) VALUES(?1,'generation','terminal','completed')"}, {"id": "lv-he5b857500820", "path": "crates/smallclaims/src/store/projection_busy_tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "f10e20728e77f302935d7b4f1031c77dca8743a680305ee8cdf2f818e584bbb5", "owner": "smallclaims::store::projection_busy_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 318, "excerpt": "\"ROLLBACK\""}, @@ -4900,6 +4902,7 @@ {"id": "lv-hf0974ef66e69", "path": "crates/smallclaims/src/store/principals.rs", "kind": "unresolved-sql-sink", "fingerprint": "fca27624a99881cb812d2f482979708658d687b456b87ffec99ab9acc7c172a8", "owner": "smallclaims::store::principals", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 772, "excerpt": "[ claim_id ] , | row | { Ok ( verdict_from ( & row . get : : < _ , String > ( 0 ) ? , row . get ( 1 ) ? ) ) }"}, {"id": "lv-hf09bc1c4e6af", "path": "crates/smallclaims/src/ivm/install.rs", "kind": "unresolved-sql-sink", "fingerprint": "1cdfa254e19b78a4d6e7e73f2358f7376b62e5e71f836a583c9cdb8c2b0794b3", "owner": "smallclaims::ivm::install", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 911, "excerpt": "\"RELEASE ivm_install_page\""}, {"id": "lv-hf0b1a3c8593a", "path": "crates/smallclaims/src/store/fleet_residue_page_tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "e56ba14f3c17963563604362bf9a2c8ed1b09bd1cd41220868ea016f85b8dec7", "owner": "smallclaims::store::fleet_residue_page_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 107, "excerpt": "& format ! ( \"EXPLAIN QUERY PLAN {RESIDUE_PAGE_QUERY}\" )"}, + {"id": "lv-hf0bbe1f00bc2", "path": "crates/fractal-store/src/store/operation_conflict_tests.rs", "kind": "sql-template", "fingerprint": "94dedd5db98c1e80254ce79345f6724cee48d90b3acfe79f248d3adebe5f661a", "owner": "fractal_store::store::operation_conflict_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 228, "excerpt": "UPDATE projection_health SET status='stale' WHERE aggregate='graph'"}, {"id": "lv-hf0c8dec17096", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "04da194219fdf11d1582ecd207ce8407c1186c3555cb10ea8cfd7698be780b1e", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 22244, "excerpt": "& canonical_sql ( \"SELECT body FROM claims WHERE subject=?1 AND kind='loop.state' ORDER BY CANONICAL_DESC(claims) LIMIT 1\" ) , [ & subject ] , | row | row . get : : < _ , String > ( 0 ) ,"}, {"id": "lv-hf0d3cfe78aeb", "path": "crates/fractal-store/src/store/message_sent.rs", "kind": "unresolved-sql-sink", "fingerprint": "57470a573d498940163f4ec29867ced9d7754875cf05eca107b231109921302a", "owner": "fractal_store::store::message_sent", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 160, "excerpt": "& graph_sent_claim ( ) , [ & input . subject ] , claim_from_row ,"}, {"id": "lv-hf0d539537e15", "path": "crates/smallclaims/src/store/sealed_view_cache_tests/index_cost_tests.rs", "kind": "sql-template", "fingerprint": "2b19bbaefbb45b1f86a6cfbdd9fbabfa53fc1bce0f991d252432f0d1af4e77ec", "owner": "smallclaims::store::sealed_view_cache_tests::index_cost_tests", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 42, "excerpt": "WITH RECURSIVE counters(n) AS ( VALUES(1) UNION ALL SELECT n+1 FROM counters WHERE n Date: Wed, 11 Nov 2026 01:29:40 +0100 Subject: [PATCH 4/6] fix(store): persist bounded conflict discovery and recovery visibility agent-identity: unknown agent-persona: unknown agent-supervisor: unavailable agent-tool: unknown agent-runtime: unknown agent-model: unknown --- crates/fractal-store/src/store.rs | 23 ++-- .../src/store/operation_conflict_tests.rs | 108 +++++++++++++++- crates/smallclaims/src/store.rs | 116 ++++++++++++++---- docs/st3/large-value-sites.json | 5 +- docs/st3/large-values.md | 8 ++ ...onflict-checkpoint-exclusion-20261111.json | 11 ++ 6 files changed, 233 insertions(+), 38 deletions(-) create mode 100644 release-notes/operation-conflict-checkpoint-exclusion-20261111.json diff --git a/crates/fractal-store/src/store.rs b/crates/fractal-store/src/store.rs index 00b23d5c39b..f1ba8442b16 100644 --- a/crates/fractal-store/src/store.rs +++ b/crates/fractal-store/src/store.rs @@ -17278,13 +17278,8 @@ pub fn try_project_all_simple_replication_tx( // explicit: receipt must not repeatedly strand unrelated claims behind an unhealthy graph. fn defer_replay_for_operation_conflict(tx: &Transaction<'_>, reason: &str) -> Result { let conflict: bool = tx.query_row( - "SELECT EXISTS(SELECT 1 FROM operations WHERE state='conflict') - OR EXISTS(SELECT 1 FROM claims incoming INDEXED BY claims_operation_index - JOIN claims other INDEXED BY claims_operation_index - ON json_extract(other.body,'$._operation.id')=json_extract(incoming.body,'$._operation.id') - WHERE json_extract(incoming.body,'$._operation.id') IS NOT NULL - AND json_extract(other.body,'$._operation.request_digest')<>json_extract(incoming.body,'$._operation.request_digest') - AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=incoming.id OR id=other.id))", + "SELECT EXISTS(SELECT 1 FROM operations INDEXED BY operations_conflict_index WHERE state='conflict') + OR EXISTS(SELECT 1 FROM projection_health WHERE aggregate GLOB 'projection:operation-conflict-fence:*')", [], |row| row.get(0), ).map_err(internal)?; if conflict { @@ -17989,6 +17984,20 @@ pub fn try_project_simple_replication_tx( None => holds::stage("rebuild-planning", || rebuild_planning_tx(transaction)) .map_err(|error| internal(error).with_detail("projection_stage", "planning-rebuild"))?, } + // A later receipt must not implicitly repair a set excluded by an earlier chunk. + let mut excluded = transaction.prepare( + "SELECT aggregate FROM projection_health WHERE status='stale' + AND error_code='operation-conflict-no-bounded-refold' + AND aggregate GLOB 'projection:operation-conflict:*'", + ).map_err(internal)?; + for key in excluded.query_map([], |row| row.get::<_, String>(0)).map_err(internal)? { + let key = key.map_err(internal)?; + if let Some(identity) = key.strip_prefix("projection:operation-conflict:") + && let Ok((_, subject)) = serde_json::from_str::<(String, String)>(identity) { + unsupported_conflict_subjects.insert(subject); + } + } + drop(excluded); owned_sets::project_excluding_tx(transaction, &unsupported_conflict_subjects) .map_err(|error| error.with_detail("projection_stage", "owned-set-project"))?; // Prefer a degraded subject's identifiers over an unrelated later event-only conflict. diff --git a/crates/fractal-store/src/store/operation_conflict_tests.rs b/crates/fractal-store/src/store/operation_conflict_tests.rs index 0514d9491f2..47ae5f34491 100644 --- a/crates/fractal-store/src/store/operation_conflict_tests.rs +++ b/crates/fractal-store/src/store/operation_conflict_tests.rs @@ -150,10 +150,34 @@ fn operation_conflict_refuses_a_guard_requested_full_replay() { tx.commit().unwrap(); } FULL_REPLAYS.with(|count| count.set(0)); - assert!(!store.project_replication_backlog().unwrap()); + assert!(store.project_replication_backlog().unwrap()); assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); assert!(store.replication_status(false, None, &[]).unwrap().unhealthy.iter().any(|row| - row.error_code.as_deref() == Some("operation-conflict-projection-degraded"))); + row.error_code.as_deref() == Some("operation-conflict-replay-deferred"))); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + append_claim_record_tx(&tx, "peer", "exec/after-checkpoint-conflict", "intent.desired", None, + &json!({"subject":"exec/after-checkpoint-conflict","kind":"exec","desired":{}}), + &[], None).unwrap(); + tx.commit().unwrap(); + } + assert!(store.project_replication_backlog().unwrap()); + assert!(store.readers.get().query_row( + "SELECT EXISTS(SELECT 1 FROM desired WHERE subject='exec/after-checkpoint-conflict')", + [], |row| row.get::<_, bool>(0), + ).unwrap()); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + assert!(store.replication_projection_needs_recovery().unwrap()); + let health = store.replication_status(false, None, &[]).unwrap(); + assert!(health.unhealthy.iter().any(|row| row.aggregate == "graph" && row.status == "stale")); + assert!(health.unhealthy.iter().any(|row| row.aggregate == "projection:deferred-replay")); + let before_repair = (operation_rows(&store.readers.get()), graph_digest_of(&store)); + store.replay_replication_graph().unwrap(); + assert!(!store.replication_projection_needs_recovery().unwrap()); + assert!(!store.replication_status(false, None, &[]).unwrap().unhealthy.iter() + .any(|row| row.aggregate == "projection:deferred-replay")); + assert_eq!(before_repair, (operation_rows(&store.readers.get()), graph_digest_of(&store))); } } @@ -228,7 +252,29 @@ fn operation_conflict_in_second_receipt_chunk_fences_first_chunk_fallback() { /* sqlite-large-value: allow-for-now lv-hf0bbe1f00bc2 owner=fractal_store::store::operation_conflict_tests TODO=bound-or-reference-values-and-reclamation */ "UPDATE projection_health SET status='stale' WHERE aggregate='graph'", [], ).unwrap(); FULL_REPLAYS.with(|count| count.set(0)); - target.project_replication_backlog().unwrap(); + let scanned_target = target.index().unwrap(); + let injected = std::cell::Cell::new(false); + target.project_replication_backlog_with_yield(|| { + if injected.replace(true) { return; } + let mut connection = target.connection.write(); + let tx = connection.transaction().unwrap(); + for digest in ["a", "b"] { + append_claim_record_tx(&tx, "peer", "custom/after-target", "custom.test.observed", None, + &json!({"fields":{},"_operation":{"id":"op/after-target","request_digest":digest}}), + &[], None).unwrap(); + } + tx.commit().unwrap(); + }).unwrap(); + let watermark = || target.readers.get().query_row( + "SELECT last_good_store_index FROM projection_health WHERE aggregate='projection:operation-conflict-scan'", + [], |row| row.get::<_, u64>(0), + ).unwrap(); + let after_target_discovered = || target.readers.get().query_row( + "SELECT EXISTS(SELECT 1 FROM projection_health WHERE aggregate='projection:operation-conflict-fence:op/after-target')", + [], |row| row.get::<_, bool>(0), + ).unwrap(); + assert_eq!(watermark(), scanned_target); + assert!(!after_target_discovered()); assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); assert!(target.replication_status(false, None, &[]).unwrap().unhealthy.iter().any(|row| row.error_code.as_deref() == Some("operation-conflict-replay-deferred"))); @@ -241,6 +287,8 @@ fn operation_conflict_in_second_receipt_chunk_fences_first_chunk_fallback() { tx.commit().unwrap(); } target.project_replication_backlog().unwrap(); + assert_eq!(watermark(), target.index().unwrap()); + assert!(after_target_discovered()); assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); assert!(target.readers.get().query_row( "SELECT EXISTS(SELECT 1 FROM desired WHERE subject='exec/after-deferred')", @@ -282,3 +330,57 @@ fn operation_conflict_refold_failure_restores_every_prior_desired_column() { assert!(store.replication_status(false, None, &[]).unwrap().unhealthy.iter().any(|row| row.status == "stale" && row.aggregate.contains("op/refold-failure"))); } + +#[test] +fn operation_conflict_owned_set_exclusion_survives_later_receipt_chunk() { + let store = Store::open_memory("fixture").unwrap(); + store.project_replication_backlog().unwrap(); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + let body = json!({"subject":"exec/excluded-member","kind":"exec","desired":{"value":"excluded"}, + "owned_set":"owned-set/excluded"}); + let desired: DesiredSubject = serde_json::from_value(body.clone()).unwrap(); + let member = append_claim_record_tx(&tx, "peer", "exec/excluded-member", "intent.desired", + None, &body, &[], None).unwrap(); + let revision = owned_sets::Revision { + rollout: None, previous: None, + source: owned_sets::Source { repository:"acme/garden".into(), r#ref:"main".into(), sha:"abc".into(), sequence:1 }, + bundle_digest:"bundle".into(), + members:BTreeMap::from([("exec/excluded-member".into(), owned_sets::Member { + manual_rollout:false, one_shot:false, kind:"exec".into(), claim:member.id, + revision:desired_revision(&desired), + })]), + retired:BTreeMap::new(), adoptions:BTreeMap::new(), + }; + for digest in ["a", "b"] { + append_claim_record_tx(&tx, "peer", "owned-set/excluded", "owned-set.revised", None, + &json!({"fields":{"revision":canonical_hash(&revision).unwrap(),"body":revision}, + "_operation":{"id":"op/excluded-set","request_digest":digest}}), &[], None).unwrap(); + } + tx.commit().unwrap(); + } + FULL_REPLAYS.with(|count| count.set(0)); + let absent = || !store.readers.get().query_row( + "SELECT EXISTS(SELECT 1 FROM desired WHERE subject='exec/excluded-member')", + [], |row| row.get::<_, bool>(0), + ).unwrap(); + assert!(store.project_replication_backlog().unwrap()); + assert!(absent()); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + append_claim_record_tx(&tx, "peer", "custom/unrelated-set-event", "custom.test.observed", + None, &json!({"fields":{}}), &[], None).unwrap(); + tx.commit().unwrap(); + } + assert!(store.project_replication_backlog().unwrap()); + assert!(absent(), "an unrelated later chunk must not repair the excluded set"); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + // The set really is projectable: only an explicit repair may apply its member. + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + owned_sets::project_tx(&tx).unwrap(); + tx.commit().unwrap(); + assert!(!absent()); +} diff --git a/crates/smallclaims/src/store.rs b/crates/smallclaims/src/store.rs index bbeffef0c55..1ca097718b9 100644 --- a/crates/smallclaims/src/store.rs +++ b/crates/smallclaims/src/store.rs @@ -8679,6 +8679,7 @@ impl Store { let transaction = connection.transaction()?; self.runtime.replay_from_nothing(&transaction)?; self.runtime.after_projection(&transaction)?; + transaction.execute(/* sqlite-large-value: allow-for-now lv-h362e35c78646 owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "DELETE FROM projection_health WHERE aggregate='projection:deferred-replay'", [])?; transaction.execute( /* sqlite-large-value: allow-for-now lv-h9513a9e75a35 owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate, status, last_good_store_index, updated_at_unix_ms) VALUES ('graph', 'healthy', ?1, ?2) @@ -8793,6 +8794,76 @@ impl Store { } } + /// Discover only the new admitted range, on readers, before lending the projection writer. + /// A first upgrade scans the old log once; each metadata page commits its own watermark. + /// Receipts beyond this pass's fixed target belong to the next pass, never this scan. + fn discover_operation_conflicts(&self, target: u64) -> Result<()> { + let mut scanned: u64 = self.readers.get().query_row( + "SELECT last_good_store_index FROM projection_health WHERE aggregate='projection:operation-conflict-scan'", + [], |row| row.get(0), + ).optional()?.unwrap_or(0); + while scanned < target { + let (through, conflicts) = { + let connection = self.readers.get(); + let tx = connection.unchecked_transaction()?; + let mut page = tx.prepare( + "SELECT store_index,id, + CASE WHEN json_type(body,'$._operation.id')='text' THEN json_extract(body,'$._operation.id') END, + CASE WHEN json_type(body,'$._operation.request_digest')='text' THEN json_extract(body,'$._operation.request_digest') END + FROM claims WHERE store_index>?1 AND store_index<=?2 ORDER BY store_index LIMIT ?3", + )?; + let rows = page.query_map(params![scanned,target,PROJECTION_CHUNK_CLAIMS as i64], |row| + Ok((row.get::<_,u64>(0)?,row.get::<_,String>(1)?,row.get::<_,Option>(2)?,row.get::<_,Option>(3)?)) + )?.collect::,_>>()?; + let through = rows.last().map_or(target, |row| row.0); + let mut conflicts = BTreeMap::new(); + for (_, claim_id, operation_id, digest) in rows { + let (Some(operation_id), Some(digest)) = (operation_id, digest) else { continue; }; + if conflicts.contains_key(&operation_id) { continue; } + if tx.query_row( + "SELECT EXISTS(SELECT 1 FROM projection_health WHERE aggregate=?1)", + [format!("projection:operation-conflict-fence:{operation_id}")], |row| row.get::<_,bool>(0), + )? { continue; } + let previous: Option = tx.query_row( + "SELECT id FROM claims INDEXED BY claims_operation_index + WHERE json_extract(body,'$._operation.id')=?1 + AND json_extract(body,'$._operation.request_digest')<>?2 AND store_index<=?3 + AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=claims.id OR id=?4) + UNION ALL SELECT id FROM checkpoint_claims WHERE operation_id=?1 AND request_digest<>?2 + AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=checkpoint_claims.id OR id=?4) + LIMIT 1", + params![operation_id,digest,target,claim_id], |row| row.get(0), + ).optional()?; + if let Some(previous) = previous { + conflicts.insert(operation_id.clone(), (claim_id,operation_id,Some(previous))); + } + } + (through, conflicts) + }; + // No receipt history is read under this loan: only the bounded page's evidence + // and watermark are committed. The evidence set survives subsequent passes. + let mut connection = self.connection.write(); + let tx = connection.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + for (operation, evidence) in conflicts { + tx.execute( + /* sqlite-large-value: allow-for-now lv-hb7532db3c65d owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) + VALUES (?1,'healthy','operation-conflict-fence',?2,?3) + ON CONFLICT(aggregate) DO UPDATE SET error_message=excluded.error_message", + params![format!("projection:operation-conflict-fence:{operation}"),serde_json::to_string(&evidence)?,now_ms().to_string()], + )?; + } + tx.execute( + /* sqlite-large-value: allow-for-now lv-hc1d2cd2957a7 owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,last_good_store_index,updated_at_unix_ms) + VALUES ('projection:operation-conflict-scan','healthy',?1,?2) + ON CONFLICT(aggregate) DO UPDATE SET last_good_store_index=MAX(last_good_store_index,excluded.last_good_store_index)", + params![through,now_ms().to_string()], + )?; + tx.commit()?; + scanned = through; + } + Ok(()) + } + fn project_replication_backlog_chunks( &self, mut between: impl FnMut(), @@ -8812,6 +8883,7 @@ impl Store { // Finish the backlog observed at entry. New receives can admit more between chunks; // their own projection pass will take those up without keeping this one alive forever. let target = self.index()?; + self.discover_operation_conflicts(target)?; let mut chunked = false; let mut processed = 0_u64; loop { @@ -8952,28 +9024,12 @@ impl Store { // A failed bounded repair must never turn into a whole-log fold merely // because another guard (or a projector error) requested fallback first. let conflict: Option<(String, String, Option)> = transaction.query_row( - "SELECT incoming.id,json_extract(incoming.body,'$._operation.id'), - COALESCE(operations.canonical_claim_id,(SELECT id FROM checkpoint_claims - WHERE operation_id=json_extract(incoming.body,'$._operation.id') - AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=checkpoint_claims.id) - ORDER BY request_digest,id LIMIT 1)) - FROM claims incoming INDEXED BY claims_operation_index LEFT JOIN operations - ON operations.id=json_extract(incoming.body,'$._operation.id') - WHERE json_extract(incoming.body,'$._operation.id') IS NOT NULL - AND (operations.state='conflict' - OR operations.request_digest<>json_extract(incoming.body,'$._operation.request_digest') - OR EXISTS(SELECT 1 FROM claims other INDEXED BY claims_operation_index - WHERE json_extract(other.body,'$._operation.id')=json_extract(incoming.body,'$._operation.id') - AND json_extract(other.body,'$._operation.request_digest')<>json_extract(incoming.body,'$._operation.request_digest') - AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=other.id)) - OR EXISTS(SELECT 1 FROM checkpoint_claims - WHERE operation_id=json_extract(incoming.body,'$._operation.id') - AND request_digest<>json_extract(incoming.body,'$._operation.request_digest') - AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=checkpoint_claims.id))) - AND NOT EXISTS(SELECT 1 FROM projection_digest_repaired_claims WHERE id=incoming.id) - ORDER BY incoming.store_index DESC LIMIT 1", - [], |row| Ok((row.get(0)?,row.get(1)?,row.get(2)?)), - ).optional().map_err(internal)?; + "SELECT error_message FROM projection_health + WHERE aggregate GLOB 'projection:operation-conflict-fence:*' + UNION ALL SELECT json_array(canonical_claim_id,id,canonical_claim_id) + FROM operations INDEXED BY operations_conflict_index WHERE state='conflict' LIMIT 1", + [], |row| row.get::<_,String>(0), + ).optional()?.map(|message| serde_json::from_str(&message)).transpose()?; if let Some((claim_id, operation_id, previous_claim_id)) = conflict { let error = FractalError::new("operation-conflict-projection-degraded", json!({"operation_id":operation_id,"claim_id":claim_id, @@ -8986,6 +9042,11 @@ impl Store { self.log_projection_failure(&error, phase, frontier, target, &mut log_staged); return Err(error); } + // Replay is not target-bounded. A later admission has not been scanned: + // defer until the next pass discovers that newly admitted range. + if current_index_tx(&transaction)? != target { + return Err(FractalError::new("projection-target-changed", "retry replay with the newly admitted range")); + } log_staged(&full_replay_log_line(phase, &reason, frontier, target)); stages.enter("replay"); #[cfg(any(test, feature = "test-support"))] @@ -9033,10 +9094,11 @@ impl Store { transaction.rollback()?; } else { transaction.execute( - /* sqlite-large-value: allow-for-now lv-h50cb263d1821 owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate, status, last_good_store_index, updated_at_unix_ms) - VALUES ('graph', 'healthy', ?1, ?2) - ON CONFLICT(aggregate) DO UPDATE SET status='healthy', last_good_store_index=excluded.last_good_store_index, - error_code=NULL, error_message=NULL, updated_at_unix_ms=excluded.updated_at_unix_ms", + /* sqlite-large-value: allow-for-now lv-hcc82e02fe20f owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate, status, last_good_store_index, updated_at_unix_ms) + VALUES ('graph', CASE WHEN EXISTS(SELECT 1 FROM projection_health WHERE aggregate='projection:deferred-replay') THEN 'stale' ELSE 'healthy' END, ?1, ?2) + ON CONFLICT(aggregate) DO UPDATE SET status=excluded.status, last_good_store_index=excluded.last_good_store_index, + error_code=CASE WHEN excluded.status='stale' THEN 'operation-conflict-replay-deferred' END, + error_message=CASE WHEN excluded.status='stale' THEN 'explicit projection repair required' END, updated_at_unix_ms=excluded.updated_at_unix_ms", params![through, now_ms().to_string()], )?; stages.enter("commit"); @@ -9146,7 +9208,7 @@ impl Store { let connection = self.readers.get(); let status: Option = connection .query_row( - "SELECT status FROM projection_health WHERE aggregate='graph'", + "SELECT CASE WHEN EXISTS(SELECT 1 FROM projection_health WHERE aggregate='projection:deferred-replay') THEN 'stale' ELSE status END FROM projection_health WHERE aggregate='graph'", [], |row| row.get(0), ) diff --git a/docs/st3/large-value-sites.json b/docs/st3/large-value-sites.json index 0e9f52a2675..3a324445f0a 100644 --- a/docs/st3/large-value-sites.json +++ b/docs/st3/large-value-sites.json @@ -1719,6 +1719,7 @@ {"id": "lv-h35d0067b8c2e", "path": "crates/fractal-store/src/store/agent_messages.rs", "kind": "sql-template", "fingerprint": "5147cedfcf87a1681c4f9f229c629e418c030fd1f552bca95fcdd0ff7ac12495", "owner": "fractal_store::store::agent_messages", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 249, "excerpt": "DELETE FROM agent_message_sends WHERE subject=?1"}, {"id": "lv-h35d80536dc09", "path": "crates/smallclaims/src/store/principals/migration.rs", "kind": "unresolved-sql-sink", "fingerprint": "c8b9bfcfa45a834a03c52aab4a2eed2f9d8b5b23448456e9f38bbed6ddcbbfbf", "owner": "smallclaims::store::principals::migration", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 312, "excerpt": "& format ! ( \"SELECT rowid,claim_id,signature FROM {table} WHERE rowid>?1 AND rowid<=?2 ORDER BY rowid LIMIT ?3\" )"}, {"id": "lv-h361bb9e86be3", "path": "crates/smallclaims/src/store/membership_query_tests.rs", "kind": "sql-template", "fingerprint": "123c9e6a289acc6c2f2a532673c783de9df5f87aa866159531bacc1652d889c5", "owner": "smallclaims::store::membership_query_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 284, "excerpt": "INSERT INTO meta VALUES('fleet_anchor_key','anchor-key')"}, + {"id": "lv-h362e35c78646", "path": "crates/smallclaims/src/store.rs", "kind": "sql-template", "fingerprint": "acf3734665e2cf4d72bb3957ca49978425d3755e34db3736637bb84502a937ee", "owner": "smallclaims::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 8682, "excerpt": "DELETE FROM projection_health WHERE aggregate='projection:deferred-replay'"}, {"id": "lv-h3636893be705", "path": "crates/fractal-store/src/private_notes.rs", "kind": "sql-template", "fingerprint": "c16b933977b08668e612f5860fc561e8bf9756720913c9c7a0b25a3c676a5507", "owner": "fractal_store::private_notes", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 223, "excerpt": "INSERT INTO notes_operations(operation, uri, request_digest, generation, old_revision, new_revision) VALUES(?1,?2,?3,?4,?5,'unprepared')"}, {"id": "lv-h3644dd9ee6f6", "path": "crates/fractal-store/src/store/message_marks.rs", "kind": "sql-template", "fingerprint": "a5d64e94a3c277dd9f6d636211a9448d6c9c69f3b1e2260078a6614a556f20ff", "owner": "fractal_store::store::message_marks", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 192, "excerpt": "INSERT OR IGNORE INTO local_message_marks(message, kind, consumer, recorded_at_unix_ms, receipt) SELECT ?1, ?2, ?3, ?4, ?5 WHERE EXISTS(SELECT 1 FROM message_index WHERE subject=?1)"}, {"id": "lv-h365c0ca698ad", "path": "crates/smallclaims/src/ivm/installed.rs", "kind": "unresolved-sql-sink", "fingerprint": "b7aa2fa1245e1f133d6afe28cac4c0c959263ef0310413dc18633e8c436e1ed2", "owner": "smallclaims::ivm::installed", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 103, "excerpt": "\"RELEASE ivm_installed_bridge\""}, @@ -2180,7 +2181,6 @@ {"id": "lv-h50b057a680ab", "path": "crates/smallclaims/src/store/repair_ledger.rs", "kind": "sql-template", "fingerprint": "f92c290b28487ac3c1541520111d9acb0662e65c62379184e2179d29c6cd88a9", "owner": "smallclaims::store::repair_ledger", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 59, "excerpt": "INSERT OR IGNORE INTO meta(key, value) VALUES (?1, '1')"}, {"id": "lv-h50b120964acf", "path": "crates/fractal-store/src/store/custom.rs", "kind": "sql-template", "fingerprint": "712a4c8d297136a90432f136b240c8c6644d547ab961a3e111cb85795e0d48db", "owner": "fractal_store::store::custom", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 614, "excerpt": "INSERT OR REPLACE INTO custom_sources VALUES(?1,'','','','unavailable',?2,NULL,NULL,0,'0')"}, {"id": "lv-h50c19d221504", "path": "crates/fractal-store/src/store/attention_snapshot.rs", "kind": "unresolved-sql-sink", "fingerprint": "a62379ab131f4f1e4fb082cc2fd799519c47de22cdb7aec7c5c4399d47868fd9", "owner": "fractal_store::store::attention_snapshot", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 1860, "excerpt": "[ smallclaims : : store : : operation_id_for_key ( key ) ] , | row | { row . get : : < _ , Option < String > > ( 0 ) }"}, - {"id": "lv-h50cb263d1821", "path": "crates/smallclaims/src/store.rs", "kind": "sql-template", "fingerprint": "c103bf627448939d25fe033f56fc648b85140684aa50c4ee076629cc9940c2e6", "owner": "smallclaims::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 8935, "excerpt": "INSERT INTO projection_health(aggregate, status, last_good_store_index, updated_at_unix_ms) VALUES ('graph', 'healthy', ?1, ?2) ON CONFLICT(aggregate) DO UPDATE SET status='healthy', last_good_store_i"}, {"id": "lv-h50daecb5c0e9", "path": "crates/fractal-store/src/store/custom.rs", "kind": "unresolved-sql-sink", "fingerprint": "7578a20027cf0b16ba65d57bb5b4719f7190c81a0e25b582a8c49b0ee02524d0", "owner": "fractal_store::store::custom", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 162, "excerpt": "& format ! ( \"DROP {kind} {name}\" )"}, {"id": "lv-h50e7e097ad49", "path": "crates/smallclaims/src/ivm/install.rs", "kind": "sql-template", "fingerprint": "5232721a828eac77a4fd1f84ef398ba70d6db095d986e3f2cefc4fa933b720f7", "owner": "smallclaims::ivm::install", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 348, "excerpt": "UPDATE ivm_install_sources SET fingerprint=?2,epoch=?3,available=1 WHERE name=?1"}, {"id": "lv-h510ad1338598", "path": "crates/smallclaims/src/store/repair_operation_tests.rs", "kind": "sql-template", "fingerprint": "9e02c26bc8a76d2db0acf9d61d03ab735ae9fcf031277b5b3b1c7c4cb8d634a7", "owner": "smallclaims::store::repair_operation_tests", "todo": "stage-generation-and-keyed-reclamation", "disposition": "allow-for-now", "line_hint": 915, "excerpt": "DELETE FROM checkpoint_claims WHERE id='cp/late'"}, @@ -3899,6 +3899,7 @@ {"id": "lv-hb721d709d069", "path": "crates/fractal/src/main_tests.rs", "kind": "included-source", "fingerprint": "3de9fb3a1e47835bc55956730f8f5d75a6c7d967e8609f93a898683c3dfc671b", "owner": "fractal::main_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 4426, "excerpt": "include_str ! ( \"../../../docs/st3/client-v0/fixtures/events.json\" )"}, {"id": "lv-hb7260ab5b0c9", "path": "crates/smallclaims/src/ivm/install.rs", "kind": "unresolved-sql-sink", "fingerprint": "65994db488b6c5b347f755d8fa7f53e0d8cd2e6b44a600c978e5206d11dc2c1a", "owner": "smallclaims::ivm::install", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 277, "excerpt": "prepared : : SCHEMA"}, {"id": "lv-hb73bd4f2f38c", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "700d14798fd37c5da5b5389e3903a51a34c8c957866a0dd94482dfcc246b196b", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 10143, "excerpt": "& canonical_sql ( \"SELECT json_extract(body, '$.fields.session_id') FROM claims WHERE subject=?1 AND kind='harness.session-file' AND json_extract(body, '$.fields.incarnation_id')=?2 AND json_extract(b"}, + {"id": "lv-hb7532db3c65d", "path": "crates/smallclaims/src/store.rs", "kind": "sql-template", "fingerprint": "a58f11e89b1304cbf16f7187e2d185f0a7c1629a3dc2b12400497b21a9135b81", "owner": "smallclaims::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 8849, "excerpt": "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) VALUES (?1,'healthy','operation-conflict-fence',?2,?3) ON CONFLICT(aggregate) DO UPDATE SET error_message=ex"}, {"id": "lv-hb757b662d7d4", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "253b8beb3c76640213eee0502dead3be25e07331aa1a8de8dd5a004e807c0270", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 15269, "excerpt": "params ! [ step , crate : : mission : : AFTER_RUN_STEP , admitted_at_unix_ms . to_string ( ) ] ,"}, {"id": "lv-hb7696ce642f6", "path": "crates/fractal-store/src/store/tests.rs", "kind": "sql-template", "fingerprint": "ce26828499f2a677143d89d2d20702a5ab8c10d50e044eeb571ce5fca2e1d3d9", "owner": "fractal_store::store::tests", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 5587, "excerpt": "PRAGMA foreign_keys=OFF; INSERT INTO claims SELECT * FROM dropped_claims; INSERT INTO desired SELECT * FROM dropped_desired; PRAGMA foreign_keys=ON;"}, {"id": "lv-hb770cadb12c6", "path": "crates/fractal-reconcile/src/reconcile.rs", "kind": "sql-template", "fingerprint": "1d61a2b78fe7cd16c66608b4c35960e465f0be7d0f9871b0ac0fc934ae819bcf", "owner": "fractal_reconcile::reconcile", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 47339, "excerpt": "DROP TRIGGER fail_session_start"}, @@ -4110,6 +4111,7 @@ {"id": "lv-hc1ac3d35bbf4", "path": "crates/fractal-store/src/store/client_messages.rs", "kind": "sql-template", "fingerprint": "b00efd86cc227528fbfa988943c50c7bc8f88c73ef6908072ddf7ed7cac1bc74", "owner": "fractal_store::store::client_messages", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 1523, "excerpt": "DELETE FROM meta WHERE key=?1"}, {"id": "lv-hc1b4b7df7d60", "path": "crates/smallclaims/src/sqlite.rs", "kind": "unresolved-sql-sink", "fingerprint": "ce922519a3c3ecaf9b0986c2449c7680895c15f4b0e9818e994e14a4d28b6aaf", "owner": "smallclaims::sqlite", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 995, "excerpt": "transaction"}, {"id": "lv-hc1c6ea0cb228", "path": "crates/fractal-store/src/store/terminal_gate_evidence/native_source_tests_shared.rs", "kind": "sql-template", "fingerprint": "740d295c2a21b85f7e9f3cfc6e07c1cddb61c6d0698f1d79e9cb732ef1e0c259", "owner": "fractal_store::store::terminal_gate_evidence::native_source_tests_shared", "todo": "chunk-or-reference-exact-wire-values", "disposition": "allow-for-now", "line_hint": 87, "excerpt": "WITH RECURSIVE pages(n) AS (SELECT 0 UNION ALL SELECT n+1 FROM pages WHERE n<(?1-1)/64) INSERT INTO batches(id,origin,replica_sequence,hash,accepted_at_unix_ms) SELECT 'foreign-page/'||n,'node',1000+n"}, + {"id": "lv-hc1d2cd2957a7", "path": "crates/smallclaims/src/store.rs", "kind": "sql-template", "fingerprint": "2757c79a329d4159d2c1ee47a342658ebf97772333f56b9264a4d35d847d72cb", "owner": "smallclaims::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 8856, "excerpt": "INSERT INTO projection_health(aggregate,status,last_good_store_index,updated_at_unix_ms) VALUES ('projection:operation-conflict-scan','healthy',?1,?2) ON CONFLICT(aggregate) DO UPDATE SET last_good_st"}, {"id": "lv-hc1db65df6dce", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "f364d8ca6168ffa8cad7d686c9cbe491d919c3fcdda7ba967af6912aaffb62ae", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 6701, "excerpt": "& format ! ( \"{} LIMIT 1\" , newest_claims_of_kind_query ( \"claims.origin\" , \"runtime.observed\" ) )"}, {"id": "lv-hc1e68820d94d", "path": "crates/smallclaims/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "789b44b9990c7d88da3542aa5c6b25148409a57d64b88f00362513cda78a031f", "owner": "smallclaims::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 1188, "excerpt": "\"COMMIT\""}, {"id": "lv-hc1efad5f0359", "path": "crates/fractal-store/src/store/resources_preparation_tests.rs", "kind": "sql-template", "fingerprint": "fd3f455464159e17ece91a42ad4675d290a1d23ae78f3310dfac39c06c99e3ec", "owner": "fractal_store::store::resources_preparation_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 497, "excerpt": "UPDATE claim_positions SET position=CASE WHEN claim_id=?1 THEN 9 ELSE 1 END WHERE claim_id IN (?1,?2)"}, @@ -4280,6 +4282,7 @@ {"id": "lv-hcc772813cbfc", "path": "crates/fractal-store/src/store/custom.rs", "kind": "sql-template", "fingerprint": "bf8b7ce251e5ac1dbc9e7dd4af2c85b9013e1812fe61bf53980c6786ab71a4e9", "owner": "fractal_store::store::custom", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 186, "excerpt": "DELETE FROM custom_registrations; DELETE FROM custom_sources; DELETE FROM custom_dependencies; DELETE FROM local_custom_dirty; INSERT INTO local_custom_dirty SELECT DISTINCT subject FROM claims WHERE "}, {"id": "lv-hcc7d9c00ac1a", "path": "crates/smallclaims/src/store/receive_writer_tests.rs", "kind": "sql-template", "fingerprint": "eaa3febaed206ae00cf3c6f5d2bdf1e3d1d53d1762ddbea3121235af1bf71b04", "owner": "smallclaims::store::receive_writer_tests", "todo": "chunk-or-reference-exact-wire-values", "disposition": "allow-for-now", "line_hint": 1175, "excerpt": "CREATE TEMP TRIGGER signature_gate BEFORE INSERT ON replica_envelope_signatures BEGIN SELECT signature_gate(); END;"}, {"id": "lv-hcc7ebd85e8b8", "path": "crates/fractal-store/src/store.rs", "kind": "sql-template", "fingerprint": "41e17ed85cd509f4ae857aa4545a18a7f1f4da1c41f8b0fefc95594eb545bb1b", "owner": "fractal_store::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 17866, "excerpt": "INSERT INTO desired(subject, kind, revision, claim_id, body, member, owner_run, owner_generation, owner_step) VALUES (?1, ?2, ?3, ?4, ?5, ?6, CASE WHEN ?1 LIKE 'resource/uri/%' THEN NULL ELSE ?7 END, "}, + {"id": "lv-hcc82e02fe20f", "path": "crates/smallclaims/src/store.rs", "kind": "sql-template", "fingerprint": "e3bc5a2caeeff4bbf1679781d610618eaba61c3f7d2b5a7f19ec7ccbf9e1688f", "owner": "smallclaims::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 9097, "excerpt": "INSERT INTO projection_health(aggregate, status, last_good_store_index, updated_at_unix_ms) VALUES ('graph', CASE WHEN EXISTS(SELECT 1 FROM projection_health WHERE aggregate='projection:deferred-repla"}, {"id": "lv-hcc836a720503", "path": "crates/fractal-store/src/store.rs", "kind": "sql-template", "fingerprint": "4c240a001a3552ccccf487d62f3dc3fcef0d2eb3f0771b65566041dbfea659fc", "owner": "fractal_store::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 7247, "excerpt": "INSERT INTO capabilities(secret_hash, kind, subject, incarnation_id, expires_at_unix_ms, used) VALUES (?1, ?2, ?3, ?4, ?5, 0)"}, {"id": "lv-hcc86b5e9b027", "path": "crates/fractal-store/src/store/local_trim_tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "c32b884bb11aa9d33ad9fa5b6cbd7b18be6b5eec40e5397aa0319a5e4069ab0d", "owner": "fractal_store::store::local_trim_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 1889, "excerpt": "& sql"}, {"id": "lv-hccb25d68371e", "path": "crates/smallclaims/src/store/principals.rs", "kind": "unresolved-sql-sink", "fingerprint": "02820da43529d8e32af360db422cb8a9b2f70e20b0ffb83a1a5c770f7991156c", "owner": "smallclaims::store::principals", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 210, "excerpt": "[ claim_id ] , | row | row . get : : < _ , String > ( 0 )"}, diff --git a/docs/st3/large-values.md b/docs/st3/large-values.md index ecb7c0c05b0..17b9a9f0585 100644 --- a/docs/st3/large-values.md +++ b/docs/st3/large-values.md @@ -623,3 +623,11 @@ The atomic-degradation correction also owns `lv-ha8b524108446` and advances, and the latter clears only the successfully refolded subject's stale diagnostic. Both retain metadata, never claim bodies, with the same bounding debt. `lv-hf0bbe1f00bc2` is the second-chunk stale-health regression fixture. + +`smallclaims::store` owns `lv-hb7532db3c65d` (full operation/claim identity evidence), +`lv-hc1d2cd2957a7` (the incremental reader scan watermark), `lv-hcc82e02fe20f` +(graph health that retains deferred-repair visibility), and `lv-h362e35c78646` +(clearing that deferred row only after explicit full repair succeeds). +Evidence contains identifiers, not claim bodies; identifier representation remains +the same bounding debt. Discovery reads only the newly admitted target range on +readers; writer metadata pages are capped at the projection chunk size. diff --git a/release-notes/operation-conflict-checkpoint-exclusion-20261111.json b/release-notes/operation-conflict-checkpoint-exclusion-20261111.json new file mode 100644 index 00000000000..cebf430571f --- /dev/null +++ b/release-notes/operation-conflict-checkpoint-exclusion-20261111.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "summary": "Keep checkpoint conflicts moving through receipt and preserve owned-set exclusions across chunks.", + "replay": {"status": "changed", "detail": "Reader-side conflict discovery includes checkpoint receipts and is bounded by the pass target. A durable watermark and conflict evidence set avoid rescanning the log on each receipt; metadata commits in bounded pages. Writer-side replay fences consult indexed evidence only. The first upgraded pass scans historical claims once, outside the writer."}, + "schema": {"status": "none", "detail": "No schema changes."}, + "rules": {"status": "none", "detail": "Canonical registry and graph fold rules are unchanged."}, + "compatibility": {"status": "changed", "detail": "Each receipt loads durable degraded owned-set identities before applying set members, so a later unrelated chunk cannot silently repair an excluded set."}, + "manual": {"status": "none", "detail": "Explicit repair remains required for excluded sets and deferred replay."}, + "recovery": {"status": "changed", "detail": "Checkpoint-conflict receipt no longer remains stuck behind the no-replay fence; owned-set degradation persists until explicit repair. Deferred replay keeps graph health stale and recovery required even as the frontier advances. A successful explicit replay_replication_graph clears the deferred row."}, + "downtime": {"status": "unknown", "detail": "No production timing measurement is claimed."} +} From 004e3e449edf97be4a27ea37541985edcee22678 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:42:17 +0100 Subject: [PATCH 5/6] fix(store): invalidate conflict proof on checkpoint mutation agent-identity: unknown agent-persona: unknown agent-supervisor: unavailable agent-tool: unknown agent-runtime: unknown agent-model: unknown --- crates/fractal-store/src/store.rs | 15 ++--- .../src/store/operation_conflict_tests.rs | 57 +++++++++++++++++++ crates/smallclaims/src/store.rs | 36 +++++++++--- .../src/store/operation_conflict_fence.rs | 52 +++++++++++++++++ docs/st3/large-value-sites.json | 7 ++- docs/st3/large-values.md | 10 +++- ...onflict-discovery-generation-20261111.json | 11 ++++ 7 files changed, 170 insertions(+), 18 deletions(-) create mode 100644 crates/smallclaims/src/store/operation_conflict_fence.rs create mode 100644 release-notes/operation-conflict-discovery-generation-20261111.json diff --git a/crates/fractal-store/src/store.rs b/crates/fractal-store/src/store.rs index f1ba8442b16..21a8b981328 100644 --- a/crates/fractal-store/src/store.rs +++ b/crates/fractal-store/src/store.rs @@ -17276,13 +17276,14 @@ pub fn try_project_all_simple_replication_tx( // A whole-log replay is unsafe while this log contains conflicting operations. Keep the // independent replay requirement visible, but let bounded receipt move past it. Repair is // explicit: receipt must not repeatedly strand unrelated claims behind an unhealthy graph. -fn defer_replay_for_operation_conflict(tx: &Transaction<'_>, reason: &str) -> Result { +fn defer_replay_for_operation_conflict(tx: &Transaction<'_>, through: u64, reason: &str) -> Result { let conflict: bool = tx.query_row( "SELECT EXISTS(SELECT 1 FROM operations INDEXED BY operations_conflict_index WHERE state='conflict') OR EXISTS(SELECT 1 FROM projection_health WHERE aggregate GLOB 'projection:operation-conflict-fence:*')", [], |row| row.get(0), ).map_err(internal)?; - if conflict { + let unproven = !smallclaims::store::operation_conflict_discovery_valid(tx, through).map_err(internal)?; + if conflict || unproven { tx.execute( /* sqlite-large-value: allow-for-now lv-ha8b524108446 owner=fractal_store::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) VALUES ('projection:deferred-replay','stale','operation-conflict-replay-deferred',?1,?2) @@ -17290,7 +17291,7 @@ fn defer_replay_for_operation_conflict(tx: &Transaction<'_>, reason: &str) -> Re params![reason, now_ms().to_string()], ).map_err(internal)?; } - Ok(conflict) + Ok(conflict || unproven) } pub fn try_project_simple_replication_tx( @@ -17308,11 +17309,11 @@ pub fn try_project_simple_replication_tx( .map_err(internal)?; let (status, frontier) = match health { Some(health) => health, - None if defer_replay_for_operation_conflict(transaction, "missing-health")? => ("healthy".to_owned(), 0), + None if defer_replay_for_operation_conflict(transaction, through, "missing-health")? => ("healthy".to_owned(), 0), None => return Ok(replay_needed("missing-health")), }; if status != "healthy" - && !defer_replay_for_operation_conflict(transaction, "unhealthy-projection")? { + && !defer_replay_for_operation_conflict(transaction, through, "unhealthy-projection")? { return Ok(replay_needed("unhealthy-projection")); } if frontier > current_index_tx(transaction).map_err(internal)? { @@ -17496,7 +17497,7 @@ pub fn try_project_simple_replication_tx( continue; } } - if defer_replay_for_operation_conflict(transaction, reason)? { + if defer_replay_for_operation_conflict(transaction, through, reason)? { unsupported_conflict_subjects.insert(claim.subject.clone()); continue; } @@ -17986,7 +17987,7 @@ pub fn try_project_simple_replication_tx( } // A later receipt must not implicitly repair a set excluded by an earlier chunk. let mut excluded = transaction.prepare( - "SELECT aggregate FROM projection_health WHERE status='stale' + "SELECT aggregate FROM projection_health INDEXED BY projection_health_degraded_operation_subject WHERE status='stale' AND error_code='operation-conflict-no-bounded-refold' AND aggregate GLOB 'projection:operation-conflict:*'", ).map_err(internal)?; diff --git a/crates/fractal-store/src/store/operation_conflict_tests.rs b/crates/fractal-store/src/store/operation_conflict_tests.rs index 47ae5f34491..af4a453434b 100644 --- a/crates/fractal-store/src/store/operation_conflict_tests.rs +++ b/crates/fractal-store/src/store/operation_conflict_tests.rs @@ -335,6 +335,17 @@ fn operation_conflict_refold_failure_restores_every_prior_desired_column() { fn operation_conflict_owned_set_exclusion_survives_later_receipt_chunk() { let store = Store::open_memory("fixture").unwrap(); store.project_replication_backlog().unwrap(); + { + let connection = store.readers.get(); + let mut plan = connection.prepare( + "EXPLAIN QUERY PLAN SELECT aggregate FROM projection_health INDEXED BY projection_health_degraded_operation_subject + WHERE status='stale' AND error_code='operation-conflict-no-bounded-refold' + AND aggregate GLOB 'projection:operation-conflict:*'", + ).unwrap(); + let steps = plan.query_map([], |row| row.get::<_, String>(3)).unwrap() + .collect::, _>>().unwrap(); + assert!(steps.iter().any(|step| step.contains("projection_health_degraded_operation_subject"))); + } { let mut connection = store.connection.write(); let tx = connection.transaction().unwrap(); @@ -384,3 +395,49 @@ fn operation_conflict_owned_set_exclusion_survives_later_receipt_chunk() { tx.commit().unwrap(); assert!(!absent()); } + +#[test] +fn operation_conflict_checkpoint_adoption_invalidates_discovery_during_projection() { + let store = Store::open_memory("fixture").unwrap(); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + append_claim_record_tx(&tx, "peer", "custom/late-checkpoint", "custom.test.observed", None, + &json!({"fields":{},"_operation":{"id":"op/late-checkpoint","request_digest":"b"}}), + &[], None).unwrap(); + tx.commit().unwrap(); + } + store.project_replication_backlog().unwrap(); + { + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + for number in 0..=smallclaims::store::PROJECTION_CHUNK_CLAIMS { + append_claim_record_tx(&tx, "peer", "custom/checkpoint-race", "custom.test.observed", None, + &json!({"fields":{"number":number}}), &[], None).unwrap(); + } + tx.commit().unwrap(); + } + let target = store.index().unwrap(); + let adopted = std::cell::Cell::new(false); + FULL_REPLAYS.with(|count| count.set(0)); + store.project_replication_backlog_with_yield(|| { + if adopted.replace(true) { return; } + let mut connection = store.connection.write(); + let tx = connection.transaction().unwrap(); + tx.execute(/* sqlite-large-value: allow-for-now lv-hd5cb81fd9252 owner=fractal_store::store::operation_conflict_tests TODO=stage-generation-and-keyed-reclamation */ "INSERT INTO checkpoint_claims(id,writer,sequence,envelope_hash,subject,kind, + actor,predecessors,operation_id,request_digest,accepted_at_unix_ms,checkpoint) + VALUES ('late-tombstone','peer',1,'envelope','custom/late-checkpoint','custom.test.observed', + NULL,'[]','op/late-checkpoint','a',0,'checkpoint/2026-10-01')", []).unwrap(); + tx.execute(/* sqlite-large-value: allow-for-now lv-h69417713b2cb owner=fractal_store::store::operation_conflict_tests TODO=bound-or-reference-values-and-reclamation */ "UPDATE projection_health SET status='stale' WHERE aggregate='graph'", []).unwrap(); + tx.commit().unwrap(); + }).unwrap(); + assert!(adopted.get()); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); + assert!(!smallclaims::store::operation_conflict_discovery_valid(&store.readers.get(), target).unwrap()); + assert!(store.replication_projection_needs_recovery().unwrap()); + // The next reader pass invalidates the old watermark and discovers the new tombstone, + // even though no new claim index was admitted after the preceding scan. + store.project_replication_backlog().unwrap(); + assert!(smallclaims::store::operation_conflict_discovery_valid(&store.readers.get(), target).unwrap()); + assert_eq!(FULL_REPLAYS.with(std::cell::Cell::get), 0); +} diff --git a/crates/smallclaims/src/store.rs b/crates/smallclaims/src/store.rs index 1ca097718b9..a4547a44be6 100644 --- a/crates/smallclaims/src/store.rs +++ b/crates/smallclaims/src/store.rs @@ -44,6 +44,8 @@ pub use admitted::{AdmittedStore, STORE_DATABASE_FILES}; mod proof; pub mod prepared_envelope; mod replica_record; +mod operation_conflict_fence; +pub use operation_conflict_fence::valid as operation_conflict_discovery_valid; use crate::operation_metadata::OperationMetadata; #[cfg(test)] mod full_inventory_tests; @@ -1249,6 +1251,7 @@ impl Store { })?; projection_digest::initialize(connection, runtime.digest_tables())?; checkpoint_capture_epoch::initialize(connection)?; + operation_conflict_fence::initialize(connection)?; Ok(fresh) } @@ -8798,14 +8801,25 @@ impl Store { /// A first upgrade scans the old log once; each metadata page commits its own watermark. /// Receipts beyond this pass's fixed target belong to the next pass, never this scan. fn discover_operation_conflicts(&self, target: u64) -> Result<()> { - let mut scanned: u64 = self.readers.get().query_row( - "SELECT last_good_store_index FROM projection_health WHERE aggregate='projection:operation-conflict-scan'", - [], |row| row.get(0), - ).optional()?.unwrap_or(0); - while scanned < target { + let (mut scanned, generation, mut unproven) = { + let connection = self.readers.get(); + let tx = connection.unchecked_transaction()?; + let generation = operation_conflict_fence::generation(&tx)?; + let saved: Option<(u64, Option)> = tx.query_row( + "SELECT last_good_store_index,error_message FROM projection_health WHERE aggregate='projection:operation-conflict-scan'", + [], |row| Ok((row.get(0)?,row.get(1)?)), + ).optional()?; + match saved { + Some((scanned, Some(saved_generation))) if saved_generation == generation => (scanned,generation,false), + _ => (0,generation,true), + } + }; + while scanned < target || unproven { + unproven = false; let (through, conflicts) = { let connection = self.readers.get(); let tx = connection.unchecked_transaction()?; + if operation_conflict_fence::generation(&tx)? != generation { return Ok(()); } let mut page = tx.prepare( "SELECT store_index,id, CASE WHEN json_type(body,'$._operation.id')='text' THEN json_extract(body,'$._operation.id') END, @@ -8844,6 +8858,7 @@ impl Store { // and watermark are committed. The evidence set survives subsequent passes. let mut connection = self.connection.write(); let tx = connection.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + if operation_conflict_fence::generation(&tx)? != generation { return Ok(()); } for (operation, evidence) in conflicts { tx.execute( /* sqlite-large-value: allow-for-now lv-hb7532db3c65d owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,error_code,error_message,updated_at_unix_ms) @@ -8853,10 +8868,10 @@ impl Store { )?; } tx.execute( - /* sqlite-large-value: allow-for-now lv-hc1d2cd2957a7 owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,last_good_store_index,updated_at_unix_ms) - VALUES ('projection:operation-conflict-scan','healthy',?1,?2) - ON CONFLICT(aggregate) DO UPDATE SET last_good_store_index=MAX(last_good_store_index,excluded.last_good_store_index)", - params![through,now_ms().to_string()], + /* sqlite-large-value: allow-for-now lv-hb0e4de996f4f owner=smallclaims::store TODO=bound-or-reference-values-and-reclamation */ "INSERT INTO projection_health(aggregate,status,last_good_store_index,updated_at_unix_ms,error_message) + VALUES ('projection:operation-conflict-scan','healthy',?1,?2,?3) + ON CONFLICT(aggregate) DO UPDATE SET last_good_store_index=excluded.last_good_store_index,error_message=excluded.error_message", + params![through,now_ms().to_string(),generation], )?; tx.commit()?; scanned = through; @@ -9042,6 +9057,9 @@ impl Store { self.log_projection_failure(&error, phase, frontier, target, &mut log_staged); return Err(error); } + if !operation_conflict_discovery_valid(&transaction, target)? { + return Err(FractalError::new("operation-conflict-discovery-unproven", "checkpoint or restored history changed during discovery; retry")); + } // Replay is not target-bounded. A later admission has not been scanned: // defer until the next pass discovers that newly admitted range. if current_index_tx(&transaction)? != target { diff --git a/crates/smallclaims/src/store/operation_conflict_fence.rs b/crates/smallclaims/src/store/operation_conflict_fence.rs new file mode 100644 index 00000000000..52dffb9cc56 --- /dev/null +++ b/crates/smallclaims/src/store/operation_conflict_fence.rs @@ -0,0 +1,52 @@ +//! A discovery watermark is valid only for the same checkpoint/restored claim prefix. +use super::*; + +const INSTALL: &str = /* sqlite-large-value: allow-for-now lv-hdd122545f0d0 owner=smallclaims::store::operation_conflict_fence TODO=chunk-or-reference-variable-columns */ r#" +INSERT OR IGNORE INTO meta(key,value) VALUES('operation_conflict_source_generation','0'); +CREATE INDEX IF NOT EXISTS projection_health_degraded_operation_subject +ON projection_health(error_code,aggregate) +WHERE status='stale' AND error_code='operation-conflict-no-bounded-refold'; +CREATE TRIGGER IF NOT EXISTS operation_conflict_checkpoint_insert AFTER INSERT ON checkpoint_claims BEGIN + UPDATE meta SET value=CAST(value AS INTEGER)+1 WHERE key='operation_conflict_source_generation'; +END; +CREATE TRIGGER IF NOT EXISTS operation_conflict_checkpoint_update AFTER UPDATE ON checkpoint_claims BEGIN + UPDATE meta SET value=CAST(value AS INTEGER)+1 WHERE key='operation_conflict_source_generation'; +END; +CREATE TRIGGER IF NOT EXISTS operation_conflict_checkpoint_delete AFTER DELETE ON checkpoint_claims BEGIN + UPDATE meta SET value=CAST(value AS INTEGER)+1 WHERE key='operation_conflict_source_generation'; +END; +CREATE TRIGGER IF NOT EXISTS operation_conflict_prefix_insert AFTER INSERT ON claims +WHEN NEW.store_index <= (SELECT last_good_store_index FROM projection_health WHERE aggregate='projection:operation-conflict-scan') BEGIN + UPDATE meta SET value=CAST(value AS INTEGER)+1 WHERE key='operation_conflict_source_generation'; +END; +CREATE TRIGGER IF NOT EXISTS operation_conflict_prefix_update AFTER UPDATE OF id,body,store_index ON claims BEGIN + UPDATE meta SET value=CAST(value AS INTEGER)+1 WHERE key='operation_conflict_source_generation'; +END; +CREATE TRIGGER IF NOT EXISTS operation_conflict_prefix_delete AFTER DELETE ON claims BEGIN + UPDATE meta SET value=CAST(value AS INTEGER)+1 WHERE key='operation_conflict_source_generation'; +END; +"#; + +pub(super) fn initialize(connection: &Connection) -> Result<()> { + let tx = connection.unchecked_transaction()?; + tx.execute_batch(/* sqlite-large-value: allow-for-now lv-h12783970e005 owner=smallclaims::store::operation_conflict_fence TODO=trace-statement-and-value-representation */ INSTALL)?; + tx.commit()?; + Ok(()) +} + +pub(super) fn generation(connection: &Connection) -> Result { + Ok(connection.query_row( + "SELECT value FROM meta WHERE key='operation_conflict_source_generation'", + [], |row| row.get(0), + )?) +} + +/// Call while holding the projection transaction: a racing checkpoint mutation makes +/// discovery unproven, never permission to replay. No receipt history is read here. +pub fn valid(connection: &Connection, through: u64) -> Result { + Ok(connection.query_row( + "SELECT EXISTS(SELECT 1 FROM projection_health WHERE aggregate='projection:operation-conflict-scan' + AND last_good_store_index>=?1 AND error_message=(SELECT value FROM meta WHERE key='operation_conflict_source_generation'))", + [through], |row| row.get(0), + )?) +} diff --git a/docs/st3/large-value-sites.json b/docs/st3/large-value-sites.json index 3a324445f0a..0ed8e7f8276 100644 --- a/docs/st3/large-value-sites.json +++ b/docs/st3/large-value-sites.json @@ -683,6 +683,7 @@ {"site": "lv-h3bf56234d7c1", "path": "crates/smallclaims/src/store/operation_claim_digests.rs", "table_pattern": "operation_claim_digests", "columns": [{"name": "store_index", "declared_type": "INTEGER", "declaration": "INTEGER PRIMARY KEY", "value_size_proven": false}, {"name": "claim_id", "declared_type": "TEXT", "declaration": "TEXT NOT NULL", "value_size_proven": false}, {"name": "operation_id", "declared_type": "TEXT", "declaration": "TEXT NOT NULL", "value_size_proven": false}, {"name": "request_digest", "declared_type": "TEXT", "declaration": "TEXT NOT NULL", "value_size_proven": false}], "unresolved_template": false}, {"site": "lv-h3bf56234d7c1", "path": "crates/smallclaims/src/store/operation_claim_digests.rs", "table_pattern": "operation_claim_digests_backfill", "columns": [{"name": "singleton", "declared_type": "INTEGER", "declaration": "INTEGER PRIMARY KEY CHECK(singleton=1)", "value_size_proven": false}, {"name": "operation", "declared_type": "TEXT", "declaration": "TEXT NOT NULL", "value_size_proven": false}, {"name": "claim_rowid", "declared_type": "INTEGER", "declaration": "INTEGER NOT NULL", "value_size_proven": false}, {"name": "ready", "declared_type": "INTEGER", "declaration": "INTEGER NOT NULL CHECK(ready IN (0,1))", "value_size_proven": false}], "unresolved_template": false}, {"site": "lv-h3bf56234d7c1", "path": "crates/smallclaims/src/store/operation_claim_digests.rs", "index_pattern": "operation_claim_digests_operation", "table_pattern": "operation_claim_digests", "key_expression_and_predicate": "(operation_id, store_index, request_digest, claim_id)", "value_size_proven": false}, + {"site": "lv-hdd122545f0d0", "path": "crates/smallclaims/src/store/operation_conflict_fence.rs", "index_pattern": "projection_health_degraded_operation_subject", "table_pattern": "projection_health", "key_expression_and_predicate": "(error_code,aggregate) WHERE status='stale' AND error_code='operation-conflict-no-bounded-refold'", "value_size_proven": false}, {"site": "lv-hf22585109b10", "path": "crates/smallclaims/src/store/prepared_envelope_tests.rs", "table_pattern": "admission_writes", "columns": [{"name": "table_name", "declared_type": "TEXT", "declaration": "TEXT", "value_size_proven": false}], "unresolved_template": false}, {"site": "lv-h6fb8658a1947", "path": "crates/smallclaims/src/store/principals/migration.rs", "table_pattern": "claim_signature_migration", "operation": "add-column", "columns": [{"name": "serving_legacy", "declared_type": "INTEGER", "declaration": "INTEGER NOT NULL DEFAULT 0 CHECK(serving_legacy IN (0,1))", "value_size_proven": false}], "unresolved_template": false}, {"site": "lv-h766cb1f13e91", "path": "crates/smallclaims/src/store/principals/migration.rs", "table_pattern": "claim_signature_migration", "operation": "add-column", "columns": [{"name": "legacy_present", "declared_type": "INTEGER", "declaration": "INTEGER NOT NULL DEFAULT 0 CHECK(legacy_present IN (0,1))", "value_size_proven": false}], "unresolved_template": false}, @@ -1113,6 +1114,7 @@ {"id": "lv-h124eae47987b", "path": "crates/fractal-delivery/src/store/terminal_gate_evidence/native_source/tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "15097144cc01106359c012692c9860687e97d1f3bdaa47a74ae5d595cf6f6cc2", "owner": "fractal_delivery::store::terminal_gate_evidence::native_source::tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 415, "excerpt": "mutation , [ ]"}, {"id": "lv-h125e4b6c7f33", "path": "crates/smallclaims/src/store/repair_operation_tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "636a1034297cf04d5b70979126813cff10dbde27a79d5357575f604217d93684", "owner": "smallclaims::store::repair_operation_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 919, "excerpt": "\"ROLLBACK TO changed_candidates; RELEASE changed_candidates\""}, {"id": "lv-h1269cd039065", "path": "crates/smallclaims/src/ivm.rs", "kind": "unresolved-sql-sink", "fingerprint": "5181fb085f960c22081d1093ec4dd97ff39592ed341fbd28070d0538751d9676", "owner": "smallclaims::ivm", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 914, "excerpt": "\"ROLLBACK TO ivm_view_change; RELEASE ivm_view_change\""}, + {"id": "lv-h12783970e005", "path": "crates/smallclaims/src/store/operation_conflict_fence.rs", "kind": "unresolved-sql-sink", "fingerprint": "b83b327f0b62e8168163c61f15fb0ac2c14169729e4f4184678337a15cd04dec", "owner": "smallclaims::store::operation_conflict_fence", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 32, "excerpt": "INSTALL"}, {"id": "lv-h127af3ce1c01", "path": "crates/fractal-seats/src/store/resource_preparation/tests.rs", "kind": "sql-template", "fingerprint": "30d1f4233a95a0d6973aaa00f7eda6b416be061a920c4f1474b2353f56752d66", "owner": "fractal_seats::store::resource_preparation::tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 288, "excerpt": "CREATE TEMP VIEW"}, {"id": "lv-h128053eb6892", "path": "crates/fractal-github/src/changes/tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "f53e8377ff4b2422fab653f6e042b378308d2c565d23ae68d0e6e46622780bcd", "owner": "fractal_github::changes::tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 677, "excerpt": "retargeted"}, {"id": "lv-h12810b4d74c0", "path": "crates/fractal-store/src/store/person_rename.rs", "kind": "unresolved-sql-sink", "fingerprint": "671d6e13a497da1096904b356ae147dc6420d74d11adc525a5e2a942d1918d28", "owner": "fractal_store::store::person_rename", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 27, "excerpt": "& canonical_sql ( \"SELECT id,store_index,batch_id,subject,kind,origin,actor,body,predecessors,accepted_at_unix_ms FROM claims WHERE kind='person.rename.completed' AND json_extract(body,'$.idempotency_"}, @@ -2590,6 +2592,7 @@ {"id": "lv-h6907f5361ba9", "path": "crates/smallclaims/src/store/checkpoint_staging/scoped.rs", "kind": "sql-template", "fingerprint": "773e60c5c971aca9fba1a24e45049e65f6c0271c8144818a9c063ad09b13d2ec", "owner": "smallclaims::store::checkpoint_staging::scoped", "todo": "chunk-or-reference-exact-wire-values", "disposition": "allow-for-now", "line_hint": 97, "excerpt": "INSERT OR IGNORE INTO envelopes VALUES (?1, ?2, ?3, ?4, ?5, 1)"}, {"id": "lv-h6926004f4a7e", "path": "crates/fractal-store/src/store/resources_preparation.rs", "kind": "sql-template", "fingerprint": "2e2a39a05e79d74ff1c9d8c85b7e96a9e91bc4ff3a13a9dbbad62bf271187f76", "owner": "fractal_store::store::resources_preparation", "todo": "chunk-or-reference-variable-columns", "disposition": "allow-for-now", "line_hint": 19, "excerpt": " CREATE TABLE IF NOT EXISTS local_resource_preparation ( subject TEXT PRIMARY KEY, actual TEXT, kind TEXT, observed_at TEXT NOT NULL DEFAULT '0', actual_key BLOB, kind_key BLOB, observation_key BLOB, "}, {"id": "lv-h69371e298251", "path": "crates/fractal-github/src/changes/tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "70ce871f8a3d3fb449bc3c3ace6547cef02dfc74ffe48d912532a724bfdbe5b9", "owner": "fractal_github::changes::tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 497, "excerpt": "record"}, + {"id": "lv-h69417713b2cb", "path": "crates/fractal-store/src/store/operation_conflict_tests.rs", "kind": "sql-template", "fingerprint": "94dedd5db98c1e80254ce79345f6724cee48d90b3acfe79f248d3adebe5f661a", "owner": "fractal_store::store::operation_conflict_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 431, "excerpt": "UPDATE projection_health SET status='stale' WHERE aggregate='graph'"}, {"id": "lv-h695b9626f834", "path": "crates/smallclaims/src/store/repair_operation_tests.rs", "kind": "sql-template", "fingerprint": "571e24d4ecefbc74a02fe49b1b785b46d344d9f944e01d49b04b767c5dd9d9d6", "owner": "smallclaims::store::repair_operation_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 529, "excerpt": "INSERT INTO operations VALUES('stale/extra','z',?1,'active')"}, {"id": "lv-h696ec90fbf79", "path": "crates/fractal-seats/src/store/reconcile_work_context/tests.rs", "kind": "sql-template", "fingerprint": "8e4c6a9a11198c083bc6211a87780d8a93672142af3c5f36f86dcd40f0a5f870", "owner": "fractal_seats::store::reconcile_work_context::tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 84, "excerpt": "UPDATE step_runs SET status=?2,lease_owner=?3, lease_incarnation='worker-one',lease_expires_at_unix_ms=?4, blocked_reason=?5 WHERE subject=?1"}, {"id": "lv-h6986717d6e82", "path": "crates/fractal-store/src/store/resources_preparation.rs", "kind": "unresolved-sql-sink", "fingerprint": "056af73dc3b193bc4dce97ea87272224bf6b8274db992f673d68bd73cab1dea5", "owner": "fractal_store::store::resources_preparation", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 415, "excerpt": "params ! [ id , subject , key ]"}, @@ -3782,6 +3785,7 @@ {"id": "lv-hb0d024a9c61b", "path": "crates/fractal/src/main_tests.rs", "kind": "included-source", "fingerprint": "7ac64927952fe74260c7adfccbb8e19e53cc1e383d12bcc59984757b832e5292", "owner": "fractal::main_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 3396, "excerpt": "include_str ! ( \"uninstall_cli.rs\" )"}, {"id": "lv-hb0d5de40e6ef", "path": "crates/fractal-store/src/store/resources_preparation.rs", "kind": "sql-template", "fingerprint": "a77b5750eee82ab4bbeaa0a7b6340044333f846d3e09f91d1983a7c81e3af1ce", "owner": "fractal_store::store::resources_preparation", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 295, "excerpt": "CREATE TRIGGER IF NOT EXISTS preparation_batches_old_update AFTER UPDATE ON batches WHEN OLD.id IS NOT NEW.id BEGIN INSERT INTO local_resource_preparation_dirty(subject) SELECT subject FROM claims WHE"}, {"id": "lv-hb0dab8208622", "path": "crates/fractal-api-client-v0/src/api/client_v0/summary/tests.rs", "kind": "sql-template", "fingerprint": "017ac7c0df346114a4b1269321691194ab9c3e70f7eb3d133b74f85bf9121694", "owner": "fractal_api_client_v0::api::client_v0::summary::tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 400, "excerpt": "INSERT INTO mission_definitions(mission_id,revision,state,claim_id) SELECT ?1,revision,state,claim_id FROM mission_definitions WHERE mission_id='summary-base'"}, + {"id": "lv-hb0e4de996f4f", "path": "crates/smallclaims/src/store.rs", "kind": "sql-template", "fingerprint": "fe4b988aa647c3cb1fdf742b9af323f4a88e39ac4406df5552a7f226567e4641", "owner": "smallclaims::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 8871, "excerpt": "INSERT INTO projection_health(aggregate,status,last_good_store_index,updated_at_unix_ms,error_message) VALUES ('projection:operation-conflict-scan','healthy',?1,?2,?3) ON CONFLICT(aggregate) DO UPDATE"}, {"id": "lv-hb0fb45559620", "path": "crates/fractal-store/src/store/notice_authority_tests.rs", "kind": "sql-template", "fingerprint": "7a5089e60ae9a32e486995a4efcd667a36f9e9e0a38a1d7d49ec785439b6b3b9", "owner": "fractal_store::store::notice_authority_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 332, "excerpt": "INSERT INTO mission_runs VALUES('notice-owner','example/notice-owner','revision', 'notice-generation','revision','notice-owner',NULL,'.','person/example','{}','run','running','normal','0','0')"}, {"id": "lv-hb1035261bc59", "path": "crates/smallclaims/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "c26c697f861328d42d7e93f0289393cc3deb72df5850dcb252c18c0b1db297db", "owner": "smallclaims::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 1210, "excerpt": "local_trim_citations : : SCHEMA"}, {"id": "lv-hb11b7cb6c531", "path": "crates/fractal-store/src/store/message_sent_tests.rs", "kind": "unresolved-sql-sink", "fingerprint": "35b3b55dc0d41a833a72f4009dfed3434aa87154d6b9b83da24b2d728d42ad32", "owner": "fractal_store::store::message_sent_tests", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 160, "excerpt": "& format ! ( \"EXPLAIN QUERY PLAN {sql}\" )"}, @@ -4111,7 +4115,6 @@ {"id": "lv-hc1ac3d35bbf4", "path": "crates/fractal-store/src/store/client_messages.rs", "kind": "sql-template", "fingerprint": "b00efd86cc227528fbfa988943c50c7bc8f88c73ef6908072ddf7ed7cac1bc74", "owner": "fractal_store::store::client_messages", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 1523, "excerpt": "DELETE FROM meta WHERE key=?1"}, {"id": "lv-hc1b4b7df7d60", "path": "crates/smallclaims/src/sqlite.rs", "kind": "unresolved-sql-sink", "fingerprint": "ce922519a3c3ecaf9b0986c2449c7680895c15f4b0e9818e994e14a4d28b6aaf", "owner": "smallclaims::sqlite", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 995, "excerpt": "transaction"}, {"id": "lv-hc1c6ea0cb228", "path": "crates/fractal-store/src/store/terminal_gate_evidence/native_source_tests_shared.rs", "kind": "sql-template", "fingerprint": "740d295c2a21b85f7e9f3cfc6e07c1cddb61c6d0698f1d79e9cb732ef1e0c259", "owner": "fractal_store::store::terminal_gate_evidence::native_source_tests_shared", "todo": "chunk-or-reference-exact-wire-values", "disposition": "allow-for-now", "line_hint": 87, "excerpt": "WITH RECURSIVE pages(n) AS (SELECT 0 UNION ALL SELECT n+1 FROM pages WHERE n<(?1-1)/64) INSERT INTO batches(id,origin,replica_sequence,hash,accepted_at_unix_ms) SELECT 'foreign-page/'||n,'node',1000+n"}, - {"id": "lv-hc1d2cd2957a7", "path": "crates/smallclaims/src/store.rs", "kind": "sql-template", "fingerprint": "2757c79a329d4159d2c1ee47a342658ebf97772333f56b9264a4d35d847d72cb", "owner": "smallclaims::store", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 8856, "excerpt": "INSERT INTO projection_health(aggregate,status,last_good_store_index,updated_at_unix_ms) VALUES ('projection:operation-conflict-scan','healthy',?1,?2) ON CONFLICT(aggregate) DO UPDATE SET last_good_st"}, {"id": "lv-hc1db65df6dce", "path": "crates/fractal-store/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "f364d8ca6168ffa8cad7d686c9cbe491d919c3fcdda7ba967af6912aaffb62ae", "owner": "fractal_store::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 6701, "excerpt": "& format ! ( \"{} LIMIT 1\" , newest_claims_of_kind_query ( \"claims.origin\" , \"runtime.observed\" ) )"}, {"id": "lv-hc1e68820d94d", "path": "crates/smallclaims/src/store.rs", "kind": "unresolved-sql-sink", "fingerprint": "789b44b9990c7d88da3542aa5c6b25148409a57d64b88f00362513cda78a031f", "owner": "smallclaims::store", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 1188, "excerpt": "\"COMMIT\""}, {"id": "lv-hc1efad5f0359", "path": "crates/fractal-store/src/store/resources_preparation_tests.rs", "kind": "sql-template", "fingerprint": "fd3f455464159e17ece91a42ad4675d290a1d23ae78f3310dfac39c06c99e3ec", "owner": "fractal_store::store::resources_preparation_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 497, "excerpt": "UPDATE claim_positions SET position=CASE WHEN claim_id=?1 THEN 9 ELSE 1 END WHERE claim_id IN (?1,?2)"}, @@ -4450,6 +4453,7 @@ {"id": "lv-hd586f33df6b3", "path": "crates/fractal-store/src/store/observer_subscription_detail.rs", "kind": "unresolved-sql-sink", "fingerprint": "bffa0bf47d8a75ef69b1b286b32abfe67948c2c365bf5a5abef0cd79b0198492", "owner": "fractal_store::store::observer_subscription_detail", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 148, "excerpt": "[ subject ] , | row | { let body = row . get : : < _ , String > ( 1 ) ? ; Ok ( DetailDeclaration { desired : DesiredSubject { accountable : None , subject : subject . into ( ) , kind : stored_kind . c"}, {"id": "lv-hd58e59ffadfb", "path": "crates/fractal-store/src/store/person_work.rs", "kind": "unresolved-sql-sink", "fingerprint": "fb6b4fafeac86382b8b4a1b919786b3dadfd6e8059e040261de55723208895f6", "owner": "fractal_store::store::person_work", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 828, "excerpt": "params ! [ mission . id , mission . revision ] , | row | row . get ( 0 ) ,"}, {"id": "lv-hd597c1f50002", "path": "crates/fractal-store/src/store/terminal_gate_evidence/native_source_shared.rs", "kind": "sql-template", "fingerprint": "62f9295ad69732ccd0a4f16833de551ec9f469044e2d8a95847d965052b15c6e", "owner": "fractal_store::store::terminal_gate_evidence::native_source_shared", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 201, "excerpt": "INSERT INTO test_native_terminal_guard VALUES(1,?1,?2,?3,?4,?5,?6,?7,0,0,NULL,NULL,0)"}, + {"id": "lv-hd5cb81fd9252", "path": "crates/fractal-store/src/store/operation_conflict_tests.rs", "kind": "sql-template", "fingerprint": "64ba57efd774d7e9ae822168b5f0aa12ac89bb0e425f0f23b7b8d2865bc0ccdc", "owner": "fractal_store::store::operation_conflict_tests", "todo": "stage-generation-and-keyed-reclamation", "disposition": "allow-for-now", "line_hint": 427, "excerpt": "INSERT INTO checkpoint_claims(id,writer,sequence,envelope_hash,subject,kind, actor,predecessors,operation_id,request_digest,accepted_at_unix_ms,checkpoint) VALUES ('late-tombstone','peer',1,'envelope'"}, {"id": "lv-hd5f90ba66291", "path": "crates/fractal-store/src/store/custom.rs", "kind": "unresolved-sql-sink", "fingerprint": "45de5b742be9d0e5cd0ba59d294584a610a7f05e563491431ae653f5ffca4e86", "owner": "fractal_store::store::custom", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 120, "excerpt": "& canonical_sql ( \"SELECT claims.id,claims.store_index,claims.batch_id,claims.subject,claims.kind,claims.origin,claims.actor,claims.body,claims.predecessors,claims.accepted_at_unix_ms FROM claims JOIN"}, {"id": "lv-hd5fb23096b84", "path": "crates/fractal-store/src/store/client_messages.rs", "kind": "sql-template", "fingerprint": "93a6ae20b41f3b50adacb2d9a47be1570b76c7dc285491a696c3395ce1db9f43", "owner": "fractal_store::store::client_messages", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 922, "excerpt": "INSERT INTO local_client_message_selectors_v1(subject,born_index,created_index,dirty_flags) VALUES(?1,-1,1,2) ON CONFLICT(subject,born_index) DO UPDATE SET dirty_flags=dirty_flags|2,created_index=crea"}, {"id": "lv-hd60ba3ca89ff", "path": "crates/smallclaims/src/store/inventory_generation.rs", "kind": "unresolved-sql-sink", "fingerprint": "5a8493ce00029e2325ab8e1257235dcaac72db782142700f36a73961b0cc40b5", "owner": "smallclaims::store::inventory_generation", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 108, "excerpt": "[ KEY ] , | row | row . get ( 0 )"}, @@ -4571,6 +4575,7 @@ {"id": "lv-hdd0b5313d7ac", "path": "crates/fractal-reconcile/src/reconcile.rs", "kind": "included-source", "fingerprint": "aeb5c01984393e31f49089cc86ebe7cca32d498650ac96b2c495848dec58c1ec", "owner": "fractal_reconcile::reconcile", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 46421, "excerpt": "include_str ! ( \"../../fractal-drivers/tests/fixtures/blocking-screens/claude-wide-footer.txt\" )"}, {"id": "lv-hdd0d4ca3ab13", "path": "crates/fractal-seats/src/store/seat_status.rs", "kind": "unresolved-sql-sink", "fingerprint": "1c18d7bb6781c8266b15993dc6fe484ae2e5f506d67220115c302cfaf9dbcacd", "owner": "fractal_seats::store::seat_status", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 48, "excerpt": "params ! [ subject , index , harness . incarnation_id ] , | row | { row . get ( 0 ) }"}, {"id": "lv-hdd116b500e13", "path": "crates/fractal-store/src/store/completion_state.rs", "kind": "sql-template", "fingerprint": "3501ccac67f3ab9fa5ef25d0e67d31d5d7184011cd6d9d4f8910bcd500ce0268", "owner": "fractal_store::store::completion_state", "todo": "prepare-body-and-predecessor-values", "disposition": "allow-for-now", "line_hint": 194, "excerpt": "CREATE TRIGGER IF NOT EXISTS completion_health_v1_siblings_{event}_{row} AFTER {event} ON claims{condition} BEGIN INSERT INTO local_completion_health_v1_pending(claim_id) SELECT id FROM claims WHERE b"}, + {"id": "lv-hdd122545f0d0", "path": "crates/smallclaims/src/store/operation_conflict_fence.rs", "kind": "sql-template", "fingerprint": "9040720a3e61a7490428682ad552eeba70e57085c9631a26b0c5d6dd8d7eb8f0", "owner": "smallclaims::store::operation_conflict_fence", "todo": "chunk-or-reference-variable-columns", "disposition": "allow-for-now", "line_hint": 4, "excerpt": " INSERT OR IGNORE INTO meta(key,value) VALUES('operation_conflict_source_generation','0'); CREATE INDEX IF NOT EXISTS projection_health_degraded_operation_subject ON projection_health(error_code,aggre"}, {"id": "lv-hdd12cc78c9f2", "path": "crates/fractal-missions/src/store/missions.rs", "kind": "unresolved-sql-sink", "fingerprint": "464ad1ad68d82b99fbd4c3ba5d668e93c24f9820f3e6a97a730875066684f0df", "owner": "fractal_missions::store::missions", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 8023, "excerpt": "[ claim ] , | row | row . get : : < _ , u64 > ( 0 )"}, {"id": "lv-hdd1697922ff7", "path": "crates/fractal-store/src/store/publication.rs", "kind": "unresolved-sql-sink", "fingerprint": "8edb630b01a1f78d4abc5ee08b714ce7a8578b35d2269aec45684bf19ac1e4fc", "owner": "fractal_store::store::publication", "todo": "trace-statement-and-value-representation", "disposition": "allow-for-now", "line_hint": 40, "excerpt": "& query"}, {"id": "lv-hdd207efeb1d9", "path": "crates/fractal-seats/src/store/ordered_membership_tests.rs", "kind": "sql-template", "fingerprint": "dbadb2f9c6bd395d78f39e4e1a28290fb8b8ca07856402d2c4c354fe1b941dd0", "owner": "fractal_seats::store::ordered_membership_tests", "todo": "bound-or-reference-values-and-reclamation", "disposition": "allow-for-now", "line_hint": 235, "excerpt": "UPDATE ordered_membership_counts SET changed_index=?1 WHERE container=?2"}, diff --git a/docs/st3/large-values.md b/docs/st3/large-values.md index 17b9a9f0585..132300a1c99 100644 --- a/docs/st3/large-values.md +++ b/docs/st3/large-values.md @@ -625,9 +625,17 @@ diagnostic. Both retain metadata, never claim bodies, with the same bounding deb `lv-hf0bbe1f00bc2` is the second-chunk stale-health regression fixture. `smallclaims::store` owns `lv-hb7532db3c65d` (full operation/claim identity evidence), -`lv-hc1d2cd2957a7` (the incremental reader scan watermark), `lv-hcc82e02fe20f` +`lv-hb0e4de996f4f` (the generation-qualified incremental reader scan watermark), `lv-hcc82e02fe20f` (graph health that retains deferred-repair visibility), and `lv-h362e35c78646` (clearing that deferred row only after explicit full repair succeeds). Evidence contains identifiers, not claim bodies; identifier representation remains the same bounding debt. Discovery reads only the newly admitted target range on readers; writer metadata pages are capped at the projection chunk size. + +`smallclaims::store::operation_conflict_fence` owns `lv-hdd122545f0d0` and +`lv-h12783970e005`, installing checkpoint/restored-prefix invalidation triggers +and the partial stale-exclusion index. They mutate integer generation metadata; +the index retains existing operation/subject keys, with the same bounded-identity +debt rather than a new body population. `lv-hd5cb81fd9252` and `lv-h69417713b2cb` +are synthetic checkpoint-adoption/stale-health regression sites owned by +`fractal_store::store::operation_conflict_tests`. diff --git a/release-notes/operation-conflict-discovery-generation-20261111.json b/release-notes/operation-conflict-discovery-generation-20261111.json new file mode 100644 index 00000000000..a497ec30fcb --- /dev/null +++ b/release-notes/operation-conflict-discovery-generation-20261111.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "summary": "Invalidate conflict discovery when checkpoint or restored history changes and index durable exclusions.", + "replay": {"status": "changed", "detail": "Conflict discovery records the checkpoint/restored-prefix generation alongside its claims watermark. Checkpoint insert/update/delete and restored-prefix mutations invalidate that proof. A mutation racing discovery or projection fails closed: automatic receipt defers rather than authorizing full replay."}, + "schema": {"status": "changed", "detail": "Forward-only installation adds local generation triggers and a partial projection_health index for stale operation-conflict exclusions. No claim format changes."}, + "rules": {"status": "none", "detail": "Canonical operation and graph rules are unchanged."}, + "compatibility": {"status": "changed", "detail": "Existing discovery watermarks without a source generation are unproven and rescan on readers. Exclusion lookup uses the partial index, independent of diagnostic history."}, + "manual": {"status": "none", "detail": "No additional operator steps."}, + "recovery": {"status": "changed", "detail": "A changed checkpoint generation restarts incremental reader discovery; explicit repair remains required for deferred projections."}, + "downtime": {"status": "unknown", "detail": "No production timing measurement is claimed."} +} From 43b0ee9b4796ba3122cccee43f9134534675d9f4 Mon Sep 17 00:00:00 2001 From: schickling-assistant <261620128+schickling-assistant@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:05:21 +0100 Subject: [PATCH 6/6] fix(smallclaims): convert replay fence failures to structured errors agent-identity: unknown agent-persona: unknown agent-supervisor: unavailable agent-tool: unknown agent-runtime: unknown agent-model: unknown --- crates/smallclaims/src/store.rs | 7 ++++--- ...operation-conflict-error-conversions-20261111.json | 11 +++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 release-notes/operation-conflict-error-conversions-20261111.json diff --git a/crates/smallclaims/src/store.rs b/crates/smallclaims/src/store.rs index a4547a44be6..f81992e6061 100644 --- a/crates/smallclaims/src/store.rs +++ b/crates/smallclaims/src/store.rs @@ -9044,7 +9044,8 @@ impl Store { UNION ALL SELECT json_array(canonical_claim_id,id,canonical_claim_id) FROM operations INDEXED BY operations_conflict_index WHERE state='conflict' LIMIT 1", [], |row| row.get::<_,String>(0), - ).optional()?.map(|message| serde_json::from_str(&message)).transpose()?; + ).optional().map_err(internal)? + .map(|message| serde_json::from_str(&message)).transpose().map_err(internal)?; if let Some((claim_id, operation_id, previous_claim_id)) = conflict { let error = FractalError::new("operation-conflict-projection-degraded", json!({"operation_id":operation_id,"claim_id":claim_id, @@ -9057,12 +9058,12 @@ impl Store { self.log_projection_failure(&error, phase, frontier, target, &mut log_staged); return Err(error); } - if !operation_conflict_discovery_valid(&transaction, target)? { + if !operation_conflict_discovery_valid(&transaction, target).map_err(internal)? { return Err(FractalError::new("operation-conflict-discovery-unproven", "checkpoint or restored history changed during discovery; retry")); } // Replay is not target-bounded. A later admission has not been scanned: // defer until the next pass discovers that newly admitted range. - if current_index_tx(&transaction)? != target { + if current_index_tx(&transaction).map_err(internal)? != target { return Err(FractalError::new("projection-target-changed", "retry replay with the newly admitted range")); } log_staged(&full_replay_log_line(phase, &reason, frontier, target)); diff --git a/release-notes/operation-conflict-error-conversions-20261111.json b/release-notes/operation-conflict-error-conversions-20261111.json new file mode 100644 index 00000000000..e712d1731c2 --- /dev/null +++ b/release-notes/operation-conflict-error-conversions-20261111.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "summary": "Fix typed error conversions in the operation-conflict replay fence.", + "replay": {"status": "none", "detail": "No replay behavior change; fallible SQLite, JSON, discovery-validity and frontier reads now convert to the projection closure's structured error type."}, + "schema": {"status": "none", "detail": "No schema changes."}, + "rules": {"status": "none", "detail": "No rule changes."}, + "compatibility": {"status": "none", "detail": "No compatibility changes."}, + "manual": {"status": "none", "detail": "No operator steps."}, + "recovery": {"status": "none", "detail": "No recovery changes."}, + "downtime": {"status": "none", "detail": "Compile-time correction only."} +}