From 4ee9d9e1956650011c70139ae4b1d80529a76119 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 23 Sep 2026 00:46:10 +1000 Subject: [PATCH 1/5] fix: preserve append identity across process retries Signed-off-by: wchwawa --- README.md | 31 + crates/nokv-client/src/artifact.rs | 130 +- crates/nokv-client/src/error.rs | 24 +- crates/nokv-client/src/idempotent_append.rs | 482 +++++ crates/nokv-client/src/lib.rs | 2 + crates/nokv-client/src/workbench_workflow.rs | 9 + crates/nokv-meta/src/workspace/codec.rs | 6 +- crates/nokv-meta/src/workspace/commit.rs | 9 + crates/nokv-meta/src/workspace/publication.rs | 83 +- .../workspace/publish_operation_records.rs | 35 +- crates/nokv-meta/src/workspace/restore.rs | 151 ++ crates/nokv-protocol/src/artifact_publish.rs | 2 +- crates/nokv-protocol/src/codec.rs | 77 +- crates/nokv-protocol/src/handshake.rs | 16 +- crates/nokv-protocol/src/lib.rs | 12 +- crates/nokv-protocol/src/request.rs | 47 + crates/nokv-protocol/src/response.rs | 76 + crates/nokv-python/README.md | 20 + crates/nokv-python/python/nokv/__init__.py | 34 + crates/nokv-python/src/client.rs | 139 +- crates/nokv-python/tests/test_api_surface.py | 26 + crates/nokv-server/src/executor.rs | 325 +++ crates/nokv-server/src/legacy_rejection.rs | 2 +- crates/nokv-server/src/lifecycle.rs | 4 + crates/nokv-server/src/server.rs | 25 +- crates/nokv/src/backend.rs | 127 +- crates/nokv/src/cli.rs | 155 ++ crates/nokv/src/main.rs | 166 +- docs/architecture.md | 11 +- docs/development/append-idempotency.md | 146 ++ docs/metadata-schema.md | 39 +- .../append_identity_recovery_gate.py | 1836 +++++++++++++++++ 32 files changed, 4091 insertions(+), 156 deletions(-) create mode 100644 crates/nokv-client/src/idempotent_append.rs create mode 100644 docs/development/append-idempotency.md create mode 100644 scripts/workbench/append_identity_recovery_gate.py diff --git a/README.md b/README.md index 7d6a16537..74320ad71 100644 --- a/README.md +++ b/README.md @@ -238,10 +238,41 @@ real-service acceptance gate is still **not qualified**. | `nokv materialize` | Copy a verified workspace artifact to a new local path. | The destination is disposable scratch, not a namespace or mount. | | `nokv collect` | Publish one bounded regular local file, create-only or generation-fenced replace. | Symlinks and unbounded/non-regular inputs fail closed. | | `nokv workspace-path rename` / `nokv workspace-path remove` | Apply an explicit generation- and request-id-fenced path mutation. | Custom CLI surface; not one of the 18 Workbench tools. | +| `nokv workspace-path append` | Append bytes under a caller-owned operation identity and return the original publication on replay. | Requires an existing workspace; pin its incarnation across restarts. Custom CLI surface; the 18-tool schema is unchanged. | | `nokv provision` | Bind `RootId` to `AgentId`, object namespace, logical shard, and persisted placement through etcd. | `AgentId` prevents accidental root reuse; it is not authentication. | | `nokv serve` | Start one explicit metadata owner from create, same-namespace reopen, or recovery-log state. | Shared recovery publication is opt-in and not currently qualified. | | `nokv mcp` | Deprecated stdio transport retained only because qualification runners still use it. | **Unsupported for integration.** | +For an append that must survive a lost reply or a harness restart, save one +operation id before sending the request, then reuse it with the same inputs: + +```shell +nokv [route/agent/object options] workspace-path append run-42 logs events.jsonl \ + --operation-id aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + --expected-workspace-incarnation-id bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb \ + --text 'step-7-done' --content-type text/plain +``` + +Replace the example ids with the caller's stable operation id and the existing +workspace's incarnation. Omit the incarnation only to observe and fence the +existing workspace on this call; append never creates a missing workspace. +Use `--text`, `--base64`, or `--file` for exactly one delta. Text defaults to +`text/plain; charset=utf-8` on creation; binary and file input default to +`application/octet-stream`. Existing content type is inherited unless +`--content-type` overrides it. `--max-artifact-bytes` bounds the delta; +`--max-logical-size` optionally bounds the resulting artifact. + +The receipt's operation/revision ids, generation, workspace revision, size, +digest, and incarnation describe that publication, even if another writer has +since advanced the path. `replayed` and optional `commit_version` describe this +SDK call and are not stable receipt fields; a recovered receipt may have a null +`commit_version`. An unresolved error retains the operation id and any observed +durable state. Recover the same identity; a timeout does not authorize applying +the delta under a new identity. One identity owns one publication attempt; +this command does not silently create a new attempt after a CAS conflict. +The older `workbench_append` tool retains per-invocation identities and does +not provide cross-process caller-identity replay. + ### Programmatic and non-CLI capabilities The interfaces share protocol, metadata, object, and lifecycle semantics. They diff --git a/crates/nokv-client/src/artifact.rs b/crates/nokv-client/src/artifact.rs index d8ac8f4e0..3c884b9bf 100644 --- a/crates/nokv-client/src/artifact.rs +++ b/crates/nokv-client/src/artifact.rs @@ -128,10 +128,9 @@ pub struct ArtifactPublishOutcome { /// Caller-owned identity seed and policy for one logical append. /// -/// A conflicting create/append race derives deterministic attempt identities -/// from the supplied identities. Retrying this method with the same options -/// therefore replays the same attempt sequence instead of applying the delta a -/// second time after response loss. +/// A conflicting create/append race derives attempt identities within one +/// invocation. This legacy API does not provide cross-process intent replay; +/// use `WorkspaceClient::append_artifact_idempotent` for a caller-stable action. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ArtifactAppendOptions { pub operation_id: OperationIdentity, @@ -310,6 +309,7 @@ where options.authority, options.condition, options.expected_workspace_incarnation_id, + None, object_plan, staged_objects, manifest_rows, @@ -330,6 +330,7 @@ where authority: PublicationAuthority, condition: PublishCondition, expected_workspace_incarnation_id: Option, + append_intent_digest: Option, object_plan: ArtifactUploadPlan, staged_objects: Vec, manifest_rows: Vec, @@ -349,6 +350,7 @@ where &authority, &condition, expected_workspace_incarnation_id, + append_intent_digest, &object_plan, &staged_objects, &manifest_rows, @@ -386,6 +388,7 @@ where authority: &PublicationAuthority, condition: &PublishCondition, expected_workspace_incarnation_id: Option, + append_intent_digest: Option, object_plan: &ArtifactUploadPlan, staged_objects: &[StagedObject], manifest_rows: &[ArtifactManifestRow], @@ -394,6 +397,26 @@ where bytes: &[u8], upload_stats: &mut ArtifactUploadStats, ) -> Result { + let fail = |known_token, stage, source| { + if append_intent_digest.is_some() { + publication_failure_without_abort(stage, source) + } else { + self.failed_publication(logical_shard, operation_id, known_token, stage, source) + } + }; + let fail_or_resume = |token, stage, source| { + if append_intent_digest.is_some() { + publication_failure_without_abort(stage, source) + } else { + self.failed_or_resumable_publication( + logical_shard, + operation_id, + token, + stage, + source, + ) + } + }; let seals = seal_artifact_publish_plan(artifact_revision_id, staged_objects, manifest_rows)?; descriptor.validate()?; @@ -406,6 +429,7 @@ where authority: *authority, condition: *condition, expected_workspace_incarnation_id, + append_intent_digest, staged_object_count: seals.staged_object_count, staged_object_seal: seals.staged_object_seal, manifest_row_count: seals.manifest_row_count, @@ -419,13 +443,7 @@ where let status = match validated_publish_status(call.value, operation_id) { Ok(status) => status, Err(source) => { - return Err(self.failed_publication( - logical_shard, - operation_id, - None, - ArtifactPublishStage::Begin, - source, - )); + return Err(fail(None, ArtifactPublishStage::Begin, source)); } }; // Begin observes whichever durable state the operation row is @@ -441,13 +459,7 @@ where let value = match published_result_from_status(&status) { Ok(value) => value, Err(source) => { - return Err(self.failed_publication( - logical_shard, - operation_id, - None, - ArtifactPublishStage::Begin, - source, - )); + return Err(fail(None, ArtifactPublishStage::Begin, source)); } }; // The replay branch is the one path the engine takes @@ -458,7 +470,9 @@ where // live state before returning a replayed terminal // result; publish does the same rather than handing // back a generation the caller could CAS against. - if matches!(authority, PublicationAuthority::Visible) { + if append_intent_digest.is_none() + && matches!(authority, PublicationAuthority::Visible) + { if let Err(source) = self.confirm_replayed_publication_is_live(logical_shard, &value) { @@ -502,13 +516,7 @@ where ) { Ok(resume) => (resume.token, resume, replayed), Err(source) => { - return Err(self.failed_publication( - logical_shard, - operation_id, - None, - ArtifactPublishStage::Begin, - source, - )); + return Err(fail(None, ArtifactPublishStage::Begin, source)); } }, } @@ -536,9 +544,7 @@ where Ok(status) => match running_publish_token(status.value, operation_id) { Ok(next_token) => token = next_token, Err(source) => { - return Err(self.failed_publication( - logical_shard, - operation_id, + return Err(fail( Some(token), ArtifactPublishStage::StageObjects, source, @@ -546,9 +552,7 @@ where } }, Err(source) => { - return Err(self.failed_or_resumable_publication( - logical_shard, - operation_id, + return Err(fail_or_resume( token, ArtifactPublishStage::StageObjects, source, @@ -561,9 +565,7 @@ where let upload = match upload_artifact_from_plan(store, object_plan, bytes) { Ok(upload) => upload, Err(source) => { - return Err(self.failed_or_resumable_publication( - logical_shard, - operation_id, + return Err(fail_or_resume( token, ArtifactPublishStage::UploadObjects, ClientError::ArtifactUpload(Box::new(source)), @@ -588,9 +590,7 @@ where Ok(status) => match running_publish_token(status.value, operation_id) { Ok(next_token) => token = next_token, Err(source) => { - return Err(self.failed_publication( - logical_shard, - operation_id, + return Err(fail( Some(token), ArtifactPublishStage::MarkObjectsUploaded, source, @@ -598,9 +598,7 @@ where } }, Err(source) => { - return Err(self.failed_or_resumable_publication( - logical_shard, - operation_id, + return Err(fail_or_resume( token, ArtifactPublishStage::MarkObjectsUploaded, source, @@ -623,9 +621,7 @@ where Ok(status) => match running_publish_token(status.value, operation_id) { Ok(next_token) => token = next_token, Err(source) => { - return Err(self.failed_publication( - logical_shard, - operation_id, + return Err(fail( Some(token), ArtifactPublishStage::StageManifest, source, @@ -633,9 +629,7 @@ where } }, Err(source) => { - return Err(self.failed_or_resumable_publication( - logical_shard, - operation_id, + return Err(fail_or_resume( token, ArtifactPublishStage::StageManifest, source, @@ -658,18 +652,14 @@ where Err(source) => match self.recover_completed_publish(logical_shard, operation_id) { Ok(Some(recovered)) => recovered, Ok(None) => { - return Err(self.failed_or_resumable_publication( - logical_shard, - operation_id, + return Err(fail_or_resume( token, ArtifactPublishStage::Complete, source, )); } Err(recovery_error) => { - return Err(self.failed_or_resumable_publication( - logical_shard, - operation_id, + return Err(fail_or_resume( token, ArtifactPublishStage::Complete, recovery_error, @@ -712,6 +702,8 @@ where operation_id, artifact_revision_id, delta, + None, + None, ) { Err(error) if is_append_retry_error(&error) && attempt + 1 < self.max_attempts() => @@ -730,14 +722,18 @@ where unreachable!("validated max_attempts is non-zero") } - fn append_artifact_attempt( + #[allow(clippy::too_many_arguments)] + pub(crate) fn append_artifact_attempt( &self, store: &dyn ArtifactObjectStore, options: &ArtifactAppendOptions, operation_id: OperationIdentity, artifact_revision_id: ArtifactRevisionIdentity, delta: &[u8], + expected_workspace_incarnation_id: Option, + append_intent_digest: Option, ) -> Result { + require_provider_admission(store, options.block_size)?; let route = self.resolve_artifact_route()?; require_object_namespace(store, route)?; let logical_shard = route.logical_shard_id; @@ -971,7 +967,8 @@ where options.target.clone(), PublicationAuthority::Visible, condition, - None, + expected_workspace_incarnation_id, + append_intent_digest, object_plan, staged_objects, manifest_rows, @@ -2495,7 +2492,8 @@ fn is_append_retry_error(error: &ClientError) -> bool { | ClientError::MissingCapabilities(_) | ClientError::ArtifactIntegrity(_) | ClientError::Object(_) - | ClientError::ArtifactUpload(_) => false, + | ClientError::ArtifactUpload(_) + | ClientError::AppendUnresolved { .. } => false, } } @@ -2923,10 +2921,21 @@ mod tests { Some(status.clone()) } + fn publish_preparation(&self) -> Option> { + let begin = self.begin.as_ref().expect("publication has begun"); + Some(Box::new(nokv_protocol::PublishPreparation { + append_intent_digest: begin.append_intent_digest, + target: begin.target.clone(), + workspace_incarnation_id: WorkspaceIdentity([9; 16]), + artifact_revision_id: begin.artifact_revision_id, + })) + } + fn running_status(&mut self, operation_id: OperationIdentity) -> OperationStatus { OperationStatus { token: self.next_token(operation_id), kind: OperationKind::ArtifactPublish, + publish_preparation: self.publish_preparation(), commit_preparation: None, restore_preparation: None, state: OperationState::Running, @@ -3027,6 +3036,7 @@ mod tests { let status = OperationStatus { token: state.next_token(stage.token.operation_id), kind: OperationKind::ArtifactPublish, + publish_preparation: state.publish_preparation(), commit_preparation: None, restore_preparation: None, state: OperationState::Running, @@ -3211,6 +3221,7 @@ mod tests { let status = OperationStatus { token: state.next_token(begin.operation_id), kind: OperationKind::ArtifactPublish, + publish_preparation: state.publish_preparation(), commit_preparation: None, restore_preparation: None, state: OperationState::Succeeded, @@ -3248,6 +3259,7 @@ mod tests { let status = OperationStatus { token: state.next_token(abort.token.operation_id), kind: OperationKind::ArtifactPublish, + publish_preparation: state.publish_preparation(), commit_preparation: None, restore_preparation: None, state: OperationState::Aborting, @@ -4188,6 +4200,12 @@ mod tests { state_digest: Digest([0x55; 32]), }, kind: OperationKind::ArtifactPublish, + publish_preparation: Some(Box::new(nokv_protocol::PublishPreparation { + append_intent_digest: None, + target: target(), + workspace_incarnation_id: WorkspaceIdentity([9; 16]), + artifact_revision_id: ArtifactRevisionIdentity([8; 16]), + })), commit_preparation: None, restore_preparation: None, state: OperationState::Running, diff --git a/crates/nokv-client/src/error.rs b/crates/nokv-client/src/error.rs index 9c32514a2..2ad7b9f21 100644 --- a/crates/nokv-client/src/error.rs +++ b/crates/nokv-client/src/error.rs @@ -6,7 +6,9 @@ use std::fmt; use nokv_object::{ArtifactUploadFailure, ObjectError}; -use nokv_protocol::{ErrorCode, ProtocolError, RpcFailure, WorkspaceCapability}; +use nokv_protocol::{ + ErrorCode, OperationIdentity, OperationState, ProtocolError, RpcFailure, WorkspaceCapability, +}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct TransportError { @@ -67,6 +69,13 @@ pub enum ClientError { source: Box, abort_failure: Option>, }, + /// The stable append identity must be retained until its recorded outcome + /// is understood. A missing state is not proof that publication is absent. + AppendUnresolved { + operation_id: OperationIdentity, + state: Option, + source: Box, + }, RetryExhausted { attempts: u32, last_error: Box, @@ -86,6 +95,9 @@ impl ClientError { .. } => abort_failure.is_none() && source.retryable(), Self::RetryExhausted { last_error, .. } => last_error.retryable(), + // Generic retry loops must not turn an unresolved append into a + // fresh operation. The caller can explicitly retry the same intent. + Self::AppendUnresolved { .. } => false, Self::ArtifactReadFenceChanged => true, Self::InvalidOptions(_) | Self::InvalidRoute(_) @@ -100,6 +112,7 @@ impl ClientError { match self { Self::Rpc(failure) => Some(failure), Self::ArtifactPublishFailed { source, .. } => source.rpc_failure(), + Self::AppendUnresolved { source, .. } => source.rpc_failure(), Self::RetryExhausted { last_error, .. } => last_error.rpc_failure(), _ => None, } @@ -175,6 +188,14 @@ impl fmt::Display for ClientError { formatter, "workspace request exhausted {attempts} attempts: {last_error}" ), + Self::AppendUnresolved { + operation_id, + state, + source, + } => write!( + formatter, + "append operation {operation_id:?} has state {state:?}: {source}; retain the same operation identity" + ), } } } @@ -200,6 +221,7 @@ impl std::error::Error for ClientError { Self::Object(error) => Some(error), Self::ArtifactUpload(error) => Some(error.as_ref()), Self::ArtifactPublishFailed { source, .. } => Some(source.as_ref()), + Self::AppendUnresolved { source, .. } => Some(source.as_ref()), Self::RetryExhausted { last_error, .. } => Some(last_error.as_ref()), Self::InvalidOptions(_) | Self::InvalidRoute(_) diff --git a/crates/nokv-client/src/idempotent_append.rs b/crates/nokv-client/src/idempotent_append.rs new file mode 100644 index 000000000..922db6dc9 --- /dev/null +++ b/crates/nokv-client/src/idempotent_append.rs @@ -0,0 +1,482 @@ +// Copyright 2024-2026 The NoKV Authors. +// SPDX-License-Identifier: Apache-2.0 + +//! Cross-process append identity over the existing publication lifecycle. +//! +//! One operation owns one publication attempt. An admitted but unfinished +//! attempt is never replaced or aborted merely because another caller retries. + +use nokv_object::{ArtifactObjectStore, DEFAULT_ARTIFACT_BLOCK_SIZE}; +use nokv_protocol::{ + ArtifactRevisionIdentity, ConflictKind, ContentType, Digest, ErrorCode, GetOperationRequest, + OperationIdentity, OperationKind, OperationResult, OperationState, OperationStatus, + PublishResult, RootIdentity, RpcFailure, WorkspaceIdentity, WorkspacePath, +}; +use sha2::{Digest as _, Sha256}; + +use crate::{ + ArtifactAppendOptions, ClientCall, ClientError, RouteResolver, RpcTransport, WorkspaceClient, +}; + +/// An append intent whose identity the caller persists before the first send. +/// +/// Content-type inheritance and explicit overrides are distinct intents. The +/// workspace incarnation is an owner-checked fence, not an authentication token. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IdempotentAppendOptions { + pub operation_id: OperationIdentity, + pub target: WorkspacePath, + pub expected_workspace_incarnation_id: WorkspaceIdentity, + pub create_content_type: ContentType, + pub content_type: Option, + pub block_size: usize, + pub max_logical_size: Option, +} + +impl IdempotentAppendOptions { + pub fn new( + operation_id: OperationIdentity, + target: WorkspacePath, + expected_workspace_incarnation_id: WorkspaceIdentity, + create_content_type: ContentType, + ) -> Self { + Self { + operation_id, + target, + expected_workspace_incarnation_id, + create_content_type, + content_type: None, + block_size: DEFAULT_ARTIFACT_BLOCK_SIZE, + max_logical_size: None, + } + } + + pub fn with_content_type(mut self, content_type: ContentType) -> Self { + self.content_type = Some(content_type); + self + } + + pub fn with_block_size(mut self, block_size: usize) -> Self { + self.block_size = block_size; + self + } + + pub fn with_max_logical_size(mut self, max_logical_size: u64) -> Self { + self.max_logical_size = Some(max_logical_size); + self + } +} + +impl WorkspaceClient +where + Transport: RpcTransport, + Resolver: RouteResolver, +{ + /// Append at most once for one caller-owned identity and exact intent. + /// + /// A successful replay returns the original publication receipt even when + /// another writer has since replaced or removed the path. It does not + /// assert that the original revision is still the live head or retained. + /// + /// A running, failed, quarantined or uncertain operation is returned as + /// `AppendUnresolved`. Retrying preserves the identity; it never allocates + /// another append attempt or interprets a timeout as proof of absence. + pub fn append_artifact_idempotent( + &self, + store: &dyn ArtifactObjectStore, + options: IdempotentAppendOptions, + delta: &[u8], + ) -> Result, ClientError> { + let intent = append_intent_digest(self.root_id(), &options, delta)?; + let revision = append_revision_identity(self.root_id(), &options); + match self.get_operation(GetOperationRequest { + operation_id: options.operation_id, + }) { + Ok(status) => return append_receipt(status, &options, intent, revision), + Err(error) if error.rpc_code() == Some(ErrorCode::NotFound) => {} + Err(error) => return Err(unresolved(&options, None, error)), + } + + // No durable attempt was observed. Racing callers still use the exact + // same operation/revision: server-side admission decides the winner. + // Unlike legacy append, there is no sequence of potentially empty + // attempt IDs that independent callers could fill out of order. + let mut publication = ArtifactAppendOptions::new( + options.operation_id, + revision, + options.target.clone(), + options.create_content_type.clone(), + ) + .with_block_size(options.block_size); + publication.content_type = options.content_type.clone(); + publication.max_logical_size = options.max_logical_size; + match self.append_artifact_attempt( + store, + &publication, + options.operation_id, + revision, + delta, + Some(options.expected_workspace_incarnation_id), + Some(intent), + ) { + Ok(outcome) => Ok(outcome.publication), + Err(source) => { + // In particular, a Begin race may have lost to the same + // intent, and Complete may have committed before losing its + // response. Query before interpreting either as failure. + match self.get_operation(GetOperationRequest { + operation_id: options.operation_id, + }) { + Ok(status) => append_receipt(status, &options, intent, revision), + Err(error) if error.rpc_code() == Some(ErrorCode::NotFound) => { + Err(unresolved(&options, None, source)) + } + Err(error) => Err(unresolved(&options, None, error)), + } + } + } + } +} + +fn append_receipt( + status: ClientCall, + options: &IdempotentAppendOptions, + intent: Digest, + revision: ArtifactRevisionIdentity, +) -> Result, ClientError> { + let value = status.value; + let matches = value.token.operation_id == options.operation_id + && value.kind == OperationKind::ArtifactPublish + && value + .publish_preparation + .as_ref() + .is_some_and(|preparation| { + preparation.append_intent_digest == Some(intent) + && preparation.target == options.target + && preparation.workspace_incarnation_id + == options.expected_workspace_incarnation_id + && preparation.artifact_revision_id == revision + }); + if !matches { + return Err(unresolved( + options, + None, + append_failure( + ErrorCode::RequestReplayMismatch, + "append operation identity is already bound to a different intent or lifecycle", + ), + )); + } + if value.state != OperationState::Succeeded { + let message = value.failure.map_or_else( + || { + format!( + "append publication is {:?}; query or retry this same identity", + value.state + ) + }, + |failure| failure.message, + ); + return Err(unresolved( + options, + Some(value.state), + append_failure( + if value.state == OperationState::Running { + ErrorCode::Conflict + } else { + ErrorCode::OperationFailed + }, + message, + ), + )); + } + let Some(OperationResult::ArtifactPublish(result)) = value.result else { + return Err(unresolved( + options, + Some(value.state), + ClientError::ResponseMismatch( + "successful append has no publication receipt".to_owned(), + ), + )); + }; + if result.operation_id != options.operation_id + || result.artifact_revision_id != revision + || result.target != options.target + { + return Err(unresolved( + options, + Some(value.state), + ClientError::ResponseMismatch( + "append receipt differs from its bound intent".to_owned(), + ), + )); + } + Ok(ClientCall { + value: result, + commit_version: status.commit_version, + replayed: true, + }) +} + +fn unresolved( + options: &IdempotentAppendOptions, + state: Option, + source: ClientError, +) -> ClientError { + ClientError::AppendUnresolved { + operation_id: options.operation_id, + state, + source: Box::new(source), + } +} + +fn append_failure(code: ErrorCode, message: impl Into) -> ClientError { + ClientError::Rpc(RpcFailure { + code, + message: message.into(), + retryable: false, + conflict: Some(ConflictKind::OperationState), + current_generation: None, + route_hint: None, + }) +} + +fn append_intent_digest( + root: RootIdentity, + options: &IdempotentAppendOptions, + delta: &[u8], +) -> Result { + let block_size = u64::try_from(options.block_size) + .map_err(|_| ClientError::InvalidOptions("append block size exceeds u64".to_owned()))?; + let delta_len = u64::try_from(delta.len()) + .map_err(|_| ClientError::InvalidOptions("append delta length exceeds u64".to_owned()))?; + let mut hasher = Sha256::new(); + hasher.update(b"nokv.append.intent.v1\0"); + hasher.update(root.0); + hasher.update(options.operation_id.0); + hasher.update(options.expected_workspace_incarnation_id.0); + for field in [ + options.target.workbench.as_str(), + options.target.path.as_str(), + options.create_content_type.as_str(), + ] { + hasher.update((field.len() as u64).to_be_bytes()); + hasher.update(field.as_bytes()); + } + match &options.content_type { + Some(content_type) => { + hasher.update([1]); + hasher.update((content_type.as_str().len() as u64).to_be_bytes()); + hasher.update(content_type.as_str().as_bytes()); + } + None => hasher.update([0]), + } + hasher.update(block_size.to_be_bytes()); + match options.max_logical_size { + Some(maximum) => { + hasher.update([1]); + hasher.update(maximum.to_be_bytes()); + } + None => hasher.update([0]), + } + hasher.update(delta_len.to_be_bytes()); + hasher.update(Sha256::digest(delta)); + Ok(Digest(hasher.finalize().into())) +} + +fn append_revision_identity( + root: RootIdentity, + options: &IdempotentAppendOptions, +) -> ArtifactRevisionIdentity { + let mut hasher = Sha256::new(); + hasher.update(b"nokv.append.stable-revision.v1\0"); + hasher.update(root.0); + hasher.update(options.operation_id.0); + let digest = hasher.finalize(); + let mut revision = [0; 16]; + revision.copy_from_slice(&digest[..16]); + ArtifactRevisionIdentity(revision) +} + +#[cfg(test)] +mod tests { + use super::*; + use nokv_protocol::{ + OperationProgress, OperationToken, PublishPreparation, RelativePath, WorkbenchName, + }; + + fn options() -> IdempotentAppendOptions { + IdempotentAppendOptions::new( + OperationIdentity([1; 16]), + WorkspacePath { + workbench: WorkbenchName::new("run-42").unwrap(), + path: RelativePath::new("logs/events.jsonl").unwrap(), + }, + WorkspaceIdentity([2; 16]), + ContentType::new("text/plain").unwrap(), + ) + } + + fn successful_status( + options: &IdempotentAppendOptions, + delta: &[u8], + ) -> ClientCall { + let root = RootIdentity([3; 16]); + let revision = append_revision_identity(root, options); + ClientCall { + commit_version: None, + replayed: false, + value: OperationStatus { + token: OperationToken { + operation_id: options.operation_id, + state_digest: Digest([9; 32]), + }, + kind: OperationKind::ArtifactPublish, + publish_preparation: Some(Box::new(PublishPreparation { + append_intent_digest: Some(append_intent_digest(root, options, delta).unwrap()), + target: options.target.clone(), + workspace_incarnation_id: options.expected_workspace_incarnation_id, + artifact_revision_id: revision, + })), + commit_preparation: None, + restore_preparation: None, + state: OperationState::Succeeded, + progress: OperationProgress { + completed_rows: 1, + total_rows: Some(1), + completed_bytes: delta.len() as u64, + total_bytes: Some(delta.len() as u64), + }, + result: Some(OperationResult::ArtifactPublish(PublishResult { + operation_id: options.operation_id, + target: options.target.clone(), + workspace_revision: 7, + generation: 4, + artifact_revision_id: revision, + logical_size: 100, + body_digest: nokv_protocol::sha256_digest_uri(Digest([4; 32])), + })), + failure: None, + }, + } + } + + #[test] + fn persisted_append_identity_has_frozen_v1_encoding() { + // Existing durable intents must remain recoverable after SDK changes. + // This vector fixes option tags, string lengths, byte order and domains. + let options = options(); + let root = RootIdentity([3; 16]); + assert_eq!( + append_intent_digest(root, &options, b"event\n").unwrap(), + Digest([ + 252, 30, 220, 186, 179, 149, 245, 9, 148, 196, 201, 191, 224, 64, 103, 147, 49, + 197, 154, 223, 18, 234, 168, 87, 99, 0, 52, 10, 45, 175, 183, 3, + ]) + ); + assert_eq!( + append_revision_identity(root, &options), + ArtifactRevisionIdentity([ + 138, 32, 231, 5, 12, 133, 71, 175, 10, 237, 44, 92, 128, 156, 4, 168, + ]) + ); + } + + #[test] + fn historical_receipt_preserves_original_result_without_live_metadata() { + let options = options(); + let status = successful_status(&options, b"event\n"); + let expected = status.value.result.clone(); + let receipt = append_receipt( + status, + &options, + append_intent_digest(RootIdentity([3; 16]), &options, b"event\n").unwrap(), + append_revision_identity(RootIdentity([3; 16]), &options), + ) + .unwrap(); + assert!(receipt.replayed); + assert_eq!( + Some(OperationResult::ArtifactPublish(receipt.value)), + expected + ); + } + + #[test] + fn existing_identity_rejects_every_changed_caller_constraint() { + let original = options().with_max_logical_size(1024); + let status = successful_status(&original, b"event\n"); + let mut changed = Vec::new(); + let mut value = original.clone(); + value.target.path = RelativePath::new("logs/other.jsonl").unwrap(); + changed.push(value); + let mut value = original.clone(); + value.target.workbench = WorkbenchName::new("run-43").unwrap(); + changed.push(value); + let mut value = original.clone(); + value.expected_workspace_incarnation_id = WorkspaceIdentity([5; 16]); + changed.push(value); + let mut value = original.clone(); + value.create_content_type = ContentType::new("application/json").unwrap(); + changed.push(value); + changed.push( + original + .clone() + .with_content_type(ContentType::new("text/plain").unwrap()), + ); + changed.push(original.clone().with_block_size(64)); + changed.push(original.clone().with_max_logical_size(2048)); + let mut value = original.clone(); + value.max_logical_size = None; + changed.push(value); + for options in changed { + let error = append_receipt( + status.clone(), + &options, + append_intent_digest(RootIdentity([3; 16]), &options, b"event\n").unwrap(), + append_revision_identity(RootIdentity([3; 16]), &options), + ) + .unwrap_err(); + assert_eq!(error.rpc_code(), Some(ErrorCode::RequestReplayMismatch)); + assert!(!error.retryable()); + } + let error = append_receipt( + status, + &original, + append_intent_digest(RootIdentity([3; 16]), &original, b"other\n").unwrap(), + append_revision_identity(RootIdentity([3; 16]), &original), + ) + .unwrap_err(); + assert_eq!(error.rpc_code(), Some(ErrorCode::RequestReplayMismatch)); + } + + #[test] + fn unfinished_or_failed_operation_never_becomes_a_new_attempt() { + let options = options(); + for state in [ + OperationState::Running, + OperationState::Aborting, + OperationState::Failed, + OperationState::Quarantined, + ] { + let mut status = successful_status(&options, b"event\n"); + status.value.state = state; + status.value.result = None; + let error = append_receipt( + status, + &options, + append_intent_digest(RootIdentity([3; 16]), &options, b"event\n").unwrap(), + append_revision_identity(RootIdentity([3; 16]), &options), + ) + .unwrap_err(); + assert!(matches!( + error, + ClientError::AppendUnresolved { + operation_id, + state: Some(observed), + .. + } if observed == state && operation_id == options.operation_id + )); + assert!(!error.retryable()); + } + } +} diff --git a/crates/nokv-client/src/lib.rs b/crates/nokv-client/src/lib.rs index b08814338..596710249 100644 --- a/crates/nokv-client/src/lib.rs +++ b/crates/nokv-client/src/lib.rs @@ -12,6 +12,7 @@ mod artifact; mod error; mod generic_index; +mod idempotent_append; mod route; mod sdk; mod snapshot_workflow; @@ -30,6 +31,7 @@ pub use error::{ArtifactPublishStage, ClientError, TransportError}; pub use generic_index::{ GenericIndexAbortOutcome, GenericIndexRegistrationOutcome, GenericIndexRegistrationPlan, }; +pub use idempotent_append::IdempotentAppendOptions; #[cfg(feature = "control")] pub use route::ControlRouteResolver; #[cfg(feature = "etcd")] diff --git a/crates/nokv-client/src/workbench_workflow.rs b/crates/nokv-client/src/workbench_workflow.rs index b79eaf660..d66e27f49 100644 --- a/crates/nokv-client/src/workbench_workflow.rs +++ b/crates/nokv-client/src/workbench_workflow.rs @@ -2115,6 +2115,7 @@ mod tests { state_digest: Digest([8; 32]), }, kind: OperationKind::Commit, + publish_preparation: None, commit_preparation: Some(Box::new(preparation)), restore_preparation: None, state: if result.is_some() { @@ -2178,6 +2179,12 @@ mod tests { state_digest: Digest([0x18; 32]), }, kind: OperationKind::ArtifactPublish, + publish_preparation: Some(Box::new(nokv_protocol::PublishPreparation { + append_intent_digest: None, + target: options.manifest_target.clone(), + workspace_incarnation_id: binding.workspace_incarnation_id, + artifact_revision_id: binding.artifact_revision_id, + })), commit_preparation: None, restore_preparation: None, state: OperationState::Succeeded, @@ -2431,6 +2438,7 @@ mod tests { state_digest: Digest([10; 32]), }, kind: OperationKind::Restore, + publish_preparation: None, commit_preparation: None, restore_preparation: Some(Box::new(restore_operation_preparation( options, @@ -2455,6 +2463,7 @@ mod tests { state_digest: Digest([10; 32]), }, kind: OperationKind::Restore, + publish_preparation: None, commit_preparation: None, restore_preparation: Some(Box::new(restore_operation_preparation( options, diff --git a/crates/nokv-meta/src/workspace/codec.rs b/crates/nokv-meta/src/workspace/codec.rs index 65b20a348..b0e6b0bb4 100644 --- a/crates/nokv-meta/src/workspace/codec.rs +++ b/crates/nokv-meta/src/workspace/codec.rs @@ -24,7 +24,7 @@ const GENERIC_INDEX_APPEND_RECEIPT_DISCRIMINATOR: u8 = 3; const FIXED_ID_BYTES: usize = 16; pub(crate) const SYSTEM_SCHEMA_KEY: &[u8] = b"schema"; -const SYSTEM_FORMAT_VERSION: u32 = 10; +const SYSTEM_FORMAT_VERSION: u32 = 11; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum SchemaMarkerVersion { @@ -1308,7 +1308,7 @@ mod tests { #[test] fn schema_marker_is_exact_and_versioned() { - assert_eq!(SYSTEM_FORMAT_VERSION, 10); + assert_eq!(SYSTEM_FORMAT_VERSION, 11); let marker = encode_schema_marker(); assert_eq!(marker[0], VALUE_FORMAT_VERSION); assert_eq!( @@ -1321,7 +1321,7 @@ mod tests { SchemaMarkerVersion::Current ); - let previous_layout = encode_schema_marker_version(9); + let previous_layout = encode_schema_marker_version(10); assert_eq!( validate_schema_marker(&previous_layout), Err(SchemaMarkerError) diff --git a/crates/nokv-meta/src/workspace/commit.rs b/crates/nokv-meta/src/workspace/commit.rs index db088d558..18c9b24fc 100644 --- a/crates/nokv-meta/src/workspace/commit.rs +++ b/crates/nokv-meta/src/workspace/commit.rs @@ -542,6 +542,15 @@ impl<'a> CommitService<'a> { .encode(); let mut plan = CommandPlan::default(); + // Exclude publish/restore identities atomically with this build's + // admission so GetOperation can never become ambiguous after success. + for kind in [OperationKind::Publish, OperationKind::Restore] { + plan.assert_value( + MetadataFamily::Operation, + super::codec::operation_key(request.context.root_id, kind, request.operation_id), + None, + )?; + } plan.put_absent( MetadataFamily::Operation, operation_key, diff --git a/crates/nokv-meta/src/workspace/publication.rs b/crates/nokv-meta/src/workspace/publication.rs index 932c4a056..5a7906792 100644 --- a/crates/nokv-meta/src/workspace/publication.rs +++ b/crates/nokv-meta/src/workspace/publication.rs @@ -1239,8 +1239,15 @@ fn next_generation( fn publish_identity_digest(operation: &PublishOperationRecord) -> [u8; SHA256_BYTES] { let mut hasher = Sha256::new(); - hasher.update(b"nokv.publish.operation.identity.v3\0"); + hasher.update(b"nokv.publish.operation.identity.v4\0"); hasher.update(operation.operation_id.as_bytes()); + match operation.append_intent_digest { + Some(digest) => { + hasher.update([1]); + hasher.update(digest); + } + None => hasher.update([0]), + } hasher.update(operation.initiating_owner_epoch.get().to_be_bytes()); match operation.authority { PublishAuthority::Visible => hasher.update([1]), @@ -1296,7 +1303,14 @@ fn publish_identity_digest(operation: &PublishOperationRecord) -> [u8; SHA256_BY fn publish_initialization_digest(operation: &PublishOperationRecord) -> [u8; SHA256_BYTES] { let mut hasher = Sha256::new(); - hasher.update(b"nokv.publish.operation.initialization.v3\0"); + hasher.update(b"nokv.publish.operation.initialization.v4\0"); + match operation.append_intent_digest { + Some(digest) => { + hasher.update([1]); + hasher.update(digest); + } + None => hasher.update([0]), + } hasher.update(operation.initiating_owner_epoch.get().to_be_bytes()); match operation.authority { PublishAuthority::Visible => hasher.update([1]), @@ -1662,6 +1676,20 @@ impl PublicationService<'_> { } let operation_payload = request.operation.encode()?; let mut plan = CommandPlan::default(); + // Public operation lookup has one root-scoped identity across all + // three lifecycle kinds. Fence competing admissions in this same + // transaction, rather than trusting a prior lookup by the client. + for kind in [OperationKind::BuildCommit, OperationKind::Restore] { + plan.assert_value( + MetadataFamily::Operation, + super::codec::operation_key( + request.context.root_id, + kind, + request.operation.operation_id, + ), + None, + )?; + } plan.put_absent( MetadataFamily::Operation, operation_key, @@ -4495,6 +4523,7 @@ mod tests { ) -> PublishOperationRecord { let dependencies = Vec::new(); let mut operation = PublishOperationRecord { + append_intent_digest: None, operation_id, identity_digest: [0; SHA256_BYTES], initialization_digest: [0; SHA256_BYTES], @@ -5390,6 +5419,52 @@ mod tests { ); } + #[test] + fn stable_append_identity_binds_all_256_intent_bits_and_cannot_drop_binding() { + let mut counter = 1; + let store = ready_store(&mut counter); + let service = PublicationService::new(&store); + let revision_id = revision(91_090); + let staged = staged_rows(revision_id, 1); + let manifest = manifest_rows(&staged); + let mut initial = publish_operation( + operation_id(91_090), + revision_id, + path("outputs/intent.bin"), + PublishClaim::CreateOnly, + &staged, + &manifest, + ); + initial.append_intent_digest = Some([0x61; SHA256_BYTES]); + seal_publish_operation(&mut initial); + let begun = begin_operation(&service, &store, &mut counter, initial.clone()); + assert_eq!(begun.append_intent_digest, initial.append_intent_digest); + let mut changed_digest = [0x61; SHA256_BYTES]; + changed_digest[SHA256_BYTES - 1] = 0x62; + for intent in [Some(changed_digest), None] { + let mut changed = initial.clone(); + changed.append_intent_digest = intent; + seal_publish_operation(&mut changed); + assert_ne!(changed.identity_digest, initial.identity_digest); + assert_ne!(changed.initialization_digest, initial.initialization_digest); + assert_eq!( + service.begin_publish(BeginPublishRequest { + context: publication_context(&store, &mut counter), + operation: changed, + }), + Err(PublicationError::OperationInputMismatch) + ); + } + let replay = service + .begin_publish(BeginPublishRequest { + context: publication_context(&store, &mut counter), + operation: initial, + }) + .unwrap(); + assert!(replay.replayed); + assert_eq!(replay.operation, begun); + } + #[test] fn successor_epoch_cannot_borrow_visible_or_commit_staging_operation_identity() { for (index, authority) in [ @@ -6731,7 +6806,7 @@ mod tests { assert_eq!(replaced.result.path_generation, Generation::new(2).unwrap()); let replace_bytes = capture.with_last_commit(crate::workspace::test_support::transaction_bytes); - assert_eq!(replace_bytes, 11_799_434); + assert_eq!(replace_bytes, 11_799_442); let removed = remove_path( &store, @@ -6910,7 +6985,7 @@ mod tests { assert_eq!( capture.with_last_commit(crate::workspace::test_support::transaction_bytes), - 9_860_707 + 9_860_715 ); } diff --git a/crates/nokv-meta/src/workspace/publish_operation_records.rs b/crates/nokv-meta/src/workspace/publish_operation_records.rs index 3d9603776..2af2067e5 100644 --- a/crates/nokv-meta/src/workspace/publish_operation_records.rs +++ b/crates/nokv-meta/src/workspace/publish_operation_records.rs @@ -19,7 +19,7 @@ use nokv_types::{ }; /// Durable value format for publication-owned payloads. -pub const PUBLISH_VALUE_FORMAT_VERSION: u8 = 4; +pub const PUBLISH_VALUE_FORMAT_VERSION: u8 = 5; /// Hard safety bound for one publish operation's staged-object ledger. pub const MAX_STAGED_OBJECTS: u32 = 1_048_576; @@ -140,6 +140,8 @@ pub struct PublishTerminalError { /// and an exact-identity initialization mismatch remain distinguishable. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PublishOperationRecord { + /// Full caller intent commitment for one stable append publication attempt. + pub append_intent_digest: Option<[u8; SHA256_BYTES]>, pub operation_id: OperationId, pub identity_digest: [u8; SHA256_BYTES], pub initialization_digest: [u8; SHA256_BYTES], @@ -445,6 +447,15 @@ impl std::error::Error for PublishRecordError {} impl PublishOperationRecord { pub fn validate(&self) -> Result<(), PublishRecordError> { + if self.append_intent_digest.is_some() + && (!matches!(self.authority, PublishAuthority::Visible) + || matches!(self.claim, PublishClaim::ReplaceOnly { .. })) + { + return Err(PublishRecordError::InvalidPhasePayload { + phase: self.phase, + reason: "stable append requires visible create-only or append authority", + }); + } if self.activity_deadline_ms == 0 { return Err(PublishRecordError::ZeroScalar { field: "activity_deadline_ms", @@ -687,6 +698,7 @@ impl PublishOperationRecord { encoded.extend_from_slice(self.operation_id.as_bytes()); encoded.extend_from_slice(&self.identity_digest); encoded.extend_from_slice(&self.initialization_digest); + push_optional_fixed(&mut encoded, &self.append_intent_digest); encoded.extend_from_slice(&self.initiating_owner_epoch.get().to_be_bytes()); encoded.extend_from_slice(&self.activity_deadline_ms.to_be_bytes()); match self.authority { @@ -754,6 +766,7 @@ impl PublishOperationRecord { let operation_id = OperationId::from_bytes(decoder.fixed("operation_id")?); let identity_digest = decoder.fixed("identity_digest")?; let initialization_digest = decoder.fixed("initialization_digest")?; + let append_intent_digest = decoder.optional_fixed("append_intent_digest")?; let initiating_owner_epoch = OwnerEpoch::new(decoder.u64("initiating_owner_epoch")?) .map_err(|_| PublishRecordError::ZeroScalar { field: "initiating_owner_epoch", @@ -838,6 +851,7 @@ impl PublishOperationRecord { operation_id, identity_digest, initialization_digest, + append_intent_digest, initiating_owner_epoch, activity_deadline_ms, authority, @@ -1797,6 +1811,7 @@ mod tests { fn uploading_operation() -> PublishOperationRecord { PublishOperationRecord { + append_intent_digest: None, operation_id: OperationId::from_bytes([0x10; 16]), identity_digest: [0x11; SHA256_BYTES], initialization_digest: [0x12; SHA256_BYTES], @@ -1885,6 +1900,7 @@ mod tests { &[0x10; 16], &[0x11; SHA256_BYTES], &[0x12; SHA256_BYTES], + &[0], &5_u64.to_be_bytes(), &50_000_u64.to_be_bytes(), &[1], @@ -1932,6 +1948,21 @@ mod tests { assert_trailing_byte_is_rejected(expected, PublishOperationRecord::decode); } + #[test] + fn stable_append_intent_round_trips_without_truncation_and_rejects_old_codec() { + let mut record = uploading_operation(); + let mut digest = [0x61; SHA256_BYTES]; + digest[SHA256_BYTES - 1] = 0x62; + record.append_intent_digest = Some(digest); + let mut encoded = record.encode().unwrap(); + assert_eq!(PublishOperationRecord::decode(&encoded).unwrap(), record); + encoded[0] = PUBLISH_VALUE_FORMAT_VERSION - 1; + assert!(matches!( + PublishOperationRecord::decode(&encoded), + Err(PublishRecordError::UnsupportedValueVersion { .. }) + )); + } + #[test] fn staged_object_codec_has_frozen_golden_bytes() { let record = StagedObjectRecord { @@ -2377,6 +2408,7 @@ mod tests { + 16 + SHA256_BYTES + SHA256_BYTES + + 1 // absent append_intent_digest tag + 8 + 8 + 1 @@ -2441,6 +2473,7 @@ mod tests { + 16 + SHA256_BYTES + SHA256_BYTES + + 1 // absent append_intent_digest tag + 8 + 8 + 1 diff --git a/crates/nokv-meta/src/workspace/restore.rs b/crates/nokv-meta/src/workspace/restore.rs index 0b61802d3..0cc36c1fc 100644 --- a/crates/nokv-meta/src/workspace/restore.rs +++ b/crates/nokv-meta/src/workspace/restore.rs @@ -1186,6 +1186,16 @@ pub fn begin_restore( owning_operation_id: Some(operation_id), }; let mut plan = CommandPlan::default(); + // Claim the root-scoped public operation identity in the same command as + // the hidden destination and source holds. A competing kind cannot make + // the eventual receipt unqueryable. + for kind in [OperationKind::Publish, OperationKind::BuildCommit] { + plan.assert_value( + MetadataFamily::Operation, + super::codec::operation_key(context.root_id, kind, operation_id), + None, + )?; + } plan.put_absent( MetadataFamily::WorkspaceIncarnationClaim, destination_claim_key, @@ -8177,6 +8187,146 @@ mod tests { )); } + #[test] + fn public_operation_identity_is_atomic_across_all_six_admission_orders() { + use super::super::build_commit_records::CommitManifestCondition; + use super::super::commit::{BeginBuildCommitRequest, CommitService}; + for (first, second) in [ + (OperationKind::Publish, OperationKind::BuildCommit), + (OperationKind::Publish, OperationKind::Restore), + (OperationKind::BuildCommit, OperationKind::Publish), + (OperationKind::BuildCommit, OperationKind::Restore), + (OperationKind::Restore, OperationKind::Publish), + (OperationKind::Restore, OperationKind::BuildCommit), + ] { + let mut counter = 0; + let owner_epoch = owner(1); + let store = crate::workspace::test_support::memory(shard()).unwrap(); + activate_root(&store, &mut counter, owner_epoch); + let source = seed_source(&store, &mut counter, owner_epoch, 1); + let snapshot = mint_source_snapshot(&store, &mut counter, owner_epoch, &source); + let restore = snapshot_restore_request( + &source, + snapshot, + "identity-restore", + incarnation(41), + 0xd1, + ); + let build_workbench = workbench("identity-build"); + let build_incarnation = incarnation(42); + create_visible_workspace( + &store, + write_context(&store, &mut counter, owner_epoch), + &build_workbench, + build_incarnation, + ) + .unwrap(); + let publish = wire_publish_operation( + restore.operation_id, + revision(43), + &source.source_workbench, + source.source_incarnation, + NormalizedRelativePath::new("outputs/identity.bin").unwrap(), + PublishAuthority::Visible, + owner_epoch, + &[], + &[], + ); + let mut admit = |kind| -> Result<(), String> { + match kind { + OperationKind::Publish => PublicationService::new(&store) + .begin_publish(BeginPublishRequest { + context: publication_context(&store, &mut counter, owner_epoch), + operation: publish.clone(), + }) + .map(|_| ()) + .map_err(|error| error.to_string()), + OperationKind::BuildCommit => CommitService::new(&store) + .begin_build(BeginBuildCommitRequest { + context: write_context(&store, &mut counter, owner_epoch), + operation_id: restore.operation_id, + workbench_id: build_workbench.clone(), + expected_source_workspace_incarnation_id: build_incarnation, + commit_id: CommitId::from_bytes([44; SHA256_BYTES]), + content_digest_uri: "sha256:identity-content".to_owned(), + manifest_digest_uri: "sha256:identity-manifest".to_owned(), + projection_input_digest: [45; SHA256_BYTES], + tree_manifest_revision_id: revision(46), + replace: false, + run_manifest_condition: CommitManifestCondition::CreateOnly, + committed_at_unix_seconds: 1, + expected_head_generation: None, + producer: None, + lineage_projection: Vec::new(), + parent_commits: Vec::new(), + }) + .map(|_| ()) + .map_err(|error| error.to_string()), + OperationKind::Restore => begin_restore( + &store, + write_context(&store, &mut counter, owner_epoch), + &restore, + ) + .map(|_| ()) + .map_err(|error| error.to_string()), + _ => unreachable!(), + } + }; + admit(first).unwrap(); + // Every request is also exercised as a successful first admission + // in another order. The rejection oracle is the unchanged durable + // identity and absence of destination side effects below, not the + // wording used by each lifecycle to report a predicate conflict. + admit(second) + .expect_err("another lifecycle must not claim the first operation identity"); + let read_version = store.current_read_version().unwrap(); + for kind in [ + OperationKind::Publish, + OperationKind::BuildCommit, + OperationKind::Restore, + ] { + let row = store + .read_at( + root(), + placement(), + owner_epoch, + MetadataFamily::Operation, + &operation_key(root(), kind, restore.operation_id), + read_version, + ) + .unwrap(); + assert_eq!( + row.is_some(), + kind == first, + "unexpected {kind:?} row after {first:?} / {second:?}" + ); + } + if second == OperationKind::Restore { + assert!(get_visible_workspace_at( + &store, + read_context(&store, owner_epoch), + &restore.destination_workbench_id + ) + .unwrap() + .is_none()); + assert!(store + .read_at( + root(), + placement(), + owner_epoch, + MetadataFamily::WorkspaceIncarnationClaim, + &workspace_incarnation_claim_key( + root(), + restore.destination_workspace_incarnation_id + ), + read_version + ) + .unwrap() + .is_none()); + } + } + } + #[test] fn mutated_snapshot_seals_against_base_commit_then_late_binds_exact_destination() { let mut counter = 0_u128; @@ -10616,6 +10766,7 @@ mod tests { manifest: &[ManifestRowInput], ) -> PublishOperationRecord { let mut operation = PublishOperationRecord { + append_intent_digest: None, operation_id, identity_digest: [0; SHA256_BYTES], initialization_digest: [0; SHA256_BYTES], diff --git a/crates/nokv-protocol/src/artifact_publish.rs b/crates/nokv-protocol/src/artifact_publish.rs index 13018ca1a..a997ca777 100644 --- a/crates/nokv-protocol/src/artifact_publish.rs +++ b/crates/nokv-protocol/src/artifact_publish.rs @@ -16,7 +16,7 @@ pub const MAX_ARTIFACT_PUBLISH_OBJECTS: usize = 1_048_576; /// Maximum provider-neutral manifest rows returned by one range-plan page. pub const MAX_ARTIFACT_READ_PLAN_ROWS: usize = 512; -const PUBLICATION_RECORD_ENCODING: u8 = 4; +const PUBLICATION_RECORD_ENCODING: u8 = 5; /// Canonical closure committed by `BeginArtifactPublish`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/nokv-protocol/src/codec.rs b/crates/nokv-protocol/src/codec.rs index ae9e8c38f..3646820ea 100644 --- a/crates/nokv-protocol/src/codec.rs +++ b/crates/nokv-protocol/src/codec.rs @@ -10,7 +10,7 @@ use crate::request::WorkspaceRpcRequest; use crate::response::WorkspaceRpcResponse; /// The exact and only accepted wire schema. -pub const WORKSPACE_PROTOCOL_SCHEMA: &str = "nokv.workspace.rpc.v9"; +pub const WORKSPACE_PROTOCOL_SCHEMA: &str = "nokv.workspace.rpc.v10"; /// Exact schema for the versioned workspace RPC preflight exchange. pub const WORKSPACE_PREFLIGHT_SCHEMA: &str = "nokv.workspace.rpc_preflight.v1"; /// Exact schema for the advertised workspace RPC capability set. @@ -238,6 +238,7 @@ mod tests { route: route(), request_id: RequestIdentity([3; 16]), operation: WorkspaceRequest::BeginArtifactPublish(crate::BeginArtifactPublishRequest { + append_intent_digest: None, operation_id: OperationIdentity([1; 16]), artifact_revision_id: crate::ArtifactRevisionIdentity([2; 16]), target: crate::WorkspacePath { @@ -271,7 +272,7 @@ mod tests { #[test] fn request_round_trips_with_exact_schema() { - assert_eq!(WORKSPACE_PROTOCOL_SCHEMA, "nokv.workspace.rpc.v9"); + assert_eq!(WORKSPACE_PROTOCOL_SCHEMA, "nokv.workspace.rpc.v10"); let expected = request(); let encoded = encode_request(&expected).unwrap(); assert!(encoded @@ -281,7 +282,7 @@ mod tests { } #[test] - fn get_path_read_version_fence_has_one_exact_v9_encoding() { + fn get_path_read_version_fence_has_one_exact_v10_encoding() { let expected = WorkspaceRpcRequest { route: route(), request_id: RequestIdentity([0x41; 16]), @@ -303,15 +304,15 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(&encoded)), [ - 152, 96, 133, 203, 129, 19, 138, 5, 38, 11, 167, 93, 209, 135, 43, 102, 106, 192, - 203, 92, 203, 127, 210, 2, 97, 154, 198, 134, 15, 231, 117, 227, + 43, 107, 167, 115, 219, 115, 94, 117, 176, 8, 147, 194, 210, 216, 56, 161, 64, 151, + 61, 250, 74, 122, 42, 225, 63, 43, 174, 194, 243, 61, 141, 87 ], - "update only for an intentional GetPath v9 wire change" + "update only for an intentional GetPath v10 wire change" ); } #[test] - fn artifact_v1_query_and_catalog_keep_one_exact_v9_encoding() { + fn artifact_v1_query_and_catalog_keep_one_exact_v10_encoding() { let search = WorkspaceRpcRequest { route: route(), request_id: RequestIdentity([0x3a; 16]), @@ -429,10 +430,10 @@ mod tests { assert_eq!( <[u8; 32]>::from(golden.finalize()), [ - 243, 187, 50, 41, 238, 93, 202, 53, 218, 51, 99, 43, 64, 54, 43, 228, 160, 117, - 242, 236, 190, 145, 147, 61, 209, 66, 62, 59, 60, 147, 224, 126, + 81, 145, 52, 165, 175, 120, 94, 153, 211, 189, 77, 0, 195, 136, 57, 177, 110, 117, + 113, 179, 217, 251, 63, 40, 249, 32, 75, 95, 159, 40, 215, 86 ], - "update only for an intentional ArtifactV1 v9 wire change" + "update only for an intentional ArtifactV1 v10 wire change" ); assert_eq!(decode_request(&encoded_search).unwrap(), search); assert_eq!( @@ -446,7 +447,7 @@ mod tests { } #[test] - fn generic_exact_catalog_path_match_has_one_exact_v9_encoding() { + fn generic_exact_catalog_path_match_has_one_exact_v10_encoding() { let expected = WorkspaceRpcRequest { route: route(), request_id: RequestIdentity([0x42; 16]), @@ -473,15 +474,15 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(&encoded)), [ - 22, 120, 49, 175, 132, 202, 191, 110, 171, 159, 138, 140, 243, 119, 170, 1, 197, - 111, 140, 115, 97, 239, 209, 181, 137, 224, 15, 0, 208, 54, 47, 150, + 5, 71, 115, 115, 113, 40, 164, 143, 127, 183, 2, 223, 147, 69, 79, 150, 111, 60, + 236, 152, 105, 119, 195, 187, 140, 187, 68, 58, 27, 49, 232, 246 ], - "update only for an intentional Catalog Exact v9 wire change" + "update only for an intentional Catalog Exact v10 wire change" ); } #[test] - fn generic_registration_surface_has_one_exact_v9_encoding() { + fn generic_registration_surface_has_one_exact_v10_encoding() { let begin = generic_begin_request(); let append = generic_append_request(); let finalize = WorkspaceRpcRequest { @@ -591,10 +592,10 @@ mod tests { assert_eq!( <[u8; 32]>::from(golden.finalize()), [ - 80, 250, 26, 67, 213, 86, 32, 15, 233, 192, 206, 142, 129, 138, 254, 229, 187, 230, - 57, 139, 111, 100, 36, 160, 85, 219, 244, 164, 59, 206, 141, 68, + 13, 220, 44, 77, 152, 142, 98, 27, 59, 110, 212, 142, 89, 211, 249, 131, 116, 211, + 160, 182, 25, 228, 109, 196, 47, 129, 80, 211, 206, 153, 180, 240 ], - "update only for an intentional Generic registration v9 wire change" + "update only for an intentional Generic registration v10 wire change" ); for request in [&begin, &append, &finalize, &abort, &get] { assert_eq!( @@ -778,7 +779,7 @@ mod tests { } #[test] - fn generic_namespace_search_rows_round_trip_and_validate_under_v9() { + fn generic_namespace_search_rows_round_trip_and_validate_under_v10() { let response = WorkspaceRpcResponse { route: route(), request_id: RequestIdentity([0x43; 16]), @@ -863,7 +864,7 @@ mod tests { } #[test] - fn rename_path_request_and_result_round_trip_under_v9() { + fn rename_path_request_and_result_round_trip_under_v10() { let source = WorkspacePath { workbench: WorkbenchName::new("run-42").unwrap(), path: RelativePath::new("outputs/a.bin").unwrap(), @@ -936,6 +937,7 @@ mod tests { ); let status = OperationStatus { + publish_preparation: None, token: OperationToken { operation_id: commit.operation_id, state_digest: Digest([0x18; 32]), @@ -971,7 +973,7 @@ mod tests { } #[test] - fn restore_v9_late_binding_and_source_manifest_read_round_trip() { + fn restore_v10_late_binding_and_source_manifest_read_round_trip() { let restore_identity = RestoreManifestIdentity { publication_operation_id: OperationIdentity([0x21; 16]), artifact_revision_id: ArtifactRevisionIdentity([0x22; 16]), @@ -999,10 +1001,10 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(&encoded_prepare)), [ - 226, 134, 226, 39, 198, 246, 100, 40, 82, 81, 12, 226, 107, 121, 36, 218, 21, 88, - 4, 251, 18, 245, 47, 185, 77, 61, 156, 189, 133, 10, 82, 238, + 101, 32, 165, 198, 170, 16, 166, 173, 232, 230, 81, 59, 208, 210, 146, 92, 30, 202, + 123, 26, 0, 200, 90, 148, 136, 164, 207, 94, 196, 99, 243, 84 ], - "update only for an intentional restore-v9 wire change" + "update only for an intentional restore-v10 wire change" ); let bind = WorkspaceRpcRequest { @@ -1114,7 +1116,7 @@ mod tests { source_manifest ); - let mut complete_v9_golden = Sha256::new(); + let mut complete_v10_golden = Sha256::new(); for encoded in [ encoded_prepare, encoded_bind, @@ -1122,16 +1124,16 @@ mod tests { encoded_prepared, encoded_source_manifest, ] { - complete_v9_golden.update((encoded.len() as u64).to_be_bytes()); - complete_v9_golden.update(encoded); + complete_v10_golden.update((encoded.len() as u64).to_be_bytes()); + complete_v10_golden.update(encoded); } assert_eq!( - <[u8; 32]>::from(complete_v9_golden.finalize()), + <[u8; 32]>::from(complete_v10_golden.finalize()), [ - 192, 172, 119, 203, 234, 108, 69, 6, 60, 113, 123, 111, 217, 92, 150, 200, 118, 26, - 191, 109, 229, 182, 216, 245, 50, 111, 123, 229, 72, 78, 20, 83, + 168, 93, 200, 28, 36, 10, 187, 93, 244, 3, 225, 144, 3, 71, 3, 129, 161, 143, 153, + 15, 144, 210, 198, 248, 244, 207, 80, 42, 204, 238, 190, 79 ], - "update only for an intentional restore-v9 wire change" + "update only for an intentional restore-v10 wire change" ); } @@ -1259,7 +1261,7 @@ mod tests { decode_request(&encoded), Err(ProtocolError::SchemaMismatch { actual: "nokv.workspace.rpc.v8".to_owned(), - expected: "nokv.workspace.rpc.v9", + expected: "nokv.workspace.rpc.v10", }) ); @@ -1312,7 +1314,7 @@ mod tests { decode_request(&actual_v4), Err(ProtocolError::SchemaMismatch { actual: "nokv.workspace.rpc.v4".to_owned(), - expected: "nokv.workspace.rpc.v9", + expected: "nokv.workspace.rpc.v10", }) ); @@ -1340,17 +1342,18 @@ mod tests { decode_response(&encoded), Err(ProtocolError::SchemaMismatch { actual: "nokv.workspace.rpc.v4".to_owned(), - expected: "nokv.workspace.rpc.v9", + expected: "nokv.workspace.rpc.v10", }) ); } #[test] - fn v6_through_v8_are_rejected_before_zero_payload_dispatch() { + fn v6_through_v9_are_rejected_before_zero_payload_dispatch() { for schema in [ "nokv.workspace.rpc.v6", "nokv.workspace.rpc.v7", "nokv.workspace.rpc.v8", + "nokv.workspace.rpc.v9", ] { let encoded = rmp_serde::to_vec_named(&Frame { schema: schema.to_owned(), @@ -1453,8 +1456,8 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(encode_request(&expected).unwrap())), [ - 31, 24, 197, 114, 99, 88, 106, 26, 248, 214, 147, 168, 109, 245, 95, 31, 88, 31, - 166, 130, 127, 59, 54, 215, 115, 133, 136, 208, 224, 225, 195, 17, + 133, 126, 69, 128, 33, 243, 208, 203, 199, 178, 159, 93, 19, 148, 84, 149, 172, 95, + 182, 51, 79, 160, 109, 23, 199, 183, 137, 14, 146, 112, 191, 126 ], "update only for an intentional capability-v2 wire change" ); diff --git a/crates/nokv-protocol/src/handshake.rs b/crates/nokv-protocol/src/handshake.rs index 6ec11752b..781e70d65 100644 --- a/crates/nokv-protocol/src/handshake.rs +++ b/crates/nokv-protocol/src/handshake.rs @@ -223,19 +223,19 @@ fn validate_schema(schema: &str) -> Result<(), HandshakeError> { mod tests { use super::*; - const V9: &str = "nokv.workspace.rpc.v9"; + const V10: &str = "nokv.workspace.rpc.v10"; #[test] fn client_hello_has_one_canonical_fixed_width_encoding() { - let hello = WorkspaceHandshake::new(HandshakeKind::ClientHello, V9).unwrap(); + let hello = WorkspaceHandshake::new(HandshakeKind::ClientHello, V10).unwrap(); let encoded = encode_handshake_frame(&hello).unwrap(); let mut expected = [0_u8; HANDSHAKE_FRAME_BYTES]; expected[..4].copy_from_slice(&(HANDSHAKE_PAYLOAD_BYTES as u32).to_be_bytes()); expected[4..12].copy_from_slice(b"NOKVHS1\0"); expected[12] = 1; - expected[13] = V9.len() as u8; - expected[20..20 + V9.len()].copy_from_slice(V9.as_bytes()); + expected[13] = V10.len() as u8; + expected[20..20 + V10.len()].copy_from_slice(V10.as_bytes()); assert_eq!(encoded, expected); assert_eq!(decode_handshake_frame(&encoded).unwrap(), hello); @@ -248,7 +248,7 @@ mod tests { HandshakeKind::Accepted, HandshakeKind::Incompatible, ] { - let expected = WorkspaceHandshake::new(kind, V9).unwrap(); + let expected = WorkspaceHandshake::new(kind, V10).unwrap(); let encoded = encode_handshake_payload(&expected).unwrap(); assert_eq!(decode_handshake_payload(&encoded).unwrap(), expected); } @@ -257,7 +257,7 @@ mod tests { #[test] fn decoder_rejects_every_noncanonical_fixed_field() { let canonical = encode_handshake_payload( - &WorkspaceHandshake::new(HandshakeKind::ClientHello, V9).unwrap(), + &WorkspaceHandshake::new(HandshakeKind::ClientHello, V10).unwrap(), ) .unwrap(); @@ -299,7 +299,7 @@ mod tests { ); let mut nonzero_padding = canonical; - nonzero_padding[16 + V9.len()] = 1; + nonzero_padding[16 + V10.len()] = 1; assert_eq!( decode_handshake_payload(&nonzero_padding), Err(HandshakeError::NonzeroPadding) @@ -308,7 +308,7 @@ mod tests { #[test] fn frame_decoder_requires_exact_length_prefix_and_payload_size() { - let hello = WorkspaceHandshake::new(HandshakeKind::ClientHello, V9).unwrap(); + let hello = WorkspaceHandshake::new(HandshakeKind::ClientHello, V10).unwrap(); let canonical = encode_handshake_frame(&hello).unwrap(); assert_eq!( diff --git a/crates/nokv-protocol/src/lib.rs b/crates/nokv-protocol/src/lib.rs index 8eab8f035..ad36bcf0e 100644 --- a/crates/nokv-protocol/src/lib.rs +++ b/crates/nokv-protocol/src/lib.rs @@ -65,12 +65,12 @@ pub use response::{ GenericIndexAppendResult, GenericIndexRegistrationPhase, GenericIndexRegistrationStatus, GenericNamespaceArtifact, GenericNamespaceHit, GenericNamespaceKind, OperationProgress, OperationResult, OperationState, OperationStatus, PathListEntry, PathPage, PathReadResult, - PublishResult, RemovePathResult, RenamePathResult, RestoreDestinationBinding, - RestoreDestinationManifestBindings, RestoreManifestBinding, RestoreOperationPreparation, - RestorePreparation, RestoreResult, RestoreSourceCommitBinding, SearchHit, SearchResult, - SearchRow, SnapshotPage, SnapshotResult, SnapshotStatus, WorkspacePreflightResult, - WorkspaceResult, WorkspaceRpcOutcome, WorkspaceRpcResponse, WorkspaceSummary, - WorkspaceSummaryWithCommit, + PublishPreparation, PublishResult, RemovePathResult, RenamePathResult, + RestoreDestinationBinding, RestoreDestinationManifestBindings, RestoreManifestBinding, + RestoreOperationPreparation, RestorePreparation, RestoreResult, RestoreSourceCommitBinding, + SearchHit, SearchResult, SearchRow, SnapshotPage, SnapshotResult, SnapshotStatus, + WorkspacePreflightResult, WorkspaceResult, WorkspaceRpcOutcome, WorkspaceRpcResponse, + WorkspaceSummary, WorkspaceSummaryWithCommit, }; pub use types::{ AppendSegment, ArtifactDescriptor, ArtifactManifestRow, ArtifactRevisionIdentity, ByteRange, diff --git a/crates/nokv-protocol/src/request.rs b/crates/nokv-protocol/src/request.rs index 9a71e925c..1d11394f3 100644 --- a/crates/nokv-protocol/src/request.rs +++ b/crates/nokv-protocol/src/request.rs @@ -425,6 +425,8 @@ impl RemovePathRequest { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct BeginArtifactPublishRequest { + /// Exact caller intent for a stable append; binds one publication attempt. + pub append_intent_digest: Option, pub operation_id: OperationIdentity, pub artifact_revision_id: ArtifactRevisionIdentity, pub target: WorkspacePath, @@ -449,6 +451,22 @@ pub struct BeginArtifactPublishRequest { impl BeginArtifactPublishRequest { fn validate(&self) -> Result<(), ProtocolError> { self.condition.validate()?; + if self.append_intent_digest.is_some() + && (!matches!(self.authority, PublicationAuthority::Visible) + || self.expected_workspace_incarnation_id.is_none() + || !matches!( + self.condition, + PublishCondition::CreateOnly + | PublishCondition::Append { + expected_generation: Some(_) + } + )) + { + return Err(ProtocolError::invalid( + "begin_artifact_publish.append_intent_digest", + "stable append requires visible authority, an incarnation fence, and create-only or generation-bound append", + )); + } if self.expected_workspace_incarnation_id.is_some() && !matches!(self.authority, PublicationAuthority::Visible) { @@ -2163,6 +2181,7 @@ mod tests { condition: PublishCondition, ) -> BeginArtifactPublishRequest { BeginArtifactPublishRequest { + append_intent_digest: None, operation_id: OperationIdentity([1; 16]), artifact_revision_id: ArtifactRevisionIdentity([2; 16]), target: WorkspacePath { @@ -2208,6 +2227,34 @@ mod tests { )); } + #[test] + fn stable_append_requires_a_visible_incarnation_and_generation_bound_plan() { + let mut request = begin_manifest_publish( + "outputs/events.log", + PublicationAuthority::Visible, + PublishCondition::CreateOnly, + ); + request.append_intent_digest = Some(Digest([0x61; 32])); + assert!(request.validate().is_err()); + request.expected_workspace_incarnation_id = Some(WorkspaceIdentity([9; 16])); + request.validate().unwrap(); + request.condition = PublishCondition::Append { + expected_generation: Some(7), + }; + request.validate().unwrap(); + for condition in [ + PublishCondition::Append { + expected_generation: None, + }, + PublishCondition::ReplaceOnly { + expected_generation: 7, + }, + ] { + request.condition = condition; + assert!(request.validate().is_err()); + } + } + #[test] fn get_path_separates_metadata_only_reads_from_bounded_plan_pages() { get_path(None, None).validate().unwrap(); diff --git a/crates/nokv-protocol/src/response.rs b/crates/nokv-protocol/src/response.rs index d8fdebb2e..5613f4793 100644 --- a/crates/nokv-protocol/src/response.rs +++ b/crates/nokv-protocol/src/response.rs @@ -1519,11 +1519,23 @@ impl OperationResult { } } +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct PublishPreparation { + /// Caller intent bound to one append publication attempt, when present. + pub append_intent_digest: Option, + pub target: WorkspacePath, + pub workspace_incarnation_id: WorkspaceIdentity, + pub artifact_revision_id: ArtifactRevisionIdentity, +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct OperationStatus { pub token: OperationToken, pub kind: OperationKind, + /// Present exactly for artifact-publication operations, including terminal ones. + pub publish_preparation: Option>, /// Present exactly for commit operations, including terminal ones. pub commit_preparation: Option>, /// Present exactly for restore operations, including terminal ones. @@ -1537,6 +1549,33 @@ pub struct OperationStatus { impl OperationStatus { fn validate(&self) -> Result<(), ProtocolError> { self.progress.validate()?; + match (self.kind, self.publish_preparation.as_ref()) { + (OperationKind::ArtifactPublish, Some(preparation)) => { + if let Some(OperationResult::ArtifactPublish(result)) = &self.result { + if result.target != preparation.target + || result.artifact_revision_id != preparation.artifact_revision_id + { + return Err(ProtocolError::invalid( + "operation.publish_preparation", + "must match the durable publication result", + )); + } + } + } + (OperationKind::ArtifactPublish, None) => { + return Err(ProtocolError::invalid( + "operation.publish_preparation", + "is required for artifact-publication operations", + )); + } + (_, Some(_)) => { + return Err(ProtocolError::invalid( + "operation.publish_preparation", + "is only valid for artifact-publication operations", + )); + } + (_, None) => {} + } match (self.kind, self.commit_preparation.as_ref()) { (OperationKind::Commit, Some(preparation)) => preparation.validate()?, (OperationKind::Commit, None) => { @@ -2297,6 +2336,17 @@ mod tests { fn running_status(kind: OperationKind) -> OperationStatus { OperationStatus { + publish_preparation: (kind == OperationKind::ArtifactPublish).then(|| { + Box::new(PublishPreparation { + append_intent_digest: None, + target: WorkspacePath { + workbench: WorkbenchName::new("run").unwrap(), + path: crate::RelativePath::new("outputs/result.bin").unwrap(), + }, + workspace_incarnation_id: WorkspaceIdentity([3; 16]), + artifact_revision_id: ArtifactRevisionIdentity([4; 16]), + }) + }), token: OperationToken { operation_id: OperationIdentity([1; 16]), state_digest: Digest([2; 32]), @@ -2452,6 +2502,32 @@ mod tests { assert!(publish.validate().is_err()); } + #[test] + fn publish_status_requires_preparation_bound_to_its_historical_result() { + let mut status = running_status(OperationKind::ArtifactPublish); + status.validate().unwrap(); + let preparation = status.publish_preparation.take().unwrap(); + assert!(status.validate().is_err()); + status.result = Some(OperationResult::ArtifactPublish(PublishResult { + operation_id: status.token.operation_id, + target: preparation.target.clone(), + workspace_revision: 7, + generation: 3, + artifact_revision_id: preparation.artifact_revision_id, + logical_size: 4, + body_digest: DigestUri::new(format!("sha256:{}", "ab".repeat(32))).unwrap(), + })); + status.state = OperationState::Succeeded; + status.publish_preparation = Some(preparation); + status.validate().unwrap(); + status + .publish_preparation + .as_mut() + .unwrap() + .artifact_revision_id = ArtifactRevisionIdentity([5; 16]); + assert!(status.validate().is_err()); + } + #[test] fn commit_preparation_requires_bounded_sorted_unique_parents() { let mut status = running_status(OperationKind::Commit); diff --git a/crates/nokv-python/README.md b/crates/nokv-python/README.md index b1bd63b86..f3d0daea1 100644 --- a/crates/nokv-python/README.md +++ b/crates/nokv-python/README.md @@ -62,6 +62,26 @@ not for installing the SDK. `expected`) with nothing written. Callers that omit the argument keep the 0.11.0 behaviour; a server older than 0.11.1 rejects a fenced request as an invalid argument instead of ignoring the fence. +- `Client.append_bytes(workbench, path, data, operation_id, ...)` appends under a + caller-owned stable identity. The workspace must exist. Pass + `expected_workspace_incarnation_id` to pin the intended workspace across + restarts; omission observes the existing incarnation and fences the owner + request with it. Reuse the same identity and inputs after a lost reply. + The result contains the original `operation_id`, `artifact_revision_id`, + `generation`, `workspace_revision`, `logical_size`, `body_digest`, and + `workspace_incarnation_id`, not the latest path head. `replayed` and nullable + `commit_version` are call metadata rather than stable receipt fields. + `content_type=None` inherits an existing artifact's type and uses + `application/octet-stream` on creation; an explicit type applies to both. + `max_logical_size` optionally caps the resulting body. One identity owns one + attempt; CAS conflicts are not retried under automatically changed identities. + `AppendError` carries `operation_id`, `state`, `code`, and the expected + incarnation in `expected`. `cause_code` preserves an underlying RPC code + such as `RequestReplayMismatch`, or is `None` for failures without an RPC + cause; `retryable` is false. This distinguishes a permanently mismatched + intent from an uncertain reply without parsing the error message. An unknown + state does not prove nothing was published. Incarnation conflicts raise `WorkspaceIncarnationMismatch` with + the append's `operation_id` attached. Recover using the original identity. - `WorkbenchFileSystem` is an fsspec compatibility adapter bound to one explicit Workbench. Paths must be one of `input`, `scripts`, `outputs`, `logs`, or `metadata`, optionally followed by an artifact-relative path. Sections and diff --git a/crates/nokv-python/python/nokv/__init__.py b/crates/nokv-python/python/nokv/__init__.py index fc6b686f3..6e79310f2 100644 --- a/crates/nokv-python/python/nokv/__init__.py +++ b/crates/nokv-python/python/nokv/__init__.py @@ -1,5 +1,7 @@ """Versioned Python SDK for NoKV Workbenches and immutable artifacts.""" +from __future__ import annotations + from importlib.metadata import PackageNotFoundError, version as _distribution_version from . import checkpoint @@ -18,6 +20,7 @@ __all__ = [ "API_VERSION", + "AppendError", "Client", "ObjectStoreConfig", "RoutingConfig", @@ -27,6 +30,37 @@ ] +class AppendError(RuntimeError): + """An append failed or needs recovery under its original operation identity. + + ``state`` is the observed durable operation state, or ``None`` when it is + unknown. Neither a timeout nor an unknown state proves that nothing was + published. Retry the same intent with the same ``operation_id``; do not + generate a replacement identity to bypass an unresolved operation. + ``cause_code`` preserves the underlying RPC code, for example + ``RequestReplayMismatch`` for a permanently mismatched intent, or ``None`` + when there is no RPC failure. ``retryable`` is always false: generic retry + handlers must not turn an unresolved operation into a new action. + """ + + def __init__( + self, + message: str, + operation_id: str, + state: str | None, + code: str, + expected: str | None, + cause_code: str | None = None, + ) -> None: + super().__init__(message) + self.operation_id = operation_id + self.state = state + self.code = code + self.expected = expected + self.cause_code = cause_code + self.retryable = False + + class WorkspaceIncarnationMismatch(RuntimeError): """A publish was refused because the workbench is not the expected incarnation. diff --git a/crates/nokv-python/src/client.rs b/crates/nokv-python/src/client.rs index 181c758a8..1862d8942 100644 --- a/crates/nokv-python/src/client.rs +++ b/crates/nokv-python/src/client.rs @@ -11,10 +11,10 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use nokv_client::{ ArtifactPublishOptions, ArtifactPublishOutcome, ArtifactRangeBatchRequest, ClientError, - ClientOptions, FramedTcpOptions, FramedTcpTransport, RouteResolver, SnapshotMintOptions, - SnapshotRenewOptions, SnapshotRetireOptions, WorkbenchCommitRequest, WorkbenchLifecycleFacade, - WorkbenchLifecycleOptions, WorkbenchRestoreOrigin, WorkbenchRestoreRequest, - WorkbenchRestoreSource, WorkbenchSnapshotSelector, WorkspaceClient, + ClientOptions, FramedTcpOptions, FramedTcpTransport, IdempotentAppendOptions, RouteResolver, + SnapshotMintOptions, SnapshotRenewOptions, SnapshotRetireOptions, WorkbenchCommitRequest, + WorkbenchLifecycleFacade, WorkbenchLifecycleOptions, WorkbenchRestoreOrigin, + WorkbenchRestoreRequest, WorkbenchRestoreSource, WorkbenchSnapshotSelector, WorkspaceClient, }; use nokv_object::ArtifactObjectStore; use nokv_protocol::{ @@ -41,9 +41,9 @@ use crate::object_store::{ConfiguredObjectStore, PythonObjectStoreConfig}; use crate::python_value::{ aggregate_result_to_py, catalog_result_to_py, find_workspaces_result_to_py, hex, parse_aggregates, parse_field_specs, parse_fixed_hex, parse_predicates, parse_sort, - path_metadata_to_py, path_page_to_py, publish_outcome_to_py, read_outcome_to_py, - search_result_to_py, snapshot_result_to_py, workspace_summary_to_py, PythonAggregateSpec, - PythonFieldSpec, PythonPredicateSpec, PythonSortSpec, + path_metadata_to_py, path_page_to_py, publish_outcome_to_py, publish_result_to_py, + read_outcome_to_py, search_result_to_py, snapshot_result_to_py, workspace_summary_to_py, + PythonAggregateSpec, PythonFieldSpec, PythonPredicateSpec, PythonSortSpec, }; use crate::routing::PythonRoutingConfig; @@ -499,6 +499,90 @@ impl PythonWorkspaceClient { Ok(dict) } + /// Append bytes once under a caller-owned operation identity. + /// + /// The workspace must already exist. Retain the identity and incarnation + /// across process restarts. Replays return the original publication receipt, + /// which may precede the current path head. Existing artifact metadata and + /// content type are inherited unless `content_type` explicitly overrides it. + #[pyo3(signature = ( + workbench, + path, + data, + operation_id, + content_type = None, + block_size = 4_194_304, + max_logical_size = None, + expected_workspace_incarnation_id = None + ))] + #[allow(clippy::too_many_arguments)] + fn append_bytes<'py>( + &self, + py: Python<'py>, + workbench: &str, + path: &str, + data: Vec, + operation_id: &str, + content_type: Option<&str>, + block_size: usize, + max_logical_size: Option, + expected_workspace_incarnation_id: Option<&str>, + ) -> PyResult> { + let operation_id = OperationIdentity(parse_fixed_hex("operation_id", operation_id)?); + let target = parse_workspace_path(workbench, path)?; + let content_type = content_type + .map(|value| ContentType::new(value.to_owned())) + .transpose() + .map_err(value_error)?; + let create_content_type = content_type + .clone() + .unwrap_or(ContentType::new("application/octet-stream").map_err(value_error)?); + let expected = expected_workspace_incarnation_id + .map(|value| { + parse_fixed_hex("expected_workspace_incarnation_id", value).map(WorkspaceIdentity) + }) + .transpose()?; + let client = Arc::clone(&self.client); + let objects = Arc::clone(&self.objects); + let (call, incarnation) = py + .detach(move || { + let incarnation = match expected { + Some(incarnation) => incarnation, + None => { + client + .get_workspace(GetWorkspaceRequest { + workbench: target.workbench.clone(), + }) + .map_err(|error| (Box::new(error), None))? + .value + .workspace_incarnation_id + } + }; + let mut options = IdempotentAppendOptions::new( + operation_id, + target, + incarnation, + create_content_type, + ) + .with_block_size(block_size); + if let Some(content_type) = content_type { + options = options.with_content_type(content_type); + } + if let Some(max_logical_size) = max_logical_size { + options = options.with_max_logical_size(max_logical_size); + } + client + .append_artifact_idempotent(objects.as_ref(), options, &data) + .map(|call| (call, incarnation)) + .map_err(|error| (Box::new(error), Some(incarnation))) + }) + .map_err(|(error, expected)| append_error(py, *error, operation_id, expected))?; + let dict = publish_result_to_py(py, &call.value)?; + dict.set_item("workspace_incarnation_id", hex(&incarnation.0))?; + set_call_metadata(&dict, call.commit_version, call.replayed)?; + Ok(dict) + } + /// Publish bytes with create-only or explicit generation-CAS replacement. /// /// `index_fields` entries are `(field_id, scalar_kind, encoded_value)`. @@ -1480,10 +1564,50 @@ fn publish_error(py: Python<'_>, error: ClientError, expected: Option<&str>) -> client_error(error) } +fn append_error( + py: Python<'_>, + error: ClientError, + operation_id: OperationIdentity, + expected: Option, +) -> PyErr { + let identity = hex(&operation_id.0); + let expected = expected.map(|value| hex(&value.0)); + if expected.is_some() + && client_error_failure(&error).is_some_and(|failure| { + failure.conflict == Some(nokv_protocol::ConflictKind::WorkspaceIncarnation) + }) + { + let mapped = publish_error(py, error, expected.as_deref()); + let _ = mapped.value(py).setattr("operation_id", identity); + return mapped; + } + let state = match &error { + ClientError::AppendUnresolved { state, .. } => state.map(|state| format!("{state:?}")), + _ => None, + }; + let cause_code = client_error_failure(&error).map(|failure| format!("{:?}", failure.code)); + let code = if matches!(&error, ClientError::AppendUnresolved { .. }) { + "AppendUnresolved".to_owned() + } else { + client_error_failure(&error) + .map(|failure| format!("{:?}", failure.code)) + .unwrap_or_else(|| "AppendFailed".to_owned()) + }; + let message = error.to_string(); + py.import("nokv") + .and_then(|module| module.getattr("AppendError")) + .and_then(|class| { + class.call1((message.clone(), identity, state, code, expected, cause_code)) + }) + .map(PyErr::from_value) + .unwrap_or_else(|_| PyRuntimeError::new_err(message)) +} + fn client_error_failure(error: &ClientError) -> Option<&nokv_protocol::RpcFailure> { match error { ClientError::Rpc(failure) => Some(failure), ClientError::ArtifactPublishFailed { source, .. } + | ClientError::AppendUnresolved { source, .. } | ClientError::RetryExhausted { last_error: source, .. } => client_error_failure(source), @@ -1505,6 +1629,7 @@ fn client_error_code(error: &ClientError) -> Option { match error { ClientError::Rpc(failure) => Some(failure.code), ClientError::ArtifactPublishFailed { source, .. } + | ClientError::AppendUnresolved { source, .. } | ClientError::RetryExhausted { last_error: source, .. } => client_error_code(source), diff --git a/crates/nokv-python/tests/test_api_surface.py b/crates/nokv-python/tests/test_api_surface.py index 385489ac7..2311d6060 100644 --- a/crates/nokv-python/tests/test_api_surface.py +++ b/crates/nokv-python/tests/test_api_surface.py @@ -9,6 +9,7 @@ def test_versioned_workbench_surface(): assert nokv.API_VERSION == 1 assert nokv.__all__ == [ "API_VERSION", + "AppendError", "Client", "ObjectStoreConfig", "RoutingConfig", @@ -29,6 +30,12 @@ def test_versioned_workbench_surface(): assert hasattr(nokv.Client, "rename") assert hasattr(nokv.Client, "publish_bytes") assert hasattr(nokv.Client, "publish_file") + assert hasattr(nokv.Client, "append_bytes") + append = inspect.signature(nokv.Client.append_bytes).parameters + assert append["operation_id"].default is inspect.Parameter.empty + assert append["content_type"].default is None + assert append["expected_workspace_incarnation_id"].default is None + assert append["max_logical_size"].default is None for method in (nokv.Client.publish_bytes, nokv.Client.publish_file): parameters = inspect.signature(method).parameters assert "expected_workspace_incarnation_id" in parameters @@ -84,3 +91,22 @@ def test_retired_filesystem_types_stay_absent(): def test_torch_adapter_is_lazy_and_optional(): assert "torch" not in nokv.__all__ + + +def test_append_error_keeps_identity_and_observed_state(): + error = nokv.AppendError("reply lost", "a" * 32, None, "AppendUnresolved", "b" * 32) + assert isinstance(error, RuntimeError) + assert error.operation_id == "a" * 32 + assert error.state is None + assert error.code == "AppendUnresolved" + assert error.expected == "b" * 32 + assert error.cause_code is None + assert error.retryable is False + assert str(error) == "reply lost" + mismatch = nokv.AppendError( + "different intent", "a" * 32, None, "AppendUnresolved", "b" * 32, + "RequestReplayMismatch", + ) + assert mismatch.cause_code == "RequestReplayMismatch" + assert mismatch.code == "AppendUnresolved" + assert mismatch.retryable is False diff --git a/crates/nokv-server/src/executor.rs b/crates/nokv-server/src/executor.rs index 1f4e2db71..776467419 100644 --- a/crates/nokv-server/src/executor.rs +++ b/crates/nokv-server/src/executor.rs @@ -1899,6 +1899,49 @@ impl MetadataWorkspaceRequestExecutor { } let context = self.publication_context(rpc.route, step_id)?; let read = read_context_from_publication(context); + // Stable append identity authenticates the original intent before any + // live path or dependency read. A receipt describes historical success; + // a later writer or expired historical artifact cannot revoke it. + if let Some(existing) = + self.find_publish_operation(rpc.route, read.read_version, request.operation_id)? + { + if existing.append_intent_digest.is_some() || request.append_intent_digest.is_some() { + if existing.append_intent_digest + != request.append_intent_digest.map(|value| value.0) + || existing.authority != meta::PublishAuthority::Visible + || request.authority != protocol::PublicationAuthority::Visible + || existing.workbench_id != workbench_id(&request.target.workbench)? + || existing.path != relative_path(&request.target.path)? + || Some(existing.workspace_incarnation_id.into()) + != request.expected_workspace_incarnation_id + || existing.artifact_revision_id + != types::ArtifactRevisionId::from(request.artifact_revision_id) + { + return Err(failure( + protocol::ErrorCode::RequestReplayMismatch, + "append operation identity is already bound to a different intent or target", + false, + None, + )); + } + if existing.phase == types::PublishPhase::Published { + return publish_operation_response(meta::PublishCommandOutcome { + commit_version: types::CommitVersion::new(read.read_version.get()) + .expect("publication contexts have a non-zero read version"), + operation: existing, + replayed: true, + }); + } + // No alternate plan, attempt, owner takeover, or cleanup is + // authorized by a retry. GetOperation reports pending/terminal + // state; the existing lifecycle retains recovery authority. + return Err(conflict( + protocol::ConflictKind::OperationState, + "append operation is already admitted; query its durable state before retrying", + None, + )); + } + } let (dependency_owner_revision_ids, dependency_depth, dependency_digest) = self .publication_dependencies( read, @@ -2049,6 +2092,7 @@ impl MetadataWorkspaceRequestExecutor { } }; let mut operation = meta::PublishOperationRecord { + append_intent_digest: request.append_intent_digest.map(|value| value.0), operation_id: request.operation_id.into(), identity_digest: [0; types::SHA256_BYTES], initialization_digest: [0; types::SHA256_BYTES], @@ -4187,6 +4231,12 @@ fn publish_operation_status( ), }; Ok(protocol::OperationStatus { + publish_preparation: Some(Box::new(protocol::PublishPreparation { + append_intent_digest: operation.append_intent_digest.map(protocol::Digest), + target: workspace_path(&operation.workbench_id, &operation.path)?, + workspace_incarnation_id: operation.workspace_incarnation_id.into(), + artifact_revision_id: operation.artifact_revision_id.into(), + })), token: protocol::OperationToken { operation_id: operation.operation_id.into(), state_digest: publish_state_digest(operation)?, @@ -4261,6 +4311,7 @@ fn build_commit_operation_status( ), }; Ok(protocol::OperationStatus { + publish_preparation: None, token: protocol::OperationToken { operation_id: operation.operation_id.into(), state_digest: build_state_digest(operation)?, @@ -5094,6 +5145,7 @@ fn restore_operation_status( ), }; Ok(protocol::OperationStatus { + publish_preparation: None, token: protocol::OperationToken { operation_id: operation.operation_id.into(), state_digest: restore_state_digest(operation)?, @@ -8159,6 +8211,7 @@ mod tests { first_request_fill, protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_intent_digest: None, operation_id, artifact_revision_id, target: target.clone(), @@ -9344,6 +9397,7 @@ mod tests { request_id: protocol::RequestIdentity([0x53; types::FIXED_ID_BYTES]), operation: protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_intent_digest: None, operation_id, artifact_revision_id, target: target.clone(), @@ -10091,6 +10145,7 @@ mod tests { request_fill, protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_intent_digest: None, operation_id, artifact_revision_id, target: protocol::WorkspacePath { @@ -10207,6 +10262,273 @@ mod tests { assert!(same_operation_current_fence.replayed); } + fn stable_append_begin_request() -> protocol::BeginArtifactPublishRequest { + let artifact_revision_id = + protocol::ArtifactRevisionIdentity([0x84; types::FIXED_ID_BYTES]); + let seals = protocol::seal_artifact_publish_plan(artifact_revision_id, &[], &[]).unwrap(); + protocol::BeginArtifactPublishRequest { + append_intent_digest: Some(protocol::Digest([0x85; types::SHA256_BYTES])), + operation_id: protocol::OperationIdentity([0x83; types::FIXED_ID_BYTES]), + artifact_revision_id, + target: protocol::WorkspacePath { + workbench: protocol::WorkbenchName::new("stable-append").unwrap(), + path: protocol::RelativePath::new("outputs/events.log").unwrap(), + }, + authority: protocol::PublicationAuthority::Visible, + condition: protocol::PublishCondition::CreateOnly, + expected_workspace_incarnation_id: Some(protocol::WorkspaceIdentity( + [0x82; types::FIXED_ID_BYTES], + )), + staged_object_count: seals.staged_object_count, + staged_object_seal: seals.staged_object_seal, + manifest_row_count: seals.manifest_row_count, + manifest_seal: seals.manifest_seal, + dependency_owner_revision_ids: Vec::new(), + } + } + + #[test] + fn public_operation_identity_remains_queryable_when_publish_and_commit_collide() { + for publish_first in [true, false] { + let (_store, executor) = ready_executor(); + executor + .execute(&create_request(0x81, "stable-append", 0x82, 1)) + .unwrap(); + let begin = stable_append_begin_request(); + let publish = restore_rpc( + 0x86, + protocol::WorkspaceRequest::BeginArtifactPublish(begin.clone()), + ); + let mut commit = commit_request(0x87); + let protocol::WorkspaceRequest::Commit(request) = &mut commit.operation else { + unreachable!() + }; + request.operation_id = begin.operation_id; + request.workbench = begin.target.workbench.clone(); + request.workspace_incarnation_id = begin.expected_workspace_incarnation_id.unwrap(); + let query = restore_rpc( + 0x88, + protocol::WorkspaceRequest::GetOperation(protocol::GetOperationRequest { + operation_id: begin.operation_id, + }), + ); + let expected_kind = if publish_first { + executor.execute(&publish).unwrap(); + // Commit may stop after admission because this fixture has no + // canonical run manifest; its admitted row is what must fence. + let _ = executor.execute(&commit); + protocol::OperationKind::ArtifactPublish + } else { + let _ = executor.execute(&commit); + let before = executor.execute(&query).unwrap(); + assert!( + matches!(before.result, protocol::WorkspaceResult::Operation(ref status) if status.kind == protocol::OperationKind::Commit) + ); + let _ = executor.execute(&publish); + protocol::OperationKind::Commit + }; + let observed = executor.execute(&query).expect( + "cross-kind admission must preserve the first operation's queryable receipt", + ); + assert!( + matches!(observed.result, protocol::WorkspaceResult::Operation(ref status) if status.kind == expected_kind) + ); + } + } + + #[test] + fn stable_append_rejects_pending_replans_and_intent_reuse_without_cleanup() { + let (store, executor) = ready_executor(); + executor + .execute(&create_request(0x81, "stable-append", 0x82, 1)) + .unwrap(); + let begin = stable_append_begin_request(); + let request = restore_rpc( + 0x86, + protocol::WorkspaceRequest::BeginArtifactPublish(begin.clone()), + ); + let begun = executor.execute(&request).unwrap(); + let protocol::WorkspaceResult::Operation(status) = begun.result else { + panic!("missing status") + }; + assert_eq!( + status + .publish_preparation + .as_ref() + .unwrap() + .append_intent_digest, + begin.append_intent_digest + ); + assert!(executor.execute(&request).unwrap().replayed); + + let mut changed = begin.clone(); + changed.manifest_seal = protocol::Digest([0x91; types::SHA256_BYTES]); + let conflict = executor + .execute(&restore_rpc( + 0x87, + protocol::WorkspaceRequest::BeginArtifactPublish(changed), + )) + .unwrap_err(); + assert_eq!( + conflict.conflict, + Some(protocol::ConflictKind::OperationState) + ); + for (index, digest) in [None, Some(protocol::Digest([0x92; types::SHA256_BYTES]))] + .into_iter() + .enumerate() + { + let mut changed = begin.clone(); + changed.append_intent_digest = digest; + let mismatch = executor + .execute(&restore_rpc( + 0x88 + index as u8, + protocol::WorkspaceRequest::BeginArtifactPublish(changed), + )) + .unwrap_err(); + assert_eq!(mismatch.code, protocol::ErrorCode::RequestReplayMismatch); + } + let mut changed = begin.clone(); + changed.append_intent_digest = Some(protocol::Digest([0x93; types::SHA256_BYTES])); + let mismatch = executor + .execute(&restore_rpc( + 0x86, + protocol::WorkspaceRequest::BeginArtifactPublish(changed), + )) + .unwrap_err(); + assert_eq!(mismatch.code, protocol::ErrorCode::RequestReplayMismatch); + let observed = executor + .execute(&restore_rpc( + 0x8a, + protocol::WorkspaceRequest::GetOperation(protocol::GetOperationRequest { + operation_id: begin.operation_id, + }), + )) + .unwrap(); + assert_eq!( + observed.result, + protocol::WorkspaceResult::Operation(status) + ); + store.advance_owner_epoch(Some(owner(1)), owner(2)).unwrap(); + let stale = executor + .execute(&restore_rpc( + 0x8b, + protocol::WorkspaceRequest::BeginArtifactPublish(begin), + )) + .unwrap_err(); + assert_eq!(stale.code, protocol::ErrorCode::NotOwner); + } + + #[test] + fn stable_append_terminal_replay_is_independent_of_live_head_and_plan() { + let (store, executor) = ready_executor(); + executor + .execute(&create_request(0x81, "stable-append", 0x82, 1)) + .unwrap(); + let begin = stable_append_begin_request(); + let begun = executor + .execute(&restore_rpc( + 0x86, + protocol::WorkspaceRequest::BeginArtifactPublish(begin.clone()), + )) + .unwrap(); + let protocol::WorkspaceResult::Operation(status) = begun.result else { + panic!("missing status") + }; + let completed = executor + .execute(&restore_rpc( + 0x87, + protocol::WorkspaceRequest::CompleteArtifactPublish( + protocol::CompleteArtifactPublishRequest { + token: status.token, + artifact: protocol::ArtifactDescriptor { + logical_size: 0, + body_digest: protocol::sha256_digest_uri(protocol::Digest( + Sha256::digest([]).into(), + )), + manifest_digest: protocol::sha256_digest_uri(begin.manifest_seal), + content_type: protocol::ContentType::new("text/plain").unwrap(), + producer: None, + manifest_identity: None, + index_fields: Vec::new(), + }, + }, + ), + )) + .unwrap(); + let protocol::WorkspaceResult::Published(original) = completed.result else { + panic!("missing receipt") + }; + executor + .execute(&restore_rpc( + 0x88, + protocol::WorkspaceRequest::RemovePath(protocol::RemovePathRequest { + target: begin.target.clone(), + expected_generation: original.generation, + }), + )) + .unwrap(); + publish_one_byte_artifact( + &executor, + 0xa0, + protocol::OperationIdentity([0xa1; types::FIXED_ID_BYTES]), + protocol::ArtifactRevisionIdentity([0xa2; types::FIXED_ID_BYTES]), + "stable-append", + "outputs/events.log", + protocol::PublicationAuthority::Visible, + 7, + ); + // Even a newly planned delta over the new head cannot replace the old + // logical result or cause provider/dependency reads before replay. + let mut replanned = begin.clone(); + replanned.condition = protocol::PublishCondition::Append { + expected_generation: Some(999), + }; + replanned.manifest_seal = protocol::Digest([0x94; types::SHA256_BYTES]); + replanned.dependency_owner_revision_ids = vec![protocol::ArtifactRevisionIdentity( + [0x95; types::FIXED_ID_BYTES], + )]; + let replayed = executor + .execute(&restore_rpc( + 0x89, + protocol::WorkspaceRequest::BeginArtifactPublish(replanned), + )) + .unwrap(); + assert!(replayed.replayed); + let protocol::WorkspaceResult::Operation(replayed) = replayed.result else { + panic!("missing status") + }; + assert_eq!(replayed.state, protocol::OperationState::Succeeded); + assert_eq!( + replayed.result, + Some(protocol::OperationResult::ArtifactPublish(original.clone())) + ); + let mut wrong_incarnation = begin.clone(); + wrong_incarnation.expected_workspace_incarnation_id = + Some(protocol::WorkspaceIdentity([0x96; types::FIXED_ID_BYTES])); + let mismatch = executor + .execute(&restore_rpc( + 0x8a, + protocol::WorkspaceRequest::BeginArtifactPublish(wrong_incarnation), + )) + .unwrap_err(); + assert_eq!(mismatch.code, protocol::ErrorCode::RequestReplayMismatch); + + store.advance_owner_epoch(Some(owner(1)), owner(2)).unwrap(); + let mut successor_request = restore_rpc( + 0x8b, + protocol::WorkspaceRequest::BeginArtifactPublish(begin), + ); + successor_request.route = route(2); + let successor = executor.execute(&successor_request).unwrap(); + let protocol::WorkspaceResult::Operation(successor) = successor.result else { + panic!("missing successor receipt") + }; + assert_eq!( + successor.result, + Some(protocol::OperationResult::ArtifactPublish(original)) + ); + } + #[test] fn exact_publish_stage_retry_resumes_after_its_heartbeat_committed() { let (store, executor) = ready_executor(); @@ -10223,6 +10545,7 @@ mod tests { request_id: protocol::RequestIdentity([0x64; types::FIXED_ID_BYTES]), operation: protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_intent_digest: None, operation_id, artifact_revision_id, target: protocol::WorkspacePath { @@ -10501,6 +10824,7 @@ mod tests { 0x74, protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_intent_digest: None, operation_id: operation_identity, artifact_revision_id: revision_identity, target: protocol::WorkspacePath { @@ -10734,6 +11058,7 @@ mod tests { 0x7C, protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_intent_digest: None, operation_id: protocol::OperationIdentity( [0x7D; types::FIXED_ID_BYTES], ), diff --git a/crates/nokv-server/src/legacy_rejection.rs b/crates/nokv-server/src/legacy_rejection.rs index 7a7468b82..fd4ebe28a 100644 --- a/crates/nokv-server/src/legacy_rejection.rs +++ b/crates/nokv-server/src/legacy_rejection.rs @@ -356,7 +356,7 @@ mod tests { } #[test] - fn post_tag_v3_gets_a_v9_failure_that_its_decoder_reports_as_schema_mismatch() { + fn post_tag_v3_gets_a_v10_failure_that_its_decoder_reports_as_schema_mismatch() { let route = v3_route(); let request = request(LEGACY_V3_SCHEMA, route, "run-42"); let response = legacy_rejection_response(&request).unwrap(); diff --git a/crates/nokv-server/src/lifecycle.rs b/crates/nokv-server/src/lifecycle.rs index ffff40f37..9d45df358 100644 --- a/crates/nokv-server/src/lifecycle.rs +++ b/crates/nokv-server/src/lifecycle.rs @@ -2921,6 +2921,7 @@ mod tests { cleanup_state: StagedCleanupState::Owned, }]; let mut operation = meta::PublishOperationRecord { + append_intent_digest: None, operation_id: OperationId::from_bytes([0x74; FIXED_ID_BYTES]), identity_digest: [0; SHA256_BYTES], initialization_digest: [0; SHA256_BYTES], @@ -3230,6 +3231,7 @@ mod tests { let operation_id = OperationId::from_bytes([0x44; FIXED_ID_BYTES]); let mut operation = meta::PublishOperationRecord { + append_intent_digest: None, operation_id, identity_digest: [0; SHA256_BYTES], initialization_digest: [0; SHA256_BYTES], @@ -3515,6 +3517,7 @@ mod tests { }]; let template = |operation_id_byte: u8| { let mut operation = meta::PublishOperationRecord { + append_intent_digest: None, operation_id: OperationId::from_bytes([operation_id_byte; FIXED_ID_BYTES]), identity_digest: [0; SHA256_BYTES], initialization_digest: [0; SHA256_BYTES], @@ -3811,6 +3814,7 @@ mod tests { }]; let template = |operation_id_byte: u8| { let mut operation = meta::PublishOperationRecord { + append_intent_digest: None, operation_id: OperationId::from_bytes([operation_id_byte; FIXED_ID_BYTES]), identity_digest: [0; SHA256_BYTES], initialization_digest: [0; SHA256_BYTES], diff --git a/crates/nokv-server/src/server.rs b/crates/nokv-server/src/server.rs index 3f132ff86..df0273178 100644 --- a/crates/nokv-server/src/server.rs +++ b/crates/nokv-server/src/server.rs @@ -883,17 +883,21 @@ mod tests { #[test] fn operation_first_v3_gets_a_readable_rejection_with_zero_dispatch() { + #[derive(Serialize)] + struct V3Frame { + schema: &'static str, + payload: WorkspaceRpcRequest, + } + let (mut client, server) = streams(); let (registry, executor) = registry(); let serving = thread::spawn(move || serve_connection(server, registry, options())); - let mut legacy = encode_request(&request()).unwrap(); - let position = legacy - .windows(WORKSPACE_PROTOCOL_SCHEMA.len()) - .position(|window| window == WORKSPACE_PROTOCOL_SCHEMA.as_bytes()) - .unwrap(); - legacy[position..position + WORKSPACE_PROTOCOL_SCHEMA.len()] - .copy_from_slice(crate::legacy_rejection::LEGACY_V3_SCHEMA.as_bytes()); + let legacy = rmp_serde::to_vec_named(&V3Frame { + schema: crate::legacy_rejection::LEGACY_V3_SCHEMA, + payload: request(), + }) + .unwrap(); write_frame(&mut client, &legacy).unwrap(); let response = read_frame(&mut client).unwrap().unwrap(); let response = decode_response(&response).unwrap(); @@ -901,6 +905,11 @@ mod tests { panic!("legacy operation-first request must be rejected"); }; assert_eq!(failure.code, nokv_protocol::ErrorCode::PreconditionFailed); + assert_eq!( + failure.message, + crate::legacy_rejection::LEGACY_CLIENT_UPGRADE_MESSAGE + ); + assert!(!failure.retryable); serving.join().unwrap().unwrap(); assert_eq!(executor.calls.load(AtomicOrdering::SeqCst), 0); } @@ -964,7 +973,7 @@ mod tests { } #[test] - fn v7_hello_gets_v9_incompatible_and_zero_dispatch() { + fn v7_hello_gets_v10_incompatible_and_zero_dispatch() { let (mut client, server) = streams(); let (registry, executor) = registry(); let serving = thread::spawn(move || serve_connection(server, registry, options())); diff --git a/crates/nokv/src/backend.rs b/crates/nokv/src/backend.rs index db4a07af3..6362b23ff 100644 --- a/crates/nokv/src/backend.rs +++ b/crates/nokv/src/backend.rs @@ -15,8 +15,8 @@ use base64::Engine as _; use nokv_agent as agent; use nokv_client::{ ArtifactAppendOptions, ArtifactPublishOptions, ArtifactReadAuthority, ClientError, - SnapshotMintOptions, SnapshotRenewOptions, SnapshotRetireOptions, WorkbenchAdmission, - WorkbenchCommitRequest, WorkbenchLifecycleError, WorkbenchLifecycleFacade, + IdempotentAppendOptions, SnapshotMintOptions, SnapshotRenewOptions, SnapshotRetireOptions, + WorkbenchAdmission, WorkbenchCommitRequest, WorkbenchLifecycleError, WorkbenchLifecycleFacade, WorkbenchLifecycleOptions, WorkbenchRestoreOrigin, WorkbenchRestoreRequest, WorkbenchRestoreSource, WorkbenchSnapshotSelector, }; @@ -105,6 +105,55 @@ impl CliWorkbenchBackend { } } + /// Append under a caller-owned identity, observing but never creating the workspace. + #[allow(clippy::too_many_arguments)] + pub fn append_idempotent( + &self, + operation_id: wire::OperationIdentity, + target: wire::WorkspacePath, + expected_workspace_incarnation_id: Option, + create_content_type: wire::ContentType, + content_type: Option, + max_logical_size: Option, + delta: &[u8], + ) -> Result< + ( + nokv_client::ClientCall, + wire::WorkspaceIdentity, + ), + agent::BackendError, + > { + let incarnation = match expected_workspace_incarnation_id { + Some(incarnation) => incarnation, + None => { + self.client + .get_workspace(wire::GetWorkspaceRequest { + workbench: target.workbench.clone(), + }) + .map_err(map_client_error)? + .value + .workspace_incarnation_id + } + }; + let mut options = + IdempotentAppendOptions::new(operation_id, target, incarnation, create_content_type); + if let Some(content_type) = content_type { + options = options.with_content_type(content_type); + } + if let Some(max_logical_size) = max_logical_size { + options = options.with_max_logical_size(max_logical_size); + } + self.client + .append_artifact_idempotent(self.objects.as_ref(), options, delta) + .map(|call| (call, incarnation)) + .map_err(|error| { + let mut mapped = map_client_error(error); + mapped.details["workspace_incarnation_id"] = + json!(super::encode_lowercase_hex(&incarnation.0)); + mapped + }) + } + fn workspace( &self, workbench_id: &WorkbenchId, @@ -2758,6 +2807,20 @@ fn map_manifest_read_error(error: ClientError, path: &agent::ScopedPath) -> agen } fn map_client_error(error: ClientError) -> agent::BackendError { + if let ClientError::AppendUnresolved { + operation_id, + state, + source, + } = error + { + let mut mapped = map_client_error(*source); + mapped.kind = agent::BackendErrorKind::Other("AppendUnresolved".to_owned()); + // Retrying means recovering this exact identity; it never authorizes a new append. + mapped.retryable = false; + mapped.details["operation_id"] = json!(super::encode_lowercase_hex(&operation_id.0)); + mapped.details["state"] = json!(state.map(|state| format!("{state:?}"))); + return mapped; + } if let Some(failure) = rpc_failure(&error).cloned() { let attempts = retry_attempts(&error); let mut mapped = map_rpc_failure(failure); @@ -2792,6 +2855,7 @@ fn map_client_error(error: ClientError) -> agent::BackendError { ClientError::ArtifactPublishFailed { .. } | ClientError::RetryExhausted { .. } => { agent::BackendErrorKind::Other("ClientFailure".to_owned()) } + ClientError::AppendUnresolved { .. } => unreachable!("append failure returned above"), ClientError::Rpc(_) => unreachable!("RPC failures returned above"), } }; @@ -3631,6 +3695,7 @@ mod tests { state_digest: wire::Digest([0x61; 32]), }, kind: wire::OperationKind::Restore, + publish_preparation: None, commit_preparation: None, restore_preparation: Some(Box::new(wire::RestoreOperationPreparation { request, @@ -3795,6 +3860,7 @@ mod tests { state_digest: wire::Digest([0x61; 32]), }, kind: wire::OperationKind::Commit, + publish_preparation: None, commit_preparation: Some(Box::new(wire::CommitPreparation { request: Box::new(exact_request.clone()), committed_at_unix_seconds, @@ -3824,6 +3890,12 @@ mod tests { state_digest: wire::Digest([0x63; 32]), }, kind: wire::OperationKind::ArtifactPublish, + publish_preparation: Some(Box::new(wire::PublishPreparation { + append_intent_digest: None, + target: manifest_target.clone(), + workspace_incarnation_id: binding.workspace_incarnation_id, + artifact_revision_id: binding.artifact_revision_id, + })), commit_preparation: None, restore_preparation: None, state: wire::OperationState::Succeeded, @@ -7005,6 +7077,57 @@ mod tests { assert!(matches!(mapped.operand, wire::QueryOperand::Set(values) if values.len() == 2)); } + #[test] + fn native_append_does_not_admit_a_missing_workspace() { + let (backend, requests, server) = scripted_backend(vec![not_found_failure()]); + let error = backend + .append_idempotent( + wire::OperationIdentity([0xaa; 16]), + wire::WorkspacePath { + workbench: wire::WorkbenchName::new("absent").unwrap(), + path: wire::RelativePath::new("logs/events.jsonl").unwrap(), + }, + None, + wire::ContentType::new("text/plain").unwrap(), + None, + None, + b"event", + ) + .unwrap_err(); + assert_eq!(error.kind, agent::BackendErrorKind::NotFound); + server.join().unwrap(); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!(matches!( + &requests[0].operation, + wire::WorkspaceRequest::GetWorkspace(_) + )); + } + + #[test] + fn unresolved_append_error_preserves_identity_state_and_underlying_cause() { + let error = map_client_error(ClientError::AppendUnresolved { + operation_id: wire::OperationIdentity([0xaa; 16]), + state: Some(wire::OperationState::Running), + source: Box::new(ClientError::Rpc(wire::RpcFailure { + code: wire::ErrorCode::Conflict, + message: "path generation changed".to_owned(), + retryable: false, + conflict: Some(wire::ConflictKind::PathGeneration), + current_generation: Some(3), + route_hint: None, + })), + }); + assert_eq!( + error.kind, + agent::BackendErrorKind::Other("AppendUnresolved".to_owned()) + ); + assert_eq!(error.details["operation_id"], "aa".repeat(16)); + assert_eq!(error.details["state"], "Running"); + assert_eq!(error.details["code"], "Conflict"); + assert!(!error.retryable); + } + #[test] fn exhausted_sdk_append_conflict_remains_a_retryable_workbench_conflict() { let mapped = map_client_error(ClientError::RetryExhausted { diff --git a/crates/nokv/src/cli.rs b/crates/nokv/src/cli.rs index 5184a9a29..9891d1259 100644 --- a/crates/nokv/src/cli.rs +++ b/crates/nokv/src/cli.rs @@ -155,6 +155,16 @@ pub enum McpProfile { #[derive(Clone, Debug, PartialEq, Eq)] pub enum WorkspacePathCommand { + Append { + workbench: String, + section: String, + path: String, + operation_id: [u8; 16], + expected_workspace_incarnation_id: Option<[u8; 16]>, + payload: AppendPayload, + content_type: Option, + max_logical_size: Option, + }, Rename { workbench: String, section: String, @@ -172,6 +182,13 @@ pub enum WorkspacePathCommand { }, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AppendPayload { + Text(String), + Base64(String), + File(PathBuf), +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum CliError { MissingValue(String), @@ -636,6 +653,9 @@ fn parse_workspace_path(arguments: &mut impl Iterator) -> Result< let first_path = arguments .next() .ok_or(CliError::MissingArgument("workspace path"))?; + if operation == "append" { + return parse_workspace_append(arguments, workbench, section, first_path); + } let destination = match operation.as_str() { "rename" => Some( arguments @@ -693,6 +713,82 @@ fn parse_workspace_path(arguments: &mut impl Iterator) -> Result< })) } +fn parse_workspace_append( + arguments: &mut impl Iterator, + workbench: String, + section: String, + path: String, +) -> Result { + let mut operation_id = None; + let mut expected_workspace_incarnation_id = None; + let mut payload = None; + let mut content_type = None; + let mut max_logical_size = None; + while let Some(argument) = arguments.next() { + match argument.as_str() { + "--operation-id" if operation_id.is_none() => { + operation_id = Some(parse_append_identity( + "--operation-id", + next_value(arguments, &argument)?, + )?); + } + "--expected-workspace-incarnation-id" + if expected_workspace_incarnation_id.is_none() => + { + expected_workspace_incarnation_id = Some(parse_append_identity( + "--expected-workspace-incarnation-id", + next_value(arguments, &argument)?, + )?); + } + "--text" if payload.is_none() => { + payload = Some(AppendPayload::Text(next_value(arguments, &argument)?)); + } + "--base64" if payload.is_none() => { + payload = Some(AppendPayload::Base64(next_value(arguments, &argument)?)); + } + "--file" if payload.is_none() => { + payload = Some(AppendPayload::File(PathBuf::from(next_value( + arguments, &argument, + )?))); + } + "--content-type" if content_type.is_none() => { + content_type = Some(next_value(arguments, &argument)?); + } + "--max-logical-size" if max_logical_size.is_none() => { + max_logical_size = Some(parse_number( + "--max-logical-size", + next_value(arguments, &argument)?, + )?); + } + "--operation-id" + | "--expected-workspace-incarnation-id" + | "--text" + | "--base64" + | "--file" + | "--content-type" + | "--max-logical-size" => return Err(CliError::UnexpectedArgument(argument)), + _ if argument.starts_with("--") => return Err(CliError::UnknownOption(argument)), + _ => return Err(CliError::UnexpectedArgument(argument)), + } + } + Ok(Command::WorkspacePath(WorkspacePathCommand::Append { + workbench, + section, + path, + operation_id: operation_id.ok_or(CliError::MissingOption("--operation-id"))?, + expected_workspace_incarnation_id, + payload: payload.ok_or(CliError::MissingArgument( + "one of --text, --base64, or --file", + ))?, + content_type, + max_logical_size, + })) +} + +fn parse_append_identity(option: &'static str, value: String) -> Result<[u8; 16], CliError> { + parse_request_id(value.clone()).map_err(|_| CliError::InvalidOption { option, value }) +} + fn parse_provision(arguments: &mut impl Iterator) -> Result { let logical_shard_id = arguments .next() @@ -951,6 +1047,65 @@ mod tests { ); } + #[test] + fn workspace_append_requires_one_payload_and_stable_identity() { + let prefix = ["append", "run-42", "logs", "events.jsonl"]; + let parse_append = |tail: &[&str]| { + let mut input = prefix.to_vec(); + input.extend_from_slice(tail); + parse_workspace_path(&mut args(&input).into_iter()) + }; + assert_eq!( + parse_append(&["--text", "event"]), + Err(CliError::MissingOption("--operation-id")) + ); + assert!(parse_append(&["--operation-id", "ABC", "--text", "event"]).is_err()); + assert!(parse_append(&[ + "--operation-id", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "--text", + "event", + "--base64", + "ZXZlbnQ=" + ]) + .is_err()); + let command = parse_append(&[ + "--operation-id", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "--expected-workspace-incarnation-id", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "--text", + "event", + "--content-type", + "text/plain", + "--max-logical-size", + "1024", + ]) + .unwrap(); + assert_eq!( + command, + Command::WorkspacePath(WorkspacePathCommand::Append { + workbench: "run-42".to_owned(), + section: "logs".to_owned(), + path: "events.jsonl".to_owned(), + operation_id: [0xaa; 16], + expected_workspace_incarnation_id: Some([0xbb; 16]), + payload: AppendPayload::Text("event".to_owned()), + content_type: Some("text/plain".to_owned()), + max_logical_size: Some(1024), + }) + ); + for payload in [["--base64", "ZXZlbnQ="], ["--file", "/tmp/delta.bin"]] { + assert!(parse_append(&[ + "--operation-id", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + payload[0], + payload[1] + ]) + .is_ok()); + } + } + #[test] fn workspace_path_mutations_require_exact_replay_inputs() { let parsed = parse(args(&[ diff --git a/crates/nokv/src/main.rs b/crates/nokv/src/main.rs index 387adf4a4..ae9a9fd79 100644 --- a/crates/nokv/src/main.rs +++ b/crates/nokv/src/main.rs @@ -31,7 +31,7 @@ use nokv_types::{NormalizedRelativePath, WorkbenchId}; use serde_json::{json, Value}; use backend::CliWorkbenchBackend; -use cli::{Command, Invocation, McpProfile, WorkspacePathCommand}; +use cli::{AppendPayload, Command, Invocation, McpProfile, WorkspacePathCommand}; #[cfg(feature = "etcd")] use object_store::CliObjectStore; @@ -323,7 +323,49 @@ fn run_workspace_path( invocation: &Invocation, command: &WorkspacePathCommand, ) -> Result<(), String> { + if let WorkspacePathCommand::Append { + workbench, + section, + path, + operation_id, + expected_workspace_incarnation_id, + payload, + content_type, + max_logical_size, + } = command + { + let operation_id = nokv_protocol::OperationIdentity(*operation_id); + let result = (|| { + let target = canonical_workspace_path(workbench, section, path)?; + let (delta, default_content_type) = + append_payload(payload, invocation.client.max_artifact_bytes)?; + let content_type = content_type + .as_ref() + .map(|value| nokv_protocol::ContentType::new(value.clone())) + .transpose() + .map_err(|error| error.to_string())?; + let create_content_type = content_type.clone().unwrap_or( + nokv_protocol::ContentType::new(default_content_type) + .map_err(|error| error.to_string())?, + ); + let backend = build_backend(invocation)?; + let (call, incarnation) = backend + .append_idempotent( + operation_id, + target, + expected_workspace_incarnation_id.map(nokv_protocol::WorkspaceIdentity), + create_content_type, + content_type, + *max_logical_size, + &delta, + ) + .map_err(|error| agent_error(error.into()))?; + print_json(&append_receipt_json(&call, incarnation)) + })(); + return result.map_err(|message| append_error_json(operation_id, message)); + } let (request_id, operation) = match command { + WorkspacePathCommand::Append { .. } => unreachable!("append dispatched above"), WorkspacePathCommand::Rename { workbench, section, @@ -371,6 +413,65 @@ fn run_workspace_path( } } +fn append_payload( + payload: &AppendPayload, + max_bytes: usize, +) -> Result<(Vec, &'static str), String> { + let (bytes, content_type) = match payload { + AppendPayload::Text(text) => (text.as_bytes().to_vec(), "text/plain; charset=utf-8"), + AppendPayload::Base64(encoded) => ( + STANDARD + .decode(encoded) + .map_err(|error| format!("append base64 payload is invalid: {error}"))?, + "application/octet-stream", + ), + AppendPayload::File(path) => ( + transfer::read_collect_source(path, max_bytes)?, + "application/octet-stream", + ), + }; + if bytes.len() > max_bytes { + return Err(format!( + "append delta is {} bytes, maximum is {max_bytes}", + bytes.len(), + )); + } + Ok((bytes, content_type)) +} + +fn append_receipt_json( + call: &nokv_client::ClientCall, + incarnation: nokv_protocol::WorkspaceIdentity, +) -> Value { + let result = &call.value; + json!({ + "status": "success", + "operation": "append", + "operation_id": encode_lowercase_hex(&result.operation_id.0), + "artifact_revision_id": encode_lowercase_hex(&result.artifact_revision_id.0), + "workbench_id": result.target.workbench.as_str(), + "path": result.target.path.as_str(), + "workspace_incarnation_id": encode_lowercase_hex(&incarnation.0), + "workspace_revision": result.workspace_revision, + "generation": result.generation, + "logical_size": result.logical_size, + "body_digest": result.body_digest.as_str(), + "replayed": call.replayed, + "commit_version": call.commit_version, + }) +} + +fn append_error_json(operation_id: nokv_protocol::OperationIdentity, message: String) -> String { + let mut error = serde_json::from_str::(&message) + .ok() + .filter(|value| value["status"] == "error") + .unwrap_or_else(|| { + nokv_agent::AgentError::backend("AppendFailed", message, false, json!({})).as_value() + }); + error["details"]["operation_id"] = json!(encode_lowercase_hex(&operation_id.0)); + serde_json::to_string(&error).expect("append error contains only JSON values") +} + fn canonical_workspace_path( workbench: &str, section: &str, @@ -591,6 +692,7 @@ USAGE: nokv [connection/object options] collect
[--replace] [--expected-generation N] [--content-type TYPE] nokv [route/agent options] workspace-path rename
--expected-generation N --request-id HEX32 nokv [route/agent options] workspace-path remove
--expected-generation N --request-id HEX32 + nokv [route/agent/object options] workspace-path append
--operation-id HEX32 (--text TEXT | --base64 BASE64 | --file PATH) [--content-type TYPE] [--max-logical-size N] [--expected-workspace-incarnation-id HEX32] nokv --root-id HEX32 --agent-id HEX32 --etcd-endpoint URL provision [adoption options] nokv [owner options] serve nokv schema @@ -617,7 +719,10 @@ AGENT PRESENTATION: or the direct Python SDK; it is retained only as the live qualification harness transport mcp defaults to the 18-tool Workbench profile; --profile agent selects the seven path tools workspace-path is a custom CLI surface; it does not add to the fixed 18 Workbench tools - workspace-path requires an explicit lowercase HEX32 request id for exact cross-process replay + workspace-path rename/remove require an explicit lowercase HEX32 request id for exact cross-process replay + workspace-path append requires a stable lowercase HEX32 operation id and an existing workspace + append returns the original publication receipt on replay; retain the id and incarnation across restarts + append limits the delta with --max-artifact-bytes; --max-logical-size optionally limits the resulting body OWNER: provision first binds RootId to an explicit AgentId, then creates namespace, shard, and affinity @@ -1101,6 +1206,63 @@ mod tests { assert!(scoped_artifact("run-1", "outputs", "../result.json").is_err()); } + #[test] + fn append_payloads_are_bounded_and_share_explicit_bytes() { + let text = append_payload(&AppendPayload::Text("event".to_owned()), 5).unwrap(); + let binary = append_payload(&AppendPayload::Base64("ZXZlbnQ=".to_owned()), 5).unwrap(); + assert_eq!(text.0, binary.0); + assert_eq!(text.1, "text/plain; charset=utf-8"); + assert_eq!(binary.1, "application/octet-stream"); + assert!(append_payload(&AppendPayload::Text("event".to_owned()), 4).is_err()); + assert!(append_payload(&AppendPayload::Base64("!".to_owned()), 5).is_err()); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("delta.bin"); + std::fs::write(&path, b"event").unwrap(); + assert_eq!( + append_payload(&AppendPayload::File(path), 5).unwrap(), + binary + ); + } + + #[test] + fn append_receipt_keeps_historical_publication_and_error_identity() { + let call = nokv_client::ClientCall { + value: nokv_protocol::PublishResult { + operation_id: nokv_protocol::OperationIdentity([0xaa; 16]), + artifact_revision_id: nokv_protocol::ArtifactRevisionIdentity([0xbb; 16]), + target: canonical_workspace_path("run-42", "logs", "events.jsonl").unwrap(), + workspace_revision: 7, + generation: 3, + logical_size: 12, + body_digest: nokv_protocol::sha256_digest_uri(nokv_protocol::Digest([0xcc; 32])), + }, + commit_version: None, + replayed: true, + }; + let receipt = append_receipt_json(&call, nokv_protocol::WorkspaceIdentity([0xdd; 16])); + assert_eq!(receipt["operation_id"], "aa".repeat(16)); + assert_eq!(receipt["artifact_revision_id"], "bb".repeat(16)); + assert_eq!(receipt["generation"], 3); + assert_eq!(receipt["logical_size"], 12); + assert_eq!(receipt["workspace_incarnation_id"], "dd".repeat(16)); + assert_eq!(receipt["replayed"], true); + assert!(receipt["commit_version"].is_null()); + let error = nokv_agent::AgentError::backend( + "AppendUnresolved", + "reply lost", + false, + json!({"state": "Running"}), + ); + let result: Value = serde_json::from_str(&append_error_json( + call.value.operation_id, + agent_error(error), + )) + .unwrap(); + assert_eq!(result["details"]["state"], "Running"); + assert_eq!(result["details"]["operation_id"], "aa".repeat(16)); + assert_eq!(result["code"], "AppendUnresolved"); + } + #[test] fn workspace_path_maps_sections_without_accepting_reserved_or_duplicated_paths() { let path = canonical_workspace_path("run-42", "outputs", "result.json").unwrap(); diff --git a/docs/architecture.md b/docs/architecture.md index 5ea65c4da..c6da54925 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -159,22 +159,23 @@ continuations retain one exact root read version. Live continuations may move to a newer root read version only while the target workspace incarnation and revision remain unchanged; target drift fails closed, and an initial bounded collection may restart in full but never merges workspace revisions. This -contract is gated by operation schema `nokv.workspace.rpc.v9`; v7 added the +contract is gated by operation schema `nokv.workspace.rpc.v10`; v10 adds exact +append-intent binding and publication preparation in operation status. v7 added the exact read-version fence used by path point reads, v8 added the required typed `Prefix`/`Exact` catalog path match so an artifact cannot inherit fields from -same-name descendants, and v9 adds an explicit query profile plus tagged query +same-name descendants, and v9 added an explicit query profile plus tagged query rows and authoritative search, aggregate, and catalog totals/facets. V3 first added the provider-neutral object namespace identity to every root route. Each TCP connection starts with the fixed-width, schema-neutral transport handshake v1. The client offers one exact operation schema, and the server -accepts only `nokv.workspace.rpc.v9`; the handshake version remains stable when +accepts only `nokv.workspace.rpc.v10`; the handshake version remains stable when the operation schema changes. For upgrade diagnostics, the server recognizes only operation-first envelopes from the public v2 client and the post-tag v3 client. It parses a bounded route/request header, ignores the operation, emits one schema-readable failure, and closes without dispatch. The v2 client gets -its exact v2 failure envelope; the v3 client gets a v9 failure envelope so its -decoder reports the v9/v3 schema mismatch. Unknown legacy schemas, malformed +its exact v2 failure envelope; the v3 client gets a v10 failure envelope so its +decoder reports the v10/v3 schema mismatch. Unknown legacy schemas, malformed envelopes, and malformed handshakes fail closed. This is a read-only rejection path, not a legacy response decoder or operation fallback. diff --git a/docs/development/append-idempotency.md b/docs/development/append-idempotency.md new file mode 100644 index 000000000..7407bf8b3 --- /dev/null +++ b/docs/development/append-idempotency.md @@ -0,0 +1,146 @@ + + +# Append identity and response-loss recovery + +A harness can persist a caller operation ID before sending an append. Retrying +that ID means one publication attempt, even after the harness or shard owner +restarts. A different operation ID represents a different action, including +when its bytes are identical. The native CLI is the primary integration +surface; the Python SDK uses the same client implementation. + +## Contract and scope + +The stable append request binds the complete caller intent with a full SHA-256 +commitment in the existing publish operation record. The intent includes the +normalized target and explicit append inputs and policies. It is distinct from +the generation-dependent immutable publication plan. The client computes this +commitment from its typed inputs; the server exact-binds it as opaque input and +does not claim to verify delta bytes that the metadata RPC does not carry. + +One operation identity admits at most one publication attempt. It never derives +a second identity after a timeout, read conflict, or path-generation conflict. +Before consulting the live path, a retry authenticates the operation's intent, +target, workspace incarnation, and artifact revision. Identity reuse with a +changed request fails closed. + +Callers must allocate operation IDs uniquely across the public lifecycle kinds +within one root: append/publication, commit build, and restore. Domain-separated +IDs persisted by the harness are one way to meet this requirement. The server +also enforces this rule atomically: each initial admission checks that both +other kinds' operation keys are absent in the same metadata command that +creates its own row. A preliminary GetOperation lookup alone cannot close this +race. No separate registry or new key family is needed. GetOperation remains +fail-closed if corrupt or externally supplied state contains multiple kinds for +one identity. Internal commit and restore manifest publications use distinct, +domain-separated identities and keep their existing lifecycle authority. + +A published operation returns its original compact receipt: operation ID, +target, workspace revision, generation, artifact revision, logical size, and +body digest. This is a historical result. A later append, removal, or replacement +of the live path does not revoke that success and does not trigger another +append. The receipt does not claim that its revision is still the live head or +that its bytes remain retained forever. + +The response envelope's `commit_version` and `replayed` are metadata about the +current call, not fields of the stable historical result. The first publication +may carry a metadata commit version; recovery through GetOperation carries no +new commit version and marks the client result as replayed. Consumers should +compare the receipt fields listed above, rather than requiring byte-identical +JSON envelopes across the first call and retries. + +An admitted operation without a terminal success returns a queryable pending or +terminal failure result. A new Begin RPC cannot replan, take over, or clean up +that operation. Byte-identical retries of the original RPC continue through the +existing request ledger. The existing fenced publish lifecycle owns recovery, +including the publication-absence proof required before Finalizing cleanup. +The caller must not interpret timeout, NotFound, or Finalizing as proof that an +append did not happen, nor automatically retry with a fresh operation ID. + +For example, a queue-driven harness persists an action ID for task 42, step 7 +before appending its completion event. If its worker dies after publication, +the replacement worker sends the same ID and event and recovers that receipt. +Two independently scheduled steps may emit identical events and still need two +different IDs. A branch of a run also needs its own action identity: changing +the target workspace under the parent's ID is an intent mismatch. + +A worker that dies during upload has a different recovery boundary. Its new +process can discover the admitted operation, but this API does not resume that +upload or create another attempt. The existing owner lifecycle can finish its +fenced cleanup. Workloads requiring automatic completion despite admitted CAS +conflicts or abandoned uploads need the durable attempt mapping described below. + +The workspace incarnation fence is mandatory for stable append. It prevents a +request for an old workspace instance from writing into a later instance with +the same name. Root routing and owner fencing continue to apply to lookups, +request deduplication, and mutation. + +Terminal operations currently have no automatic GC. Their retained records +provide the identity tombstone and historical receipt. Any future retention +policy must preserve an equivalent replay contract before removing them. This +change does not add a hidden artifact reference or pin every old revision just +to recover metadata. PublishPreparation projects existing target, incarnation, +revision, and intent fields; it does not store a full descriptor, whose legal +index fields can exceed the 16 KiB operation-record budget. + +## Architecture alternatives + +| Alternative | Reason for the decision | +| --- | --- | +| Stable caller ID alone | Re-reading the live head builds a different append plan after success; an ID flag alone cannot authenticate or recover the original intent. | +| Selected: one attempt through the existing publish lifecycle | Keeps publication, cleanup, fencing, and durability in one state machine while making response-loss recovery exact and bounded. | +| Logical append with durable attempt mapping | Appropriate if automatic rebase after concurrent-writer conflicts becomes a required product contract. It needs a durable parent or predecessor fence proving every earlier attempt cannot publish. It is outside this single-attempt change. | +| RPC request-ID reuse | Protects an exact encoded command, including route-dependent fields. It cannot alone represent a logical append across process and owner changes. | +| Content-hash deduplication | Incorrectly suppresses two intentional actions carrying identical bytes. | +| Immutable event files keyed by action ID | A useful harness design for event-oriented workloads, but changes the byte-append contract and needs an ordering/projection policy. It does not repair append. | + +Deterministic attempt numbering without durable predecessor records is unsafe: +one caller may skip an unadmitted attempt after a read conflict while another +caller later publishes that same attempt. Both it and the first caller's next +attempt can then succeed. This change deliberately has no such attempt chain. + +## Storage and protocol versions + +The strict workspace system format advances from 10 to 11; the publication +record family advances from value format 4 to 5; the RPC schema advances from +`nokv.workspace.rpc.v9` to `nokv.workspace.rpc.v10`. Older stores and protocol +schemas are rejected. No migration, compatibility decoder, or marker-only +upgrade is introduced. Red and green acceptance therefore use independently +initialized stores. Green recovery tests reopen the same green-version Holt +directory after owner termination. + +## Acceptance requirements + +Tests must retain actual owner/object-provider execution evidence for response +loss followed by a new CLI process and owner restart; one logical append must +produce one event. Additional cases cover a newer live head, changed intent, +concurrent callers with the same ID, distinct IDs with identical bytes, +incarnation changes, and admitted pending operations. Unit tests protect exact +binding, codec versions, server request deduplication, historical receipts, +owner fencing, and refusal to replan or clean up an admitted operation. + +The executable gate requires Docker, AWS CLI, etcd, and etcdctl. Build the CLI +with `cargo build -p nokv --features etcd`, freeze a copy of that binary, and +install a wheel built from the same checkout into an isolated Python environment. +Run the full qualification from the repository root: + +```shell +python3 scripts/workbench/append_identity_recovery_gate.py \ + --mode qualified --scenario all \ + --source-dir "$PWD" --nokv-bin /absolute/path/to/frozen-nokv \ + --python-executable /absolute/path/to/venv/bin/python \ + --evidence-dir /absolute/path/to/new-evidence-directory +``` + +Use `--mode baseline` with a frozen pre-fix binary and a separate evidence +directory to reproduce the duplicate append. The gate retains the dropped +successful RPC response, process/owner restart evidence, materialized bytes, +binary and Python-extension identities, current source hashes, and cleanup +results. `GREEN_PARTIAL` identifies a selected subset or a run without Python; +pending-operation cases qualify safety separately from completion. + +This contract covers NoKV publication. It does not make email, browser actions, +payments, or other external effects execute once. A downstream harness must +assign and persist identities for those effects at their own authority. diff --git a/docs/metadata-schema.md b/docs/metadata-schema.md index ce1691526..b14672397 100644 --- a/docs/metadata-schema.md +++ b/docs/metadata-schema.md @@ -19,15 +19,15 @@ Every logical-shard store has one authoritative marker: System("schema") -> value_format_version = 1 schema_id = "nokv_workspace" - format_version = 10 + format_version = 11 ``` Startup is fail-closed: - an empty store is initialized with the exact supported marker and logical keyspace catalog; -- format-9 and older stores are rejected without writes; there is no marker-only - upgrade because format 10 adds authoritative Generic index families; +- format-10 and older stores are rejected without writes; there is no marker-only + upgrade because format 11 changes the durable publication operation codec; - a nonempty current store opens only when its marker, value format, and configured adapter catalog match this contract; - a missing, malformed, unknown-version, or inconsistent store is rejected. @@ -84,7 +84,7 @@ exact key is a strict prefix of another valid path key. A child/subtree prefix appends NUL, so `a` cannot match `ab`. The empty path has no `PathCurrent` record; the workspace root is synthesized from `WorkspaceCurrent`. This path key layout was introduced by system format version 8 and is retained by -version 10. +version 11. The one shared normalizer enforces: @@ -105,19 +105,22 @@ float, timestamp, bytes, and string values. ## Durable Format Registry -`System.format_version` is `10`. Version 10 retains the format-9 RecoveryOutbox -LSN encoding as -canonical fixed-width decimal keys. Numeric ordering is unchanged, while the -sequential key shape avoids pathological underfilled Holt frames. Logical -recovery records, deterministic results, and hash-chain bytes are unchanged. +`System.format_version` is `11`. Version 11 adds the optional full SHA-256 +append-intent commitment to the existing publication operation record. It +retains the format-10 Generic index families and the format-9 RecoveryOutbox +fixed-width decimal LSN keys. Existing logical recovery and object formats are +unchanged. -Ordinary open does not migrate a format-9 marker, even when its old catalog is -otherwise internally consistent. Format 9 lacks the three authoritative -Generic index families, so marker-only upgrade would advertise records and -lifecycle invariants that were never installed. Migration remains not -qualified; every older or unknown marker is fail-closed and unchanged. +Ordinary open does not migrate a format-10 or older marker. A marker-only +upgrade would reinterpret old publication records without their required codec +version. Migration remains not qualified; every older or unknown marker is +fail-closed and unchanged. Durable codecs are independently versioned: +publish operation, staged-object, and manifest-row records use value version +`5` and reject version `4`; the operation binds an optional 256-bit append +intent before any object publication. Other publication rows retain their +existing layouts. publication-owned workspace/path/revision records use value version `2`; `CommitRecord` uses version `3` and dual-decodes version `2`, while its member, consumer, head, and tag records remain version `2`; `ChangeEvent` and the @@ -460,6 +463,14 @@ History val: previous versioned value or tombstone ``` +Public operation IDs are root-scoped across Publish, BuildCommit, and Restore. +Although their keys retain the lifecycle kind, each initial admission includes +absent predicates for the other two kinds in the same command as its own +operation insertion. A competing lifecycle cannot invalidate an acknowledged +operation's later lookup by admitting the same identity. Internal manifest +publications retain distinct operation IDs. Terminal operation retention keeps +this exclusion after success or cleanup. + `ReadChanges` treats `(commit_version, event_sequence)` as an append-only log position. Its opaque cursor is bound to the root, query scope, and optional `after_commit_version`; unlike frozen search and catalog cursors, it may resume diff --git a/scripts/workbench/append_identity_recovery_gate.py b/scripts/workbench/append_identity_recovery_gate.py new file mode 100644 index 000000000..eef9b0fa5 --- /dev/null +++ b/scripts/workbench/append_identity_recovery_gate.py @@ -0,0 +1,1836 @@ +#!/usr/bin/env python3 +# Copyright 2024-2026 The NoKV Authors. +# SPDX-License-Identifier: Apache-2.0 +"""Native CLI append identity qualification using real etcd, RustFS and Holt. + +This gate starts only isolated services and keeps every RPC fault and CLI result. +The baseline mode must reproduce response-loss duplication; qualified mode checks +caller-stable identities. A local owner SIGKILL does not qualify shared recovery. +Binary build identity and current source identity are recorded independently. +""" + +from __future__ import annotations +import argparse +import base64 +import concurrent.futures +import dataclasses +import hashlib +import json +from pathlib import Path +import shutil +import socket +import struct +import subprocess +import sys +import threading +import time +import traceback +import uuid + +import live_workbench as live +import object_namespace_recovery_gate as infra +from restore_composition_gate import PINNED_RUSTFS_IMAGE +from fork_restore_recovery_gate import ( + ResponseDropProxy, + _recv_exact, + is_handshake_frame, +) + + +def unpack(data): + """Small read-only MessagePack decoder; rejects unsupported types/trailing bytes.""" + offset = 0 + + def take(n): + nonlocal offset + b = data[offset : offset + n] + offset += n + if len(b) != n: + raise ValueError("truncated MessagePack") + return b + + def number(n, signed=False): + return int.from_bytes(take(n), "big", signed=signed) + + def read(): + tag = number(1) + if tag < 0x80: + return tag + if tag >= 0xE0: + return tag - 256 + if 0xA0 <= tag <= 0xBF: + return take(tag & 31).decode() + if 0x90 <= tag <= 0x9F: + return [read() for _ in range(tag & 15)] + if 0x80 <= tag <= 0x8F: + return {read(): read() for _ in range(tag & 15)} + if tag == 0xC0: + return None + if tag in (0xC2, 0xC3): + return tag == 0xC3 + if tag in (0xCC, 0xCD, 0xCE, 0xCF): + return number(1 << (tag - 0xCC)) + if tag in (0xD0, 0xD1, 0xD2, 0xD3): + return number(1 << (tag - 0xD0), True) + if tag in (0xCA, 0xCB): + return struct.unpack( + ">f" if tag == 0xCA else ">d", take(4 if tag == 0xCA else 8) + )[0] + if tag in (0xD9, 0xDA, 0xDB): + return take(number(1 << (tag - 0xD9))).decode() + if tag in (0xC4, 0xC5, 0xC6): + return {"binary_hex": take(number(1 << (tag - 0xC4))).hex()} + if tag in (0xDC, 0xDD): + return [read() for _ in range(number(2 if tag == 0xDC else 4))] + if tag in (0xDE, 0xDF): + return {read(): read() for _ in range(number(2 if tag == 0xDE else 4))} + raise ValueError(f"unsupported MessagePack marker {tag:#x}") + + value = read() + if offset != len(data): + raise ValueError("trailing MessagePack bytes") + return value + + +class EvidenceDropProxy(ResponseDropProxy): + def __init__(self, port, timeout, stack): + super().__init__(port, timeout) + self.stack = stack + self.capture_rpc = False + self.last_drop = None + self.handshake_request = None + self.wire_schema = None + self.latest_route = None + + def wait_dropped(self): + # Called after the only faulted CLI process has exited. At that point + # no future RPC can reach this barrier; allow only thread handoff time. + if not self._dropped.wait(1): + raise infra.WorkflowFailure( + f"append exited before the armed successful {self._drop_operation} response; " + "inspect native-append-transcript.jsonl and publication-rpc-trace.jsonl" + ) + self.raise_if_failed() + + def _forward(self, client): + try: + client.settimeout(self.timeout) + with self._lock: + target = self._target + if target is None: + raise ConnectionError("proxy target missing") + with socket.create_connection(target, timeout=self.timeout) as owner: + owner.settimeout(self.timeout) + while True: + first = client.recv(4) + if not first: + return + header = first + _recv_exact(client, 4 - len(first)) + request = _recv_exact(client, int.from_bytes(header, "big")) + owner.sendall(header + request) + rh = _recv_exact(owner, 4) + response = _recv_exact(owner, int.from_bytes(rh, "big")) + callback = None + if is_handshake_frame(request): + with self._lock: + self.handshake_request = request + else: + req = unpack(request) + rsp = unpack(response) + with self._lock: + self.wire_schema = req["schema"] + self.latest_route = req["payload"]["route"] + operation = req["payload"]["operation"]["operation"] + if self.capture_rpc: + self.stack.record( + "publication-rpc-trace.jsonl", + { + "at": live.now(), + "operation": operation, + "request": req, + "response": rsp, + }, + ) + if rsp["payload"]["outcome"]["status"] == "success": + # Claim exactly one successful response under the arm lock. + # Concurrent replies must never be labelled dropped if delivered. + with self._lock: + if self._drop_operation == operation: + callback = self._on_drop + self._drop_operation = None + self._on_drop = None + if callback is not None: + self.last_drop = { + "at": live.now(), + "operation": operation, + "request": req, + "response": rsp, + "request_hex": request.hex(), + "response_hex": response.hex(), + "delivered_to_client": False, + "owner_pid": self.stack.owner.pid, + } + self.stack.record("dropped-rpc.jsonl", self.last_drop) + if callback is not None: + callback() + self._dropped.set() + return + client.sendall(rh + response) + except (ConnectionError, OSError, TimeoutError, ValueError, KeyError) as error: + with self._lock: + armed = self._drop_operation is not None + if armed: + self._errors.append(repr(error)) + self._dropped.set() + finally: + client.close() + + +def materialize(s, wb, section, path, label): + target = s.evidence_dir / (label + ".data") + s.run([*s.client_args, "materialize", wb, section, path, str(target)], label=label) + return target.read_bytes() + + +def append_loss(s): + wb = "append-response-loss" + marker = "effect-0001\n" + args = { + "id": wb, + "section": "logs", + "path": "effects.log", + "text": marker, + "content_type": "text/plain", + } + # Admit the workbench first so the one-shot targets only the append publication. + s.cli("workbench_create", {"id": wb}) + s.proxy.arm("complete_artifact_publish", lambda: s.owner.kill()) + failed = s.cli("workbench_append", args, label="append-first-ack-lost", check=False) + s.proxy.wait_dropped() + assert failed.get("status") != "success", failed + s.owner.wait(timeout=10) + s.restart() + before = materialize( + s, wb, "logs", "effects.log", "append-after-reopen-before-retry" + ) + stat_before = s.cli( + "workbench_stat", + {"id": wb, "section": "logs", "path": "effects.log"}, + label="append-stat-before-retry", + ) + retry = s.cli("workbench_append", args, label="append-new-process-same-input-retry") + after = materialize(s, wb, "logs", "effects.log", "append-after-caller-retry") + assert before == marker.encode(), before + assert after == 2 * marker.encode(), after + return { + "status": "RED_REPRODUCED", + "ownership": "NoKV CLI / Workbench append identity", + "first_call": failed, + "durable_before_retry": before.decode(), + "stat_before_retry": stat_before, + "retry": retry, + "after_retry": after.decode(), + "effect_count_before": 1, + "effect_count_after": 2, + "caller_restart": True, + "owner_sigkill_reopen": True, + "oracle": "One logical caller effect is duplicated after its success RPC is lost; new CLI process cannot reuse durable operation identity.", + } + + +def native_append( + stack, + workbench, + operation_id, + text, + *, + label, + path="effects.log", + content_type="text/plain", + max_size=1024 * 1024, + incarnation=None, + payload_form="text", + check=True, +): + command = [ + *stack.client_args, + "workspace-path", + "append", + workbench, + "logs", + path, + "--operation-id", + operation_id, + "--max-logical-size", + str(max_size), + ] + if content_type is not None: + command += ["--content-type", content_type] + if payload_form == "text": + command += ["--text", text] + elif payload_form == "base64": + command += ["--base64", base64.b64encode(text.encode()).decode()] + elif payload_form == "file": + fixture = stack.evidence_dir / (label + ".input") + fixture.write_bytes(text.encode()) + command += ["--file", str(fixture)] + else: + raise ValueError(f"unsupported test payload form: {payload_form}") + if incarnation is not None: + command += ["--expected-workspace-incarnation-id", incarnation] + process = stack.run(command, label=label, check=False) + raw = ( + process.stdout + if process.returncode == 0 + else process.stderr.strip().removeprefix("nokv: ") + ) + try: + result = json.loads(raw) + except json.JSONDecodeError as error: + raise AssertionError( + f"native append did not return structured JSON: {raw}" + ) from error + stack.record( + "native-append-transcript.jsonl", + { + "label": label, + "operation_id": operation_id, + "workbench": workbench, + "path": path, + "payload_sha256": hashlib.sha256(text.encode()).hexdigest(), + "payload_bytes": len(text.encode()), + "returncode": process.returncode, + "result": result, + }, + ) + if check: + assert process.returncode == 0 and result.get("status") == "success", result + else: + assert (process.returncode == 0) == (result.get("status") == "success"), result + return result + + +def receipt(result): + """Compare the historical publication, excluding response timing and replay flag.""" + fields = ( + "operation_id", + "artifact_revision_id", + "generation", + "logical_size", + "body_digest", + "workspace_incarnation_id", + "workspace_revision", + ) + assert all(field in result for field in fields), result + return {field: result[field] for field in fields} + + +def assert_rejected(result, operation_id): + assert result.get("status") == "error", result + assert ( + result.get("code") + and result.get("details", {}).get("operation_id") == operation_id + ), result + + +def assert_dropped_receipt(result, dropped): + observed = dropped["response"]["payload"]["outcome"]["body"] + assert observed["result"] == "published", observed + published = observed["value"] + assert result["operation_id"] == bytes(published["operation_id"]).hex(), ( + result, + published, + ) + assert result["path"] == published["target"]["path"], (result, published) + assert result["workbench_id"] == published["target"]["workbench"], ( + result, + published, + ) + for key in ("generation", "logical_size", "body_digest", "workspace_revision"): + assert result[key] == published[key], (result, published) + assert ( + result["artifact_revision_id"] == bytes(published["artifact_revision_id"]).hex() + ), (result, published) + + +def terminal_response_loss(stack): + wb, operation_id, text = "stable-response-loss", "11" * 16, "logical-event-1\n" + stack.cli("workbench_create", {"id": wb}) + stack.proxy.arm("complete_artifact_publish", lambda: stack.owner.kill()) + failed = native_append( + stack, wb, operation_id, text, label="terminal-ack-lost", check=False + ) + stack.proxy.wait_dropped() + dropped = stack.proxy.last_drop + assert failed.get("status") == "error", failed + stack.owner.wait(timeout=10) + stack.restart() + before = materialize(stack, wb, "logs", "effects.log", "terminal-after-reopen") + assert before == text.encode(), before + replay = native_append( + stack, wb, operation_id, text, label="terminal-new-process-replay" + ) + assert replay["replayed"] is True, replay + assert_dropped_receipt(replay, dropped) + again = native_append(stack, wb, operation_id, text, label="terminal-second-replay") + assert again["replayed"] is True and receipt(again) == receipt(replay), ( + again, + replay, + ) + after = materialize(stack, wb, "logs", "effects.log", "terminal-after-replays") + assert after == before, after + return { + "status": "PASS", + "first_call": failed, + "replay": replay, + "expected_effects": 1, + "actual_effects": 1, + "exact_bytes": after.decode(), + "original_rpc_receipt_verified": True, + "owner_sigkill_reopen": True, + } + + +def identity_binding_and_history(stack): + wb, operation_id, text = "identity-binding", "22" * 16, "first\n" + stack.cli("workbench_create", {"id": wb}) + original = native_append(stack, wb, operation_id, text, label="binding-original") + assert original["replayed"] is False and original["generation"] == 1, original + for form in ("base64", "file"): + replay = native_append( + stack, + wb, + operation_id, + text, + label=f"binding-{form}-replay", + payload_form=form, + ) + assert replay["replayed"] is True and receipt(replay) == receipt(original), ( + replay + ) + rejected = [] + for label, changed_text, changed in ( + ("body", "different\n", {}), + ("path", text, {"path": "other.log"}), + ("content-type", text, {"content_type": "application/octet-stream"}), + ("limit", text, {"max_size": 1024 * 1024 + 1}), + ("incarnation", text, {"incarnation": "fe" * 16}), + ): + result = native_append( + stack, + wb, + operation_id, + changed_text, + label=f"binding-mismatch-{label}", + check=False, + **changed, + ) + assert_rejected(result, operation_id) + rejected.append({"changed": label, "result": result}) + fresh_fence = native_append( + stack, + wb, + "27" * 16, + text, + label="fresh-id-wrong-incarnation", + incarnation="fe" * 16, + check=False, + ) + assert_rejected(fresh_fence, "27" * 16) + rejected.append( + {"changed": "fresh-operation-incarnation-fence", "result": fresh_fence} + ) + other = wb + "-other" + stack.cli("workbench_create", {"id": other}) + cross_workspace = native_append( + stack, + other, + operation_id, + text, + label="existing-id-different-workspace", + check=False, + ) + assert_rejected(cross_workspace, operation_id) + rejected.append({"changed": "workspace", "result": cross_workspace}) + stack.cli( + "workbench_stat", + {"id": other, "section": "logs", "path": "effects.log"}, + label="other-workspace-path-absent", + expect_error="NotFound", + ) + assert ( + materialize(stack, wb, "logs", "effects.log", "binding-after-rejections") + == text.encode() + ) + stack.cli( + "workbench_stat", + {"id": wb, "section": "logs", "path": "other.log"}, + label="mismatch-other-path-absent", + expect_error="NotFound", + ) + duplicate_text = native_append( + stack, wb, "23" * 16, text, label="new-identity-identical-bytes" + ) + assert duplicate_text["generation"] == 2, duplicate_text + assert ( + materialize(stack, wb, "logs", "effects.log", "new-identity-body") + == 2 * text.encode() + ) + tail = native_append( + stack, wb, "24" * 16, "later\n", label="later-independent-writer" + ) + historical = native_append( + stack, wb, operation_id, text, label="history-after-later-writer" + ) + assert historical["replayed"] is True and receipt(historical) == receipt( + original + ), historical + assert ( + materialize(stack, wb, "logs", "effects.log", "history-live-head") + == (2 * text + "later\n").encode() + ) + stack.run( + [ + *stack.client_args, + "workspace-path", + "remove", + wb, + "logs", + "effects.log", + "--expected-generation", + str(tail["generation"]), + "--request-id", + "25" * 16, + ], + label="remove-historical-target", + ) + removed_replay = native_append( + stack, wb, operation_id, text, label="history-after-remove" + ) + assert receipt(removed_replay) == receipt(original), removed_replay + stack.cli( + "workbench_stat", + {"id": wb, "section": "logs", "path": "effects.log"}, + label="removed-path-remains-absent", + expect_error="NotFound", + ) + replacement = native_append( + stack, wb, "26" * 16, "replacement\n", label="recreate-path-with-new-identity" + ) + replacement_replay = native_append( + stack, wb, operation_id, text, label="history-after-replacement" + ) + assert receipt(replacement_replay) == receipt(original), replacement_replay + assert ( + materialize(stack, wb, "logs", "effects.log", "replacement-unchanged") + == b"replacement\n" + ) + stack.restart() + reopened_replay = native_append( + stack, wb, operation_id, text, label="history-after-reopen" + ) + assert receipt(reopened_replay) == receipt(original), reopened_replay + assert ( + materialize(stack, wb, "logs", "effects.log", "replacement-after-reopen") + == b"replacement\n" + ) + return { + "status": "PASS", + "original": original, + "rejected_requests": rejected, + "same_bytes_distinct_identity": duplicate_text, + "later_writer": tail, + "historical_receipt_survived": [ + "later_writer", + "remove", + "replacement", + "owner_reopen", + ], + "replacement": replacement, + "same_bytes_text_base64_file_replay": True, + } + + +def concurrent_same_identity(stack): + wb, operation_id, text, count = ( + "concurrent-same-identity", + "33" * 16, + "one-event\n", + 8, + ) + stack.cli("workbench_create", {"id": wb}) + barrier = threading.Barrier(count) + + def invoke(index): + barrier.wait(timeout=30) + return native_append( + stack, + wb, + operation_id, + text, + label=f"concurrent-same-{index:02d}", + check=False, + ) + + with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool: + initial = list(pool.map(invoke, range(count))) + # Repeating the SAME identity is allowed even when one process observed an + # intermediate pending state. There is deliberately no fresh-ID fallback. + replay = native_append( + stack, wb, operation_id, text, label="concurrent-same-terminal-replay" + ) + for result in initial: + if result.get("status") == "success": + assert receipt(result) == receipt(replay), (result, replay) + else: + assert_rejected(result, operation_id) + body = materialize(stack, wb, "logs", "effects.log", "concurrent-same-final") + assert body == text.encode() and replay["generation"] == 1, (body, replay) + stack.restart() + reopened = native_append( + stack, wb, operation_id, text, label="concurrent-same-after-reopen" + ) + assert receipt(reopened) == receipt(replay), reopened + assert ( + materialize(stack, wb, "logs", "effects.log", "concurrent-same-reopened-bytes") + == body + ) + return { + "status": "PASS", + "parallel_cli_processes": count, + "initial_outcomes": initial, + "canonical_receipt": replay, + "actual_effects": 1, + "owner_reopen_verified": True, + } + + +def begin_response_loss(stack, *, change_live): + suffix = "with-drift" if change_live else "without-drift" + wb, operation_id, text = ( + "begin-loss-" + suffix, + ("44" if change_live else "45") * 16, + "uncertain-event\n", + ) + stack.cli("workbench_create", {"id": wb}) + stack.proxy.arm("begin_artifact_publish", lambda: stack.owner.kill()) + failed = native_append( + stack, wb, operation_id, text, label="begin-ack-lost-" + suffix, check=False + ) + stack.proxy.wait_dropped() + assert failed.get("status") == "error", failed + stack.owner.wait(timeout=10) + stack.restart() + if change_live: + native_append( + stack, wb, "46" * 16, "independent\n", label="begin-drift-later-writer" + ) + outcomes = [ + native_append( + stack, + wb, + operation_id, + text, + label=f"begin-retry-{suffix}-{index}", + check=False, + ) + for index in range(2) + ] + for result in outcomes: + if result.get("status") != "success": + assert_rejected(result, operation_id) + assert result.get("details", {}).get("state"), result + if change_live: + assert all(result.get("status") == "error" for result in outcomes), outcomes + body = materialize(stack, wb, "logs", "effects.log", "begin-drift-final") + assert body == b"independent\n", body + else: + stat = stack.cli( + "workbench_stat", + {"id": wb, "section": "logs", "path": "effects.log"}, + label="begin-no-drift-stat", + check=False, + ) + if stat.get("status") == "success": + body = materialize(stack, wb, "logs", "effects.log", "begin-no-drift-final") + assert body == text.encode(), body + else: + assert stat.get("code") == "NotFound", stat + body = b"" + successful = [result for result in outcomes if result.get("status") == "success"] + if successful: + assert not change_live and body == text.encode(), (body, successful) + assert all( + receipt(result) == receipt(successful[0]) for result in successful + ), successful + stack.restart() + if body: + assert ( + materialize(stack, wb, "logs", "effects.log", "begin-reopened-" + suffix) + == body + ) + else: + stack.cli( + "workbench_stat", + {"id": wb, "section": "logs", "path": "effects.log"}, + label="begin-reopened-absent", + expect_error="NotFound", + ) + return { + "status": "PASS", + "first_call": failed, + "same_identity_outcomes": outcomes, + "live_changed_before_retry": change_live, + "uncertain_append_effects": body.count(text.encode()), + "completion_qualified": bool(successful), + "safety_qualified": True, + "oracle": "A pending operation may remain explicitly unresolved; it must not switch identity or rebase onto a later writer.", + } + + +def distinct_identity_contention(stack): + wb, count = "distinct-identity-contention", 8 + stack.cli("workbench_create", {"id": wb}) + native_append(stack, wb, "50" * 16, "seed\n", label="contention-seed") + barrier = threading.Barrier(count) + + def invoke(index): + barrier.wait(timeout=30) + return native_append( + stack, + wb, + f"{0x60 + index:02x}" * 16, + f"event-{index:02d}\n", + label=f"distinct-{index:02d}", + check=False, + ) + + with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool: + initial = list(pool.map(invoke, range(count))) + final = [] + for index in range(count): + operation_id = f"{0x60 + index:02x}" * 16 + result = native_append( + stack, + wb, + operation_id, + f"event-{index:02d}\n", + label=f"distinct-same-id-replay-{index:02d}", + check=False, + ) + if result.get("status") != "success": + assert_rejected(result, operation_id) + if initial[index].get("status") == "success": + assert result.get("status") == "success" and receipt( + initial[index] + ) == receipt(result), (initial[index], result) + final.append(result) + body = materialize(stack, wb, "logs", "effects.log", "distinct-final").decode() + assert body.splitlines().count("seed") == 1, body + for index, result in enumerate(final): + copies = body.splitlines().count(f"event-{index:02d}") + assert copies == (1 if result.get("status") == "success" else 0), ( + index, + copies, + result, + ) + stack.restart() + assert ( + materialize(stack, wb, "logs", "effects.log", "distinct-after-reopen").decode() + == body + ) + return { + "status": "PASS", + "parallel_cli_processes": count, + "initial_outcomes": initial, + "same_identity_replay_outcomes": final, + "exact_final_body": body, + "accepted_effects_exactly_once": True, + "unaccepted_effects_absent": True, + } + + +def multiblock_replay(stack): + wb, operation_id = "multiblock-identity", "77" * 16 + stack.cli("workbench_create", {"id": wb}) + size = 5 * 1024 * 1024 + 31 + pattern = "append-multiblock-fixture\n" + text = (pattern * ((size // len(pattern)) + 1))[:size] + assert len(text.encode()) == size + original = native_append( + stack, + wb, + operation_id, + text, + label="multiblock-original", + payload_form="file", + max_size=16 * 1024 * 1024, + ) + assert original["generation"] == 1 and original["logical_size"] == size, original + tail = "after-multiblock\n" + native_append( + stack, wb, "78" * 16, tail, label="multiblock-tail", max_size=16 * 1024 * 1024 + ) + replay = native_append( + stack, + wb, + operation_id, + text, + label="multiblock-original-replay", + payload_form="file", + max_size=16 * 1024 * 1024, + ) + assert replay["replayed"] is True and receipt(replay) == receipt(original), replay + expected = (text + tail).encode() + body = materialize(stack, wb, "logs", "effects.log", "multiblock-live") + assert body == expected + stack.restart() + body = materialize(stack, wb, "logs", "effects.log", "multiblock-reopened") + assert body == expected + return { + "status": "PASS", + "original_bytes": size, + "default_4mib_blocks": 2, + "final_bytes": len(body), + "sha256": hashlib.sha256(body).hexdigest(), + "original_receipt": original, + "replay_receipt": replay, + "owner_reopen_verified": True, + } + + +PYTHON_INTEROP_PROGRAM = r""" +import hashlib, json, sys +from pathlib import Path +import nokv +from nokv import _native +config = json.load(sys.stdin) +client = nokv.Client( + config["root_id"], + nokv.RoutingConfig.etcd([config["etcd_endpoint"]], key_prefix=config["etcd_prefix"]), + nokv.ObjectStoreConfig.s3( + config["bucket"], root=config["object_root"], endpoint=config["object_endpoint"], + access_key_id=config["access_key"], secret_access_key=config["secret_key"], + ), + workbench_root=config["workbench_root"], +) +request = config["request"] +identity = { + "python": sys.executable, "module": nokv.__file__, "version": nokv.__version__, + "native_module": _native.__file__, + "native_sha256": hashlib.sha256(Path(_native.__file__).read_bytes()).hexdigest(), +} +try: + if request["action"] == "seed": + result = client.publish_bytes( + request["workbench"], "logs/effects.log", request["text"].encode(), + content_type=request.get("content_type", "text/plain"), producer="downstream-harness", + manifest_identity="run-42", index_fields=[("run.step", "signed", "7")], + ) + else: + result = client.append_bytes( + request["workbench"], "logs/effects.log", request["text"].encode(), + request["operation_id"], content_type=request.get("content_type", "text/plain"), max_logical_size=1048576, + expected_workspace_incarnation_id=request.get("incarnation"), + ) + metadata = client.stat(request["workbench"], "logs/effects.log") + print(json.dumps({"status": "success", "result": result, "metadata": metadata, "identity": identity})) +except Exception as error: + print(json.dumps({ + "status": "error", "type": type(error).__name__, "message": str(error), + "operation_id": getattr(error, "operation_id", None), "state": getattr(error, "state", None), + "code": getattr(error, "code", None), "cause_code": getattr(error, "cause_code", None), + "retryable": getattr(error, "retryable", None), "expected": getattr(error, "expected", None), + "is_append_error": isinstance(error, nokv.AppendError), + "is_incarnation_mismatch": isinstance(error, nokv.WorkspaceIncarnationMismatch), + "identity": identity, + })) +""" + + +def python_call(stack, request, label): + config = { + "root_id": stack.root_id, + "etcd_endpoint": stack.etcd_endpoint, + "etcd_prefix": stack.etcd_prefix, + "bucket": stack.bucket, + "object_root": stack.object_root, + "object_endpoint": stack.object_endpoint, + "access_key": stack.access_key, + "secret_key": stack.secret_key, + "workbench_root": stack.config.workbench_root, + "request": request, + } + process = stack.run( + [stack.python_executable, "-c", PYTHON_INTEROP_PROGRAM], + label=label, + input_text=json.dumps(config), + ) + result = json.loads(process.stdout) + stack.record( + "python-interop-transcript.jsonl", + {"label": label, "request": request, "response": result}, + ) + return result + + +def python_interoperability(stack): + wb, operation_id = "python-interoperability", "88" * 16 + stack.cli("workbench_create", {"id": wb}) + seed = python_call( + stack, + {"action": "seed", "workbench": wb, "text": "seed\n"}, + "python-publish-metadata-seed", + ) + assert seed["status"] == "success", seed + expected_metadata = { + "producer": "downstream-harness", + "manifest_identity": "run-42", + "index_fields": [{"field_id": "run.step", "kind": "signed", "value": 7}], + } + for field, expected_value in expected_metadata.items(): + assert seed["metadata"][field] == expected_value, (field, seed) + original = native_append( + stack, wb, operation_id, "cli\n", label="interop-cli-original" + ) + replay = python_call( + stack, + { + "action": "append", + "workbench": wb, + "text": "cli\n", + "operation_id": operation_id, + }, + "interop-python-replays-cli", + ) + assert replay["status"] == "success" and replay["result"]["replayed"] is True, ( + replay + ) + assert receipt(replay["result"]) == receipt(original), (replay, original) + for field in ("producer", "manifest_identity", "index_fields"): + assert replay["metadata"][field] == expected_metadata[field], ( + field, + seed, + replay, + ) + python_original = python_call( + stack, + { + "action": "append", + "workbench": wb, + "text": "python\n", + "operation_id": "89" * 16, + }, + "interop-python-original", + ) + assert python_original["status"] == "success", python_original + cli_replay = native_append( + stack, wb, "89" * 16, "python\n", label="interop-cli-replays-python" + ) + assert cli_replay["replayed"] is True and receipt(cli_replay) == receipt( + python_original["result"] + ), cli_replay + mismatch = python_call( + stack, + { + "action": "append", + "workbench": wb, + "text": "different\n", + "operation_id": operation_id, + }, + "interop-python-intent-mismatch", + ) + assert ( + mismatch["status"] == "error" + and mismatch["is_append_error"] is True + and mismatch["operation_id"] == operation_id + and mismatch["code"] == "AppendUnresolved" + and mismatch["cause_code"] == "RequestReplayMismatch" + and mismatch["retryable"] is False + ), mismatch + fence = python_call( + stack, + { + "action": "append", + "workbench": wb, + "text": "fenced\n", + "operation_id": "8a" * 16, + "incarnation": "fe" * 16, + }, + "interop-python-incarnation-fence", + ) + assert ( + fence["status"] == "error" + and fence["is_incarnation_mismatch"] is True + and fence["operation_id"] == "8a" * 16 + ), fence + expected = b"seed\ncli\npython\n" + assert ( + materialize(stack, wb, "logs", "effects.log", "interop-final-bytes") == expected + ) + stack.restart() + reopened = python_call( + stack, + { + "action": "append", + "workbench": wb, + "text": "cli\n", + "operation_id": operation_id, + }, + "interop-python-replay-after-owner-reopen", + ) + assert reopened["status"] == "success" and receipt(reopened["result"]) == receipt( + original + ), reopened + assert ( + materialize(stack, wb, "logs", "effects.log", "interop-reopened-bytes") + == expected + ) + for field, expected_value in expected_metadata.items(): + assert reopened["metadata"][field] == expected_value, (field, reopened) + inheritance = default_type_inheritance(stack, wb + "-default-type") + return { + "status": "PASS", + "default_content_type_inheritance": inheritance, + "python_identity": seed["identity"], + "cli_original": original, + "python_replay": replay["result"], + "python_original": python_original["result"], + "cli_replay": cli_replay, + "typed_mismatch": mismatch, + "typed_incarnation_fence": fence, + "metadata_inheritance_verified": True, + "owner_reopen_verified": True, + "exact_bytes": expected.decode(), + } + + +def pack_rpc_value(value): + """Encode the small JSON-shaped MessagePack request used by this gate only.""" + if value is None: + return b"\xc0" + if isinstance(value, bool): + return b"\xc3" if value else b"\xc2" + if isinstance(value, int) and value >= 0: + if value < 128: + return bytes([value]) + for limit, marker, width in ( + (256, 0xCC, 1), + (65536, 0xCD, 2), + (2**32, 0xCE, 4), + (2**64, 0xCF, 8), + ): + if value < limit: + return bytes([marker]) + value.to_bytes(width, "big") + if isinstance(value, str): + body = value.encode() + length = len(body) + if length < 32: + return bytes([0xA0 | length]) + body + for limit, marker, width in ( + (256, 0xD9, 1), + (65536, 0xDA, 2), + (2**32, 0xDB, 4), + ): + if length < limit: + return bytes([marker]) + length.to_bytes(width, "big") + body + if isinstance(value, list): + count = len(value) + prefix = ( + bytes([0x90 | count]) if count < 16 else b"\xdc" + count.to_bytes(2, "big") + ) + return prefix + b"".join(pack_rpc_value(item) for item in value) + if isinstance(value, dict): + count = len(value) + prefix = ( + bytes([0x80 | count]) if count < 16 else b"\xde" + count.to_bytes(2, "big") + ) + return prefix + b"".join( + pack_rpc_value(key) + pack_rpc_value(item) for key, item in value.items() + ) + raise ValueError(f"unsupported gate request value: {type(value).__name__}") + + +def raw_rpc(stack, operation, label): + """Use a captured valid native handshake and route for one public RPC.""" + with stack.proxy._lock: + handshake = stack.proxy.handshake_request + request = { + "schema": stack.proxy.wire_schema, + "payload": { + "request_id": list(uuid.uuid4().bytes), + "route": dict(stack.proxy.latest_route), + "operation": operation, + }, + } + assert handshake is not None and request["schema"], ( + "native CLI must establish the wire profile first" + ) + encoded = pack_rpc_value(request) + assert unpack(encoded) == request + with socket.create_connection( + ("127.0.0.1", stack.proxy_port), timeout=stack.timeout + ) as client: + client.settimeout(stack.timeout) + client.sendall(len(handshake).to_bytes(4, "big") + handshake) + response = _recv_exact(client, int.from_bytes(_recv_exact(client, 4), "big")) + assert is_handshake_frame(response), response.hex() + client.sendall(len(encoded).to_bytes(4, "big") + encoded) + response = unpack( + _recv_exact(client, int.from_bytes(_recv_exact(client, 4), "big")) + ) + stack.record( + "native-rpc-transcript.jsonl", + {"label": label, "request": request, "response": response}, + ) + return response["payload"]["outcome"] + + +def cross_kind_identity_collision(stack): + wb, operation_id, text = "cross-kind-collision", "99" * 16, "published-once\n" + stack.cli("workbench_create", {"id": wb}) + original = native_append( + stack, wb, operation_id, text, label="cross-kind-original-append" + ) + query = { + "operation": "get_operation", + "request": {"operation_id": list(bytes.fromhex(operation_id))}, + } + before = raw_rpc(stack, query, "cross-kind-operation-before") + assert ( + before["status"] == "success" + and before["body"]["value"]["kind"] == "artifact_publish" + ), before + collision = raw_rpc( + stack, + { + "operation": "commit", + "request": { + "operation_id": list(bytes.fromhex(operation_id)), + "workbench": wb, + "workspace_incarnation_id": list( + bytes.fromhex(original["workspace_incarnation_id"]) + ), + "commit_id": [0xA1] * 32, + "content_digest": "sha256:" + "a2" * 32, + "manifest_digest": "sha256:" + "a3" * 32, + "projection_input_digest": [0xA4] * 32, + "tree_manifest_revision_id": [0xA5] * 16, + "replace": False, + "run_manifest_condition": "create_only", + "expected_head_generation": None, + "parents": [], + "producer": None, + "lineage_projection": [], + }, + }, + "cross-kind-commit-reuses-append-id", + ) + assert collision["status"] == "failure", collision + assert collision["body"]["code"] in ("conflict", "request_replay_mismatch"), ( + collision + ) + after = raw_rpc(stack, query, "cross-kind-operation-after") + assert after["status"] == "success", after + before_value, after_value = before["body"]["value"], after["body"]["value"] + assert ( + after_value["kind"] == "artifact_publish" + and after_value["state"] == "succeeded" + ), after + assert after_value["result"] == before_value["result"], (before, after) + replay = native_append( + stack, wb, operation_id, text, label="cross-kind-original-still-replays" + ) + assert receipt(replay) == receipt(original), (original, replay) + assert ( + materialize(stack, wb, "logs", "effects.log", "cross-kind-bytes") + == text.encode() + ) + stack.restart() + reopened = native_append( + stack, wb, operation_id, text, label="cross-kind-after-owner-reopen" + ) + assert receipt(reopened) == receipt(original), (original, reopened) + assert ( + materialize(stack, wb, "logs", "effects.log", "cross-kind-reopened-bytes") + == text.encode() + ) + return { + "status": "PASS", + "colliding_commit": collision, + "original_receipt": original, + "after_collision_receipt": replay, + "operation_kind_preserved": True, + "owner_reopen_verified": True, + } + + +def default_type_inheritance(stack, wb): + """Both surfaces preserve an existing nondefault type when override is absent.""" + stack.cli("workbench_create", {"id": wb}) + custom_type = "application/x-harness-events" + seed = python_call( + stack, + { + "action": "seed", + "workbench": wb, + "text": "seed\n", + "content_type": custom_type, + }, + "inherit-custom-type-seed", + ) + expected_metadata = { + "content_type": custom_type, + "producer": "downstream-harness", + "manifest_identity": "run-42", + "index_fields": [{"field_id": "run.step", "kind": "signed", "value": 7}], + } + assert seed["status"] == "success", seed + for field, expected in expected_metadata.items(): + assert seed["metadata"][field] == expected, (field, seed) + request = { + "action": "append", + "workbench": wb, + "text": "python-inherit\n", + "operation_id": "aa" * 16, + "content_type": None, + } + original = python_call(stack, request, "inherit-python-original") + replay = python_call(stack, request, "inherit-python-same-none-replay") + assert original["status"] == replay["status"] == "success", (original, replay) + assert replay["result"]["replayed"] is True and receipt( + original["result"] + ) == receipt(replay["result"]), (original, replay) + changed = python_call( + stack, + {**request, "content_type": custom_type}, + "inherit-none-to-explicit-mismatch", + ) + assert ( + changed["status"] == "error" + and changed["cause_code"] == "RequestReplayMismatch" + ), changed + cli_original = native_append( + stack, + wb, + "ac" * 16, + "cli-inherit\n", + label="inherit-cli-original", + content_type=None, + ) + cli_replay = native_append( + stack, + wb, + "ac" * 16, + "cli-inherit\n", + label="inherit-cli-replay", + content_type=None, + ) + assert cli_replay["replayed"] is True and receipt(cli_original) == receipt( + cli_replay + ), cli_replay + stack.restart() + reopened = python_call(stack, request, "inherit-python-after-reopen") + assert reopened["status"] == "success" and receipt(reopened["result"]) == receipt( + original["result"] + ), reopened + for field, expected in expected_metadata.items(): + assert ( + replay["metadata"][field] == expected + and reopened["metadata"][field] == expected + ), (field, replay, reopened) + body = materialize(stack, wb, "logs", "effects.log", "inherit-reopened-bytes") + assert body == b"seed\npython-inherit\ncli-inherit\n", body + return { + "status": "PASS", + "expected_metadata": expected_metadata, + "python_original": original["result"], + "cli_original": cli_original, + "none_to_explicit_mismatch": changed, + "exact_bytes": body.decode(), + "owner_reopen_verified": True, + } + + +class IsolatedStack: + def __init__( + self, + evidence_dir: str | Path, + *, + binary: Path, + source: Path, + label="append", + timeout=60, + ): + self.source = source.resolve() + self.evidence_dir = Path(evidence_dir).resolve() + self.evidence = live.Evidence(self.evidence_dir) + self.evidence.prepare() + self.label = "nokv-audit-" + label + "-" + uuid.uuid4().hex[:10] + self.timeout = timeout + self.head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=self.source, text=True + ).strip() + + def ident(name): + return hashlib.sha256((self.label + name).encode()).hexdigest()[:32] + + self.root_id, self.agent_id, self.shard_id = [ + ident(x) for x in ("root", "agent", "shard") + ] + ( + self.etcd_port, + self.peer_port, + self.s3_port, + self.console_port, + self.owner_port, + self.proxy_port, + ) = [infra.free_port() for _ in range(6)] + self.etcd_endpoint = f"http://127.0.0.1:{self.etcd_port}" + self.peer_endpoint = f"http://127.0.0.1:{self.peer_port}" + self.object_endpoint = f"http://127.0.0.1:{self.s3_port}" + self.etcd_prefix = "/nokv/" + self.label + self.bucket = self.label + self.object_root = "append-identity-gate" + self.object_region = "us-east-1" + self.access_key = self.secret_key = "rustfsadmin" + self.binary = binary.resolve() + self.container = self.label + "-rustfs" + self.volume = self.container + "-data" + executables = {} + for name in ("aws", "docker", "etcd", "etcdctl"): + executable = shutil.which(name) + if executable is None: + raise infra.NotQualified(f"required executable is unavailable: {name}") + executables[name] = Path(executable) + if not self.binary.is_file(): + raise infra.NotQualified(f"NoKV binary is unavailable: {self.binary}") + self.aws = executables["aws"] + self.docker = executables["docker"] + self.etcdctl = executables["etcdctl"] + self.etcd_binary = executables["etcd"] + self.aws_env = infra.aws_environment(self.access_key, self.secret_key) + self.proxy = EvidenceDropProxy(self.proxy_port, timeout, self) + self.advertise = f"127.0.0.1:{self.proxy_port}" + self.config = live.parse_args( + [ + "--nokv-bin", + str(self.binary), + "--evidence-dir", + str(self.evidence_dir), + "--metadata-dir", + str(self.evidence_dir / "metadata"), + "--root-id", + self.root_id, + "--agent-id", + self.agent_id, + "--logical-shard-id", + self.shard_id, + "--agent-name", + self.agent_id, + "--etcd-endpoint", + self.etcd_endpoint, + "--etcd-key-prefix", + self.etcd_prefix, + "--server-bind", + f"127.0.0.1:{self.owner_port}", + "--advertise-endpoint", + self.advertise, + "--object-endpoint", + self.object_endpoint, + "--object-bucket", + self.bucket, + "--object-root", + self.object_root, + "--object-access-key-id", + self.access_key, + "--object-secret-access-key", + self.secret_key, + "--command-timeout-seconds", + str(timeout), + ] + ) + self.processes = [] + self.logs = [] + self.lock = threading.Lock() + self.owner = None + self.epoch = 0 + self.closed = False + + def record(self, name, value): + with self.lock: + self.evidence.line(name, value) + + def json(self, name, value): + self.evidence.json(name, value) + + def run( + self, + command, + *, + label="command", + check=True, + env=None, + timeout=None, + input_text=None, + ): + command = list(map(str, command)) + started = live.now() + result = subprocess.run( + command, + cwd=self.source, + text=True, + capture_output=True, + input=input_text, + env=env, + timeout=timeout or self.timeout, + ) + redacted = live.redact_argv(command) + redacted = [ + a.split("=")[0] + "=" + if a.startswith(("RUSTFS_ACCESS_KEY=", "RUSTFS_SECRET_KEY=")) + else a + for a in redacted + ] + self.record( + "processes.jsonl", + { + "label": label, + "argv": redacted, + "started_at": started, + "ended_at": live.now(), + "returncode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + }, + ) + if check and result.returncode: + raise RuntimeError(f"{label}: {result.stderr or result.stdout}") + return result + + def start(self, label, command): + f = (self.evidence_dir / f"{label}.log").open("w") + self.logs.append(f) + p = subprocess.Popen( + list(map(str, command)), + cwd=self.source, + stdout=f, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + self.processes.append(p) + self.record( + "processes.jsonl", + { + "label": label, + "pid": p.pid, + "argv": live.redact_argv(list(map(str, command))), + "started_at": live.now(), + }, + ) + return p + + @property + def client_args(self): + return live.client_args(self.config) + + def cli( + self, + tool, + arguments, + *, + label=None, + extra=(), + expect_error=None, + check=True, + config=None, + ): + cfg = config or self.config + p = self.run( + [ + *live.client_args(cfg), + *extra, + "workbench", + tool, + live.canonical_json(arguments), + ], + label=label or tool, + check=False, + ) + raw = p.stdout if not p.returncode else p.stderr.strip().removeprefix("nokv: ") + try: + value = json.loads(raw) + except json.JSONDecodeError: + value = {"unparsed": raw, "returncode": p.returncode} + self.record( + "native-cli-transcript.jsonl", + { + "label": label or tool, + "tool": tool, + "arguments": arguments, + "extra": list(extra), + "returncode": p.returncode, + "result": value, + }, + ) + if expect_error is not None: + assert p.returncode and value.get("code") == expect_error, value + elif check: + assert p.returncode == 0 and value.get("status") == "success", value + return value + + def restart(self, *, kill=True): + if self.owner is not None and self.owner.poll() is None: + if kill: + self.owner.kill() + self.owner.wait(timeout=10) + else: + live.stop(self.owner) + infra.wait_session_absent( + self.etcdctl, + self.etcd_endpoint, + f"{self.etcd_prefix}/sessions/{self.shard_id}", + self.source, + self.timeout, + ) + self.epoch += 1 + cfg = dataclasses.replace(self.config, metadata_mode="reopen") + self.owner = self.start(f"owner-{self.epoch}", live.server_command(cfg)) + infra.wait_tcp(self.owner, self.owner_port, self.timeout) + self.record( + "faults.jsonl", + { + "kind": "owner-kill-reopen" if kill else "owner-clean-reopen", + "at": live.now(), + "owner_number": self.epoch, + }, + ) + + def source_snapshot(self): + """Fingerprint tracked and newly added product sources without equating them to build input.""" + paths = set() + for options in ((), ("--others", "--exclude-standard")): + raw = subprocess.check_output( + [ + "git", + "ls-files", + "-z", + *options, + "--", + "Cargo.toml", + "Cargo.lock", + "crates", + ], + cwd=self.source, + ) + paths.update(path.decode() for path in raw.split(b"\0") if path) + files = { + path: live.digest_file(self.source / path) + for path in sorted(paths) + if (self.source / path).is_file() + } + manifest = { + "files": files, + "sha256": hashlib.sha256(live.canonical_json(files).encode()).hexdigest(), + "qualification": "current source snapshot; binary hash and embedded build identity are independent evidence", + } + self.json("source-files.json", manifest) + return {"file": "source-files.json", "sha256": manifest["sha256"]} + + def __enter__(self): + try: + self.run( + [self.docker, "volume", "create", self.volume], label="create-volume" + ) + self.run( + infra.rustfs_container_command( + self.docker, + container=self.container, + volume=self.volume, + s3_port=self.s3_port, + console_port=self.console_port, + image=PINNED_RUSTFS_IMAGE, + access_key=self.access_key, + secret_key=self.secret_key, + ), + label="start-rustfs", + timeout=180, + ) + infra.wait_rustfs( + self.aws, + self.object_endpoint, + self.aws_env, + self.source, + self.timeout, + docker=self.docker, + container=self.container, + ) + self.run( + infra.aws_command( + self.aws, + self.object_endpoint, + "s3api", + "create-bucket", + "--bucket", + self.bucket, + ), + label="create-bucket", + env=self.aws_env, + ) + peer = self.peer_endpoint + self.etcd = self.start( + "etcd", + [ + self.etcd_binary, + "--name", + self.label, + "--data-dir", + self.evidence_dir / "etcd-data", + "--listen-client-urls", + self.etcd_endpoint, + "--advertise-client-urls", + self.etcd_endpoint, + "--listen-peer-urls", + peer, + "--initial-advertise-peer-urls", + peer, + "--initial-cluster", + f"{self.label}={peer}", + "--initial-cluster-state", + "new", + "--log-level", + "warn", + ], + ) + infra.wait_etcd( + self.etcdctl, + self.etcd_endpoint, + self.etcd, + self.source, + self.timeout, + ) + identity = json.loads( + self.run( + [self.binary, "version", "--json"], label="binary-identity" + ).stdout + ) + assert identity["holt"]["version"] == "0.8.6", identity + self.json( + "environment.json", + { + "source_head": self.head, + "source_directory": str(self.source), + "harness_path": str(Path(__file__).resolve()), + "harness_sha256": live.digest_file(Path(__file__)), + "binary_path": str(self.binary), + "binary_build_head": identity["git_commit"], + "build_head_matches_source_head": identity["git_commit"] + == self.head, + "source_snapshot": self.source_snapshot(), + "identity": identity, + "source_diff_sha256": hashlib.sha256( + subprocess.check_output( + ["git", "diff", "HEAD"], cwd=self.source + ) + ).hexdigest(), + "binary_sha256": live.digest_file(self.binary), + "provider_image": PINNED_RUSTFS_IMAGE, + "schema_sha256": live.digest_file( + self.source / "crates/nokv-agent/workbench_contract_schema.json" + ), + "started_at": live.now(), + "root_id": self.root_id, + "shard_id": self.shard_id, + "etcd_endpoint": self.etcd_endpoint, + "object_endpoint": self.object_endpoint, + "object_root": self.object_root, + "bucket": self.bucket, + "profile": "Holt local sync, isolated etcd, real S3/RustFS, native CLI", + }, + ) + self.run(live.provision_command(self.config), label="provision") + if self.proxy: + self.proxy.set_target(self.owner_port) + self.proxy.start() + self.epoch = 1 + self.owner = self.start("owner-1", live.server_command(self.config)) + infra.wait_tcp(self.owner, self.owner_port, self.timeout) + return self + except BaseException: + self.close() + raise + + def close(self): + if self.closed: + return + self.closed = True + cleanup = [] + for p in reversed(self.processes): + if p.poll() is None: + live.stop(p) + if self.proxy: + try: + self.proxy.close() + except Exception as e: + cleanup.append(str(e)) + for f in self.logs: + f.close() + self.run( + [self.docker, "logs", "--tail", "100", self.container], + label="rustfs-logs", + check=False, + ) + for args, label in ( + ([self.docker, "rm", "-f", self.container], "remove-rustfs"), + ([self.docker, "volume", "rm", self.volume], "remove-volume"), + ): + p = self.run(args, label=label, check=False) + cleanup.append({"step": label, "exit_code": p.returncode}) + self.json("cleanup.json", {"at": live.now(), "results": cleanup}) + + def __exit__(self, kind, value, tb): + self.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence-dir", type=Path, required=True) + parser.add_argument("--nokv-bin", type=Path, required=True) + parser.add_argument("--source-dir", type=Path, required=True) + parser.add_argument( + "--mode", choices=("baseline", "qualified"), default="qualified" + ) + parser.add_argument("--timeout-seconds", type=int, default=60) + parser.add_argument( + "--python-executable", + type=Path, + help="Isolated Python environment containing the candidate NoKV extension; enables cross-surface E2E.", + ) + parser.add_argument( + "--scenario", + default="all", + choices=( + "all", + "legacy_response_loss", + "terminal_response_loss", + "identity_binding_and_history", + "concurrent_same_identity", + "begin_response_loss", + "begin_response_loss_with_drift", + "distinct_identity_contention", + "multiblock_replay", + "python_interoperability", + "cross_kind_identity_collision", + ), + help="Run one acceptance scenario; all is required for complete qualification.", + ) + args = parser.parse_args() + results = { + "schema": "nokv.append_identity_recovery_gate.v1", + "command": sys.argv, + "started_at": live.now(), + "mode": args.mode, + "selection": args.scenario, + "full_matrix": False, + "qualified_scopes": [], + "python_e2e_enabled": args.python_executable is not None, + "scenarios": {}, + } + stack = IsolatedStack( + args.evidence_dir, + binary=args.nokv_bin, + source=args.source_dir, + timeout=args.timeout_seconds, + ) + stack.python_executable = args.python_executable + try: + with stack: + if args.mode == "baseline": + scenarios = [("legacy_response_loss", append_loss)] + else: + stack.proxy.capture_rpc = True + scenarios = [ + ("terminal_response_loss", terminal_response_loss), + ("identity_binding_and_history", identity_binding_and_history), + ("concurrent_same_identity", concurrent_same_identity), + ( + "begin_response_loss", + lambda value: begin_response_loss(value, change_live=False), + ), + ( + "begin_response_loss_with_drift", + lambda value: begin_response_loss(value, change_live=True), + ), + ("distinct_identity_contention", distinct_identity_contention), + ("multiblock_replay", multiblock_replay), + ("cross_kind_identity_collision", cross_kind_identity_collision), + ] + if args.python_executable is not None: + scenarios.append( + ("python_interoperability", python_interoperability) + ) + if args.scenario != "all": + scenarios = [ + (name, scenario) + for name, scenario in scenarios + if name == args.scenario + ] + if not scenarios: + raise ValueError( + f"scenario {args.scenario!r} is unavailable in {args.mode!r} mode" + ) + for name, scenario in scenarios: + print(f"START {name}", flush=True) + started = time.monotonic() + try: + result = scenario(stack) + except Exception as error: + results["scenarios"][name] = { + "status": "FAIL", + "error": repr(error), + "wall_seconds": time.monotonic() - started, + } + stack.json("results.json", results) + raise + result["wall_seconds"] = time.monotonic() - started + results["scenarios"][name] = result + stack.json("results.json", results) + print(f"{name}: {result['status']}", flush=True) + results["qualified_scopes"] = [ + name + for name, result in results["scenarios"].items() + if result["status"] == "PASS" + ] + results["full_matrix"] = ( + args.mode == "qualified" + and args.scenario == "all" + and args.python_executable is not None + and "python_interoperability" in results["qualified_scopes"] + ) + results["status"] = ( + "RED_REPRODUCED" + if args.mode == "baseline" + else "GREEN" + if results["full_matrix"] + else "GREEN_PARTIAL" + ) + results["completed_at"] = live.now() + stack.json("results.json", results) + print( + json.dumps( + {"status": results["status"], "evidence_dir": str(stack.evidence_dir)} + ), + flush=True, + ) + return 0 + except Exception: + results["status"] = "GATE_ERROR" + results["traceback"] = traceback.format_exc() + stack.json("results.json", results) + print(results["traceback"], file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) From 68f719045f719f61ff1084fea27e72633f77ce3e Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 23 Sep 2026 02:11:07 +1000 Subject: [PATCH 2/5] feat: complete durable append retries with fenced object cleanup Signed-off-by: wchwawa --- README.md | 92 +- bench/src/bin/nokv-restore-crash-owner.rs | 12 +- crates/nokv-client/src/artifact.rs | 26 +- crates/nokv-client/src/idempotent_append.rs | 1023 ++++-- crates/nokv-client/src/lib.rs | 5 +- crates/nokv-client/src/workbench_workflow.rs | 5 + crates/nokv-meta/src/workspace/codec.rs | 4 +- crates/nokv-meta/src/workspace/commit.rs | 11 +- crates/nokv-meta/src/workspace/engine.rs | 3 +- crates/nokv-meta/src/workspace/mod.rs | 10 +- crates/nokv-meta/src/workspace/publication.rs | 1420 ++++++- .../src/workspace/publication_records.rs | 9 +- .../workspace/publish_operation_records.rs | 179 +- crates/nokv-meta/src/workspace/recovery.rs | 70 +- crates/nokv-meta/src/workspace/restore.rs | 10 +- crates/nokv-object/src/admission.rs | 128 + crates/nokv-object/src/lib.rs | 11 +- crates/nokv-object/src/namespace.rs | 6 +- .../src/provider_admission_test.rs | 108 +- crates/nokv-object/src/store.rs | 123 +- crates/nokv-object/src/tests.rs | 114 + crates/nokv-object/src/tiered.rs | 18 +- crates/nokv-protocol/src/append.rs | 181 + crates/nokv-protocol/src/artifact_publish.rs | 2 +- crates/nokv-protocol/src/codec.rs | 70 +- crates/nokv-protocol/src/lib.rs | 5 + crates/nokv-protocol/src/request.rs | 24 + crates/nokv-protocol/src/response.rs | 40 + crates/nokv-protocol/src/types.rs | 5 +- crates/nokv-python/README.md | 63 +- crates/nokv-python/python/nokv/__init__.py | 9 +- crates/nokv-python/src/client.rs | 331 +- crates/nokv-python/src/object_store.rs | 130 +- crates/nokv-python/src/python_value.rs | 24 + crates/nokv-python/tests/test_api_surface.py | 26 + crates/nokv-server/src/bootstrap.rs | 230 +- crates/nokv-server/src/executor.rs | 252 +- crates/nokv-server/src/lib.rs | 10 +- crates/nokv-server/src/lifecycle.rs | 411 ++- crates/nokv-types/src/workspace.rs | 7 + crates/nokv/src/backend.rs | 100 +- crates/nokv/src/cli.rs | 112 +- crates/nokv/src/main.rs | 413 ++- crates/nokv/src/object_store.rs | 65 +- crates/nokv/src/transfer.rs | 51 +- docs/architecture.md | 11 +- docs/development/append-idempotency.md | 160 +- docs/development/append-product-spec.md | 344 ++ docs/metadata-schema.md | 66 +- .../append_product_acceptance_gate.py | 3261 +++++++++++++++++ 50 files changed, 8833 insertions(+), 957 deletions(-) create mode 100644 crates/nokv-protocol/src/append.rs create mode 100644 docs/development/append-product-spec.md create mode 100644 scripts/workbench/append_product_acceptance_gate.py diff --git a/README.md b/README.md index 74320ad71..45d172774 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ same conformance and fault evidence: | I need to... | Use | What the caller gets | | --- | --- | --- | -| Persist one run | `workbench_create`, `workbench_put_file`, `workbench_append`, `workbench_edit`, `workbench_commit` | Five-section workspace, explicit write modes, immutable revisions, and a sealed run manifest | +| Persist one run | `workbench_create`, `workbench_put_file`, `workspace-path append`, `workbench_edit`, `workbench_commit` | Five-section workspace, explicit write modes, immutable revisions, and a sealed run manifest | | Reopen work from another process | `workbench_list`, `workbench_stat`, `workbench_read`, `workbench_find` | Path-shaped discovery and verified reads without retaining the original sandbox | | Prevent stale or duplicate writes | generations, explicit request identity, exact replay | Typed conflict for stale state; the same outcome for an exact retry after response loss | | Search many artifacts or runs | `workbench_grep`, `workbench_search`, `workbench_aggregate`, `workbench_catalog`, `workbench_find` | Literal body search plus typed indexed metadata query and aggregation | @@ -238,7 +238,8 @@ real-service acceptance gate is still **not qualified**. | `nokv materialize` | Copy a verified workspace artifact to a new local path. | The destination is disposable scratch, not a namespace or mount. | | `nokv collect` | Publish one bounded regular local file, create-only or generation-fenced replace. | Symlinks and unbounded/non-regular inputs fail closed. | | `nokv workspace-path rename` / `nokv workspace-path remove` | Apply an explicit generation- and request-id-fenced path mutation. | Custom CLI surface; not one of the 18 Workbench tools. | -| `nokv workspace-path append` | Append bytes under a caller-owned operation identity and return the original publication on replay. | Requires an existing workspace; pin its incarnation across restarts. Custom CLI surface; the 18-tool schema is unchanged. | +| `nokv workspace-path append` | Append bytes under a durable logical identity and recover the original receipt across process restarts. | The first admission requires an existing workspace. Retries use fenced publication attempts under the same identity; the 18-tool schema is unchanged. | +| `nokv operation status` | Query an append by its logical identity and report its receipt or next recovery action. | Metadata only: no payload, current workspace, or object-store connection is required. | | `nokv provision` | Bind `RootId` to `AgentId`, object namespace, logical shard, and persisted placement through etcd. | `AgentId` prevents accidental root reuse; it is not authentication. | | `nokv serve` | Start one explicit metadata owner from create, same-namespace reopen, or recovery-log state. | Shared recovery publication is opt-in and not currently qualified. | | `nokv mcp` | Deprecated stdio transport retained only because qualification runners still use it. | **Unsupported for integration.** | @@ -253,25 +254,61 @@ nokv [route/agent/object options] workspace-path append run-42 logs events.jsonl --text 'step-7-done' --content-type text/plain ``` -Replace the example ids with the caller's stable operation id and the existing -workspace's incarnation. Omit the incarnation only to observe and fence the -existing workspace on this call; append never creates a missing workspace. -Use `--text`, `--base64`, or `--file` for exactly one delta. Text defaults to +Replace the example ids with the caller's stable logical operation id and the +intended workspace's incarnation. On the first call, omitting the incarnation +observes and fences the existing workspace. On recovery, omission resolves the +original durable operation first, even if the workspace was deleted or recreated. +Append never creates a missing workspace. + +Use exactly one of `--text`, `--base64`, or `--file`. Text defaults to `text/plain; charset=utf-8` on creation; binary and file input default to `application/octet-stream`. Existing content type is inherited unless -`--content-type` overrides it. `--max-artifact-bytes` bounds the delta; -`--max-logical-size` optionally bounds the resulting artifact. - -The receipt's operation/revision ids, generation, workspace revision, size, -digest, and incarnation describe that publication, even if another writer has -since advanced the path. `replayed` and optional `commit_version` describe this -SDK call and are not stable receipt fields; a recovered receipt may have a null -`commit_version`. An unresolved error retains the operation id and any observed -durable state. Recover the same identity; a timeout does not authorize applying -the delta under a new identity. One identity owns one publication attempt; -this command does not silently create a new attempt after a CAS conflict. -The older `workbench_append` tool retains per-invocation identities and does -not provide cross-process caller-identity replay. +`--content-type` overrides it. Explicitly use the same type when replaying across +text, file, and Python inputs. The delta limit is 16 MiB; `--max-artifact-bytes` +may lower it. The resulting body also defaults to a 16 MiB limit; +`--max-logical-size` raises that bound and is part of the stable intent. +`--block-size` defaults to 4 MiB and matches Python's `block_size` argument; +retain the same value across retries and surface changes. + +The receipt identifies both the logical `operation_id` and its successful +`publication_operation_id`. Its revision, generation, workspace revision, size, +digest, and incarnation describe the original publication, even if another +writer has advanced or removed the path. `replayed` and nullable `commit_version` +describe this SDK call and are not stable receipt fields. + +Query a result without resending the delta or contacting the object store: + +```shell +nokv [route/agent options] operation status aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +``` + +The response supplies a machine-readable recovery action: + +| `state` | `next_action` | Caller action | +| --- | --- | --- | +| `committed` | `none` | Consume the original `receipt`. | +| `pending` | `poll` | Query the same logical identity again after waiting. | +| `ready_to_retry` | `resubmit_same` | Resubmit the original append inputs under the same logical identity. | +| `quarantined` | `operator_reconcile` | Keep the identity and escalate the recorded inconsistency for operator reconciliation. | + +Status also retains the current physical attempt's `cause_code`, +`failure_message`, and structured `attempt_failure`, including when the logical +operation is `ready_to_retry`. + +An interrupted publication must be fenced and cleaned before a successor +attempt can start. Status queries only observe this lifecycle. Persist the delta +or a way to reproduce the exact bytes until the operation commits. An unknown +result or `NotFound` observation is not proof that a concurrent request cannot +commit; errors preserve the logical id and use `next_action=query_same`. +`cause_code=RequestReplayMismatch` means the identity belongs to different +inputs or a different lifecycle: recover the original intent before retrying. +Never generate a replacement id merely to bypass an uncertain outcome. + +The older `workbench_append` tool retains per-invocation identities and cannot +recover one action across separate processes. Use `workspace-path append` for +new retrying harness integrations. The fixed 18-tool contract remains unchanged. +The [append product specification](docs/development/append-product-spec.md) +defines the supported recovery, size, retention, and release boundaries. ### Programmatic and non-CLI capabilities @@ -280,14 +317,15 @@ do not expose identical method shapes. | Surface | Current public capability | Additional boundary | | --- | --- | --- | -| Direct Python `Client` | 23 methods covering create/stat/exists/list/remove/rename, byte/file publish, whole/range/batch reads, query/aggregate/catalog/find, commit/restore, snapshot lifecycle, materialize, and collect | Direct SDK, not a tool-for-tool copy of the 18-name CLI facade | +| Direct Python `Client` | Create/stat/exists/list/remove/rename, byte/file publish, stable append and its operation status, whole/range/batch reads, query/aggregate/catalog/find, commit/restore, snapshot lifecycle, materialize, and collect | Direct SDK, not a tool-for-tool copy of the 18-name CLI facade | | Python adapters | Workbench-scoped fsspec, checkpoint helpers, and optional torch Distributed Checkpoint reader/writer | Bounded to an explicit Workbench; not arbitrary-root POSIX or FUSE | | Rust `WorkspaceClient` | Lower-level typed workspace, publication, query, lifecycle, routing, and batch-range workflows | Recommended when the caller must own typed retry and recovery integration | -| Rust-only extensions | Polling change feed, generic custom-index registration, raw operation status, and phased publish/restore primitives | No native CLI or Python method today; change feed is polling, not push | +| Rust-only extensions | Polling change feed, generic custom-index registration, raw non-append operation status, and phased publish/restore primitives | No native CLI or Python method today; change feed is polling, not push | | Metadata and server lifecycle | snapshot reap, commit/tag holds, reference-fenced GC, quarantine reconciliation, Holt reopen, and optional shared recovery records | Internal/operator mechanisms, not independent end-user commands | -The Python SDK does not currently expose CLI-equivalent append, exact-string -edit, or body grep methods. Use the CLI contract for those exact behaviors. +Python `append_bytes` and `operation_status` share the native stable append +contract. The SDK does not expose the legacy 18-tool append facade, exact-string +edit, or body grep methods; use the CLI for those exact behaviors. Custom SDK compositions are caller-owned and are not equivalent qualification evidence. @@ -295,7 +333,7 @@ evidence. ```text Downstream skill or shell harness - -> native nokv workbench CLI exact 18-operation contract + -> native nokv CLI workspace operations and stable append recovery Embedded Python application -> nokv Python Client/adapters direct path/artifact/lifecycle API @@ -325,8 +363,10 @@ option and are rejected by the agent-facing CLI. authoritative. - Every published body has a never-reused artifact revision id, immutable revision-owned blocks, a whole-body digest, and a caller-visible generation. -- Append is not a server-side lock. It reads the current head, attempts a - generation-fenced immutable publication, and retries bounded conflicts. +- Append reads the current head and publishes an immutable generation-fenced + revision. Stable append records a logical operation before publication; a + successor attempt is allowed only after its predecessor cannot publish and + has completed cleanup. One logical append has at most one committed result. ### Commit, snapshot, and restore diff --git a/bench/src/bin/nokv-restore-crash-owner.rs b/bench/src/bin/nokv-restore-crash-owner.rs index fd20701e0..098cc647f 100644 --- a/bench/src/bin/nokv-restore-crash-owner.rs +++ b/bench/src/bin/nokv-restore-crash-owner.rs @@ -24,8 +24,8 @@ use nokv_control::{ use nokv_object::{ admit_artifact_provider, ArtifactObjectStore, ArtifactStoreCapabilities, BoundArtifactStore, ImmutableCreateOutcome, ObjectDeleteOutcome, ObjectError, ObjectInfo, ObjectKey, ObjectRange, - ProviderAdmissionProfile, ProviderAdmissionReceipt, ProviderHandleIdentity, S3ArtifactStore, - S3ArtifactStoreOptions, DEFAULT_RECOVERY_LOG_SEGMENT_CHUNK_SIZE, + ObjectSealOutcome, ProviderAdmissionProfile, ProviderAdmissionReceipt, ProviderHandleIdentity, + S3ArtifactStore, S3ArtifactStoreOptions, DEFAULT_RECOVERY_LOG_SEGMENT_CHUNK_SIZE, }; use nokv_protocol::{ GetOperationRequest, GetWorkspaceRequest, OperationIdentity, RootIdentity, WorkbenchName, @@ -722,6 +722,10 @@ impl ArtifactObjectStore for AdmittedRecoveryObjectStore { fn delete(&self, key: &ObjectKey) -> Result { self.inner.delete(key) } + + fn seal_immutable(&self, key: &ObjectKey) -> Result { + self.inner.seal_immutable(key) + } } fn recovery_object_store( @@ -741,7 +745,8 @@ fn recovery_object_store( }) .map_err(|error| error.to_string())?; let profile = ProviderAdmissionProfile::single_put(DEFAULT_RECOVERY_LOG_SEGMENT_CHUNK_SIZE) - .map_err(|error| error.to_string())?; + .map_err(|error| error.to_string())? + .with_append_sealing(); let admission = admit_artifact_provider(&raw, profile).map_err(|error| error.to_string())?; let inner = BoundArtifactStore::open(raw, namespace).map_err(|error| error.to_string())?; Ok(Arc::new(AdmittedRecoveryObjectStore { inner, admission })) @@ -797,6 +802,7 @@ fn run_server(config: ServeConfig) -> Result<(), String> { Arc::clone(®istry), recovery_objects, ShardBoot { + append_activity_lease_ms: 1_800_000, shard_id: placement.logical_shard_id, open: OpenMode::Existing(config.metadata_reopen), lease: LeaseMode::Acquire { diff --git a/crates/nokv-client/src/artifact.rs b/crates/nokv-client/src/artifact.rs index 3c884b9bf..7c31140d8 100644 --- a/crates/nokv-client/src/artifact.rs +++ b/crates/nokv-client/src/artifact.rs @@ -13,8 +13,8 @@ use nokv_object::{ }; use nokv_protocol::{ parse_sha256_digest_uri, seal_artifact_publish_plan, sha256_digest_uri, - AbortArtifactPublishRequest, AppendSegment, ArtifactDescriptor, ArtifactManifestRow, - ArtifactRevisionIdentity, BeginArtifactPublishRequest, ByteRange, + AbortArtifactPublishRequest, AppendAttemptBinding, AppendSegment, ArtifactDescriptor, + ArtifactManifestRow, ArtifactRevisionIdentity, BeginArtifactPublishRequest, ByteRange, CompleteArtifactPublishRequest, ContentType, Digest, ErrorCode, FieldValue, GetOperationRequest, GetPathRequest, LogicalShardIdentity, MarkArtifactObjectsUploadedRequest, ObjectIdentity, ObjectUploadProof, OperationIdentity, OperationKind, OperationResult, @@ -310,6 +310,7 @@ where options.condition, options.expected_workspace_incarnation_id, None, + None, object_plan, staged_objects, manifest_rows, @@ -331,6 +332,7 @@ where condition: PublishCondition, expected_workspace_incarnation_id: Option, append_intent_digest: Option, + append_attempt: Option, object_plan: ArtifactUploadPlan, staged_objects: Vec, manifest_rows: Vec, @@ -351,6 +353,7 @@ where &condition, expected_workspace_incarnation_id, append_intent_digest, + append_attempt, &object_plan, &staged_objects, &manifest_rows, @@ -389,6 +392,7 @@ where condition: &PublishCondition, expected_workspace_incarnation_id: Option, append_intent_digest: Option, + append_attempt: Option, object_plan: &ArtifactUploadPlan, staged_objects: &[StagedObject], manifest_rows: &[ArtifactManifestRow], @@ -430,6 +434,7 @@ where condition: *condition, expected_workspace_incarnation_id, append_intent_digest, + append_attempt, staged_object_count: seals.staged_object_count, staged_object_seal: seals.staged_object_seal, manifest_row_count: seals.manifest_row_count, @@ -704,6 +709,7 @@ where delta, None, None, + None, ) { Err(error) if is_append_retry_error(&error) && attempt + 1 < self.max_attempts() => @@ -732,8 +738,16 @@ where delta: &[u8], expected_workspace_incarnation_id: Option, append_intent_digest: Option, + append_attempt: Option, ) -> Result { require_provider_admission(store, options.block_size)?; + if append_attempt.is_some() + && !store + .provider_admission_receipt() + .is_some_and(|receipt| receipt.admits_append_store(store, options.block_size)) + { + return Err(ObjectError::ProviderAdmissionRequired.into()); + } let route = self.resolve_artifact_route()?; require_object_namespace(store, route)?; let logical_shard = route.logical_shard_id; @@ -969,6 +983,7 @@ where condition, expected_workspace_incarnation_id, append_intent_digest, + append_attempt, object_plan, staged_objects, manifest_rows, @@ -2925,6 +2940,7 @@ mod tests { let begin = self.begin.as_ref().expect("publication has begun"); Some(Box::new(nokv_protocol::PublishPreparation { append_intent_digest: begin.append_intent_digest, + append_attempt: begin.append_attempt, target: begin.target.clone(), workspace_incarnation_id: WorkspaceIdentity([9; 16]), artifact_revision_id: begin.artifact_revision_id, @@ -2935,6 +2951,7 @@ mod tests { OperationStatus { token: self.next_token(operation_id), kind: OperationKind::ArtifactPublish, + append_preparation: None, publish_preparation: self.publish_preparation(), commit_preparation: None, restore_preparation: None, @@ -3036,6 +3053,7 @@ mod tests { let status = OperationStatus { token: state.next_token(stage.token.operation_id), kind: OperationKind::ArtifactPublish, + append_preparation: None, publish_preparation: state.publish_preparation(), commit_preparation: None, restore_preparation: None, @@ -3221,6 +3239,7 @@ mod tests { let status = OperationStatus { token: state.next_token(begin.operation_id), kind: OperationKind::ArtifactPublish, + append_preparation: None, publish_preparation: state.publish_preparation(), commit_preparation: None, restore_preparation: None, @@ -3259,6 +3278,7 @@ mod tests { let status = OperationStatus { token: state.next_token(abort.token.operation_id), kind: OperationKind::ArtifactPublish, + append_preparation: None, publish_preparation: state.publish_preparation(), commit_preparation: None, restore_preparation: None, @@ -4200,8 +4220,10 @@ mod tests { state_digest: Digest([0x55; 32]), }, kind: OperationKind::ArtifactPublish, + append_preparation: None, publish_preparation: Some(Box::new(nokv_protocol::PublishPreparation { append_intent_digest: None, + append_attempt: None, target: target(), workspace_incarnation_id: WorkspaceIdentity([9; 16]), artifact_revision_id: ArtifactRevisionIdentity([8; 16]), diff --git a/crates/nokv-client/src/idempotent_append.rs b/crates/nokv-client/src/idempotent_append.rs index 922db6dc9..1f1bcf044 100644 --- a/crates/nokv-client/src/idempotent_append.rs +++ b/crates/nokv-client/src/idempotent_append.rs @@ -1,16 +1,16 @@ // Copyright 2024-2026 The NoKV Authors. // SPDX-License-Identifier: Apache-2.0 -//! Cross-process append identity over the existing publication lifecycle. -//! -//! One operation owns one publication attempt. An admitted but unfinished -//! attempt is never replaced or aborted merely because another caller retries. +//! Caller-owned logical appends with durable, predecessor-fenced publication attempts. + +use std::{thread, time::Duration}; use nokv_object::{ArtifactObjectStore, DEFAULT_ARTIFACT_BLOCK_SIZE}; use nokv_protocol::{ - ArtifactRevisionIdentity, ConflictKind, ContentType, Digest, ErrorCode, GetOperationRequest, + stable_append_attempt_identities, AppendAttemptBinding, AppendAttemptPhase, AppendResult, + ConflictKind, ContentType, Digest, ErrorCode, GetOperationRequest, GetWorkspaceRequest, OperationIdentity, OperationKind, OperationResult, OperationState, OperationStatus, - PublishResult, RootIdentity, RpcFailure, WorkspaceIdentity, WorkspacePath, + RootIdentity, RpcFailure, WorkspaceIdentity, WorkspacePath, }; use sha2::{Digest as _, Sha256}; @@ -18,10 +18,19 @@ use crate::{ ArtifactAppendOptions, ClientCall, ClientError, RouteResolver, RpcTransport, WorkspaceClient, }; -/// An append intent whose identity the caller persists before the first send. +/// Maximum delta accepted by the bounded append product. +pub const MAX_APPEND_DELTA_BYTES: usize = 16 * 1024 * 1024; +/// Default bound for the complete resulting artifact, including rematerialization. +pub const DEFAULT_APPEND_MAX_LOGICAL_SIZE: u64 = 16 * 1024 * 1024; + +const APPEND_CLEANUP_POLLS: usize = 12; +const APPEND_CLEANUP_POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// An append intent whose identity and payload the caller persists before sending. /// -/// Content-type inheritance and explicit overrides are distinct intents. The -/// workspace incarnation is an owner-checked fence, not an authentication token. +/// A logical operation may use another immutable publication attempt only after +/// the previous one is durably cleaned. Concurrent or uncertain attempts cannot +/// be replaced. Scheduling budgets do not change the caller's logical identity. #[derive(Clone, Debug, PartialEq, Eq)] pub struct IdempotentAppendOptions { pub operation_id: OperationIdentity, @@ -47,7 +56,7 @@ impl IdempotentAppendOptions { create_content_type, content_type: None, block_size: DEFAULT_ARTIFACT_BLOCK_SIZE, - max_logical_size: None, + max_logical_size: Some(DEFAULT_APPEND_MAX_LOGICAL_SIZE), } } @@ -67,95 +76,422 @@ impl IdempotentAppendOptions { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AppendRecoveryState { + Committed, + Pending, + ReadyToRetry, + Quarantined, +} + +impl AppendRecoveryState { + pub const fn as_str(self) -> &'static str { + match self { + Self::Committed => "committed", + Self::Pending => "pending", + Self::ReadyToRetry => "ready_to_retry", + Self::Quarantined => "quarantined", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AppendNextAction { + None, + Poll, + ResubmitSame, + OperatorReconcile, +} + +impl AppendNextAction { + pub const fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Poll => "poll", + Self::ResubmitSame => "resubmit_same", + Self::OperatorReconcile => "operator_reconcile", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AppendRecovery { + pub state: AppendRecoveryState, + pub next_action: AppendNextAction, +} + +/// Classify a durable append observation without performing a mutation. +/// +/// A failed physical attempt is not a failed logical append. Only Cleaned +/// authorizes resubmission under the same logical identity. Unknown transport +/// outcomes are errors, never synthesized observations of absence. +pub fn append_operation_recovery(status: &OperationStatus) -> Result { + if status.kind != OperationKind::ArtifactAppend { + return Err(append_failure( + ErrorCode::RequestReplayMismatch, + "operation identity belongs to another lifecycle", + )); + } + let preparation = status.append_preparation.as_ref().ok_or_else(|| { + ClientError::ResponseMismatch("logical append has no durable preparation".to_owned()) + })?; + if status.state == OperationState::Succeeded { + if preparation.attempt_phase != AppendAttemptPhase::Published + || !matches!(status.result, Some(OperationResult::ArtifactAppend(_))) + { + return Err(ClientError::ResponseMismatch( + "committed append has no published attempt and receipt".to_owned(), + )); + } + return Ok(AppendRecovery { + state: AppendRecoveryState::Committed, + next_action: AppendNextAction::None, + }); + } + if status.result.is_some() { + return Err(ClientError::ResponseMismatch( + "unfinished append cannot contain a committed receipt".to_owned(), + )); + } + Ok(match preparation.attempt_phase { + AppendAttemptPhase::Cleaned => AppendRecovery { + state: AppendRecoveryState::ReadyToRetry, + next_action: AppendNextAction::ResubmitSame, + }, + AppendAttemptPhase::Quarantined => AppendRecovery { + state: AppendRecoveryState::Quarantined, + next_action: AppendNextAction::OperatorReconcile, + }, + AppendAttemptPhase::Uploading + | AppendAttemptPhase::Finalizing + | AppendAttemptPhase::Aborting + | AppendAttemptPhase::Cleaning => AppendRecovery { + state: AppendRecoveryState::Pending, + next_action: AppendNextAction::Poll, + }, + AppendAttemptPhase::Published => { + return Err(ClientError::ResponseMismatch( + "published attempt has no atomic logical append receipt".to_owned(), + )); + } + }) +} + impl WorkspaceClient where Transport: RpcTransport, Resolver: RouteResolver, { - /// Append at most once for one caller-owned identity and exact intent. + /// Query one logical append without its delta, current workspace or object provider. + pub fn get_append_operation( + &self, + operation_id: OperationIdentity, + ) -> Result, ClientError> { + let status = self.get_operation(GetOperationRequest { operation_id })?; + if status.value.token.operation_id != operation_id { + return Err(ClientError::ResponseMismatch( + "append status returned a different logical identity".to_owned(), + )); + } + append_operation_recovery(&status.value)?; + Ok(status) + } + + /// Resolve the original workspace before considering its current name binding. /// - /// A successful replay returns the original publication receipt even when - /// another writer has since replaced or removed the path. It does not - /// assert that the original revision is still the live head or retained. + /// Explicit fences always take precedence. Only an operation with no + /// observed durable admission obtains its incarnation from the live name. + pub fn resolve_append_workspace_incarnation( + &self, + operation_id: OperationIdentity, + target: &WorkspacePath, + expected: Option, + ) -> Result { + let incarnation = match self.get_append_operation(operation_id) { + Ok(status) => { + let preparation = status + .value + .append_preparation + .expect("get_append_operation validates its preparation"); + if preparation.target != *target { + return Err(append_unresolved( + operation_id, + Some(status.value.state), + append_failure( + ErrorCode::RequestReplayMismatch, + "append identity is already bound to another target", + ), + )); + } + preparation.workspace_incarnation_id + } + Err(error) if error.rpc_code() == Some(ErrorCode::NotFound) => { + self.get_workspace(GetWorkspaceRequest { + workbench: target.workbench.clone(), + })? + .value + .workspace_incarnation_id + } + Err(error) => return Err(append_unresolved(operation_id, None, error)), + }; + if expected.is_some_and(|value| value != incarnation) { + return Err(append_unresolved( + operation_id, + None, + ClientError::Rpc(RpcFailure { + code: ErrorCode::Conflict, + message: "append workspace incarnation differs from the explicit fence" + .to_owned(), + retryable: false, + conflict: Some(ConflictKind::WorkspaceIncarnation), + current_generation: None, + route_hint: None, + }), + )); + } + Ok(incarnation) + } + + /// Recover a receipt before constructing an object provider. /// - /// A running, failed, quarantined or uncertain operation is returned as - /// `AppendUnresolved`. Retrying preserves the identity; it never allocates - /// another append attempt or interprets a timeout as proof of absence. + /// Normalizes and authenticates the caller intent using metadata only. + /// `None` means the same intent may be submitted: it is either absent or + /// its predecessor is durably cleaned. That observation is not admission; + /// `append_artifact_idempotent` still rechecks and atomically fences it. + /// Pending and quarantined actions return a queryable logical error. + pub fn recover_append_receipt( + &self, + options: &mut IdempotentAppendOptions, + delta: &[u8], + ) -> Result>, ClientError> { + validate_append_inputs(options, delta)?; + let intent = append_intent_digest(self.root_id(), options, delta)?; + let Some(status) = self.observe_append(options)? else { + return Ok(None); + }; + validate_append_intent(self.root_id(), &status.value, options, intent)?; + match append_operation_recovery(&status.value)?.state { + AppendRecoveryState::Committed => append_receipt(status, options).map(Some), + AppendRecoveryState::ReadyToRetry => Ok(None), + AppendRecoveryState::Pending | AppendRecoveryState::Quarantined => { + Err(pending_append(options, &status.value)) + } + } + } + + /// Apply the caller's logical append at most once, recovering safe attempts. + /// + /// Committed results replay without accessing Live or the object provider. + /// Resumption requires the same delta and normalized options. The server + /// admits a successor and advances its parent atomically only after cleanup + /// proves that the previous attempt cannot publish. Bounded local retries + /// return a queryable pending operation instead of allocating a new action. pub fn append_artifact_idempotent( &self, store: &dyn ArtifactObjectStore, - options: IdempotentAppendOptions, + mut options: IdempotentAppendOptions, delta: &[u8], - ) -> Result, ClientError> { + ) -> Result, ClientError> { + validate_append_inputs(&mut options, delta)?; let intent = append_intent_digest(self.root_id(), &options, delta)?; - let revision = append_revision_identity(self.root_id(), &options); - match self.get_operation(GetOperationRequest { - operation_id: options.operation_id, - }) { - Ok(status) => return append_receipt(status, &options, intent, revision), - Err(error) if error.rpc_code() == Some(ErrorCode::NotFound) => {} - Err(error) => return Err(unresolved(&options, None, error)), - } + let mut observed = self.observe_append(&options)?; + let mut submissions = 0; - // No durable attempt was observed. Racing callers still use the exact - // same operation/revision: server-side admission decides the winner. - // Unlike legacy append, there is no sequence of potentially empty - // attempt IDs that independent callers could fill out of order. - let mut publication = ArtifactAppendOptions::new( - options.operation_id, - revision, - options.target.clone(), - options.create_content_type.clone(), - ) - .with_block_size(options.block_size); - publication.content_type = options.content_type.clone(); - publication.max_logical_size = options.max_logical_size; - match self.append_artifact_attempt( - store, - &publication, - options.operation_id, - revision, - delta, - Some(options.expected_workspace_incarnation_id), - Some(intent), - ) { - Ok(outcome) => Ok(outcome.publication), - Err(source) => { - // In particular, a Begin race may have lost to the same - // intent, and Complete may have committed before losing its - // response. Query before interpreting either as failure. - match self.get_operation(GetOperationRequest { + loop { + let attempt = match observed.as_ref() { + None => 0, + Some(status) => { + validate_append_intent(self.root_id(), &status.value, &options, intent)?; + match append_operation_recovery(&status.value)?.state { + AppendRecoveryState::Committed => { + return append_receipt( + observed.take().expect("observed committed append"), + &options, + ); + } + AppendRecoveryState::ReadyToRetry => status + .value + .append_preparation + .as_ref() + .expect("validated preparation") + .attempt + .checked_add(1) + .ok_or_else(|| { + unresolved( + &options, + Some(status.value.state), + ClientError::InvalidOptions( + "append attempt counter exhausted".to_owned(), + ), + ) + })?, + _ => return Err(pending_append(&options, &status.value)), + } + } + }; + if submissions >= self.max_attempts() { + return Err(unresolved( + &options, + observed.as_ref().map(|call| call.value.state), + append_failure(ErrorCode::Conflict, + "append retry budget exhausted; query or resubmit the same logical identity"), + )); + } + submissions += 1; + let (publication_id, revision_id) = + stable_append_attempt_identities(self.root_id(), options.operation_id, attempt); + let mut publication = ArtifactAppendOptions::new( + publication_id, + revision_id, + options.target.clone(), + options.create_content_type.clone(), + ) + .with_block_size(options.block_size); + publication.content_type = options.content_type.clone(); + publication.max_logical_size = options.max_logical_size; + let outcome = self.append_artifact_attempt( + store, + &publication, + publication_id, + revision_id, + delta, + Some(options.expected_workspace_incarnation_id), + Some(intent), + Some(AppendAttemptBinding { operation_id: options.operation_id, - }) { - Ok(status) => append_receipt(status, &options, intent, revision), - Err(error) if error.rpc_code() == Some(ErrorCode::NotFound) => { - Err(unresolved(&options, None, source)) + attempt, + }), + ); + match outcome { + Ok(outcome) => { + let call = outcome.publication; + let receipt = call.value; + if receipt.operation_id != publication_id + || receipt.artifact_revision_id != revision_id + || receipt.target != options.target + { + return Err(unresolved( + &options, + None, + ClientError::ResponseMismatch( + "append publication differs from admitted attempt".to_owned(), + ), + )); + } + return Ok(ClientCall { + value: AppendResult { + operation_id: options.operation_id, + publication_operation_id: publication_id, + target: receipt.target, + workspace_incarnation_id: options.expected_workspace_incarnation_id, + workspace_revision: receipt.workspace_revision, + generation: receipt.generation, + artifact_revision_id: receipt.artifact_revision_id, + logical_size: receipt.logical_size, + body_digest: receipt.body_digest, + }, + commit_version: call.commit_version, + replayed: call.replayed, + }); + } + Err(source) => { + // Resolve response-loss ambiguity before classifying the + // failed call. Cleanup polling is bounded independently of + // the number of new durable attempts this call may admit. + observed = self.observe_append(&options)?; + if observed.is_none() { + if source.rpc_code().is_some_and(|code| { + matches!(code, ErrorCode::Conflict | ErrorCode::PreconditionFailed) + }) && submissions < self.max_attempts() + { + continue; + } + return Err(unresolved(&options, None, source)); + } + for _ in 0..APPEND_CLEANUP_POLLS { + let status = &observed.as_ref().expect("observed append").value; + validate_append_intent(self.root_id(), status, &options, intent)?; + if append_operation_recovery(status)?.state != AppendRecoveryState::Pending + { + break; + } + thread::sleep(APPEND_CLEANUP_POLL_INTERVAL); + observed = self.observe_append(&options)?; + if observed.is_none() { + return Err(unresolved( + &options, + None, + ClientError::ResponseMismatch( + "durable append identity disappeared".to_owned(), + ), + )); + } } - Err(error) => Err(unresolved(&options, None, error)), } } } } + + fn observe_append( + &self, + options: &IdempotentAppendOptions, + ) -> Result>, ClientError> { + match self.get_append_operation(options.operation_id) { + Ok(status) => Ok(Some(status)), + Err(error) if error.rpc_code() == Some(ErrorCode::NotFound) => Ok(None), + Err(error) => Err(unresolved(options, None, error)), + } + } } -fn append_receipt( - status: ClientCall, +fn validate_append_inputs( + options: &mut IdempotentAppendOptions, + delta: &[u8], +) -> Result<(), ClientError> { + if delta.len() > MAX_APPEND_DELTA_BYTES { + return Err(ClientError::InvalidOptions(format!( + "append delta exceeds the {MAX_APPEND_DELTA_BYTES}-byte product limit" + ))); + } + if options.block_size == 0 { + return Err(ClientError::InvalidOptions( + "append block size must be positive".to_owned(), + )); + } + let maximum = options + .max_logical_size + .get_or_insert(DEFAULT_APPEND_MAX_LOGICAL_SIZE); + if delta.len() as u64 > *maximum { + return Err(ClientError::InvalidOptions( + "append delta exceeds the resulting artifact size limit".to_owned(), + )); + } + Ok(()) +} + +fn validate_append_intent( + root: RootIdentity, + status: &OperationStatus, options: &IdempotentAppendOptions, intent: Digest, - revision: ArtifactRevisionIdentity, -) -> Result, ClientError> { - let value = status.value; - let matches = value.token.operation_id == options.operation_id - && value.kind == OperationKind::ArtifactPublish - && value - .publish_preparation +) -> Result<(), ClientError> { + let matches = status.token.operation_id == options.operation_id + && status.kind == OperationKind::ArtifactAppend + && status + .append_preparation .as_ref() .is_some_and(|preparation| { - preparation.append_intent_digest == Some(intent) + let (publication_id, revision_id) = stable_append_attempt_identities( + root, + options.operation_id, + preparation.attempt, + ); + preparation.intent_digest == intent && preparation.target == options.target && preparation.workspace_incarnation_id == options.expected_workspace_incarnation_id - && preparation.artifact_revision_id == revision + && preparation.publication_operation_id == publication_id + && preparation.artifact_revision_id == revision_id }); if !matches { return Err(unresolved( @@ -163,51 +499,39 @@ fn append_receipt( None, append_failure( ErrorCode::RequestReplayMismatch, - "append operation identity is already bound to a different intent or lifecycle", - ), - )); - } - if value.state != OperationState::Succeeded { - let message = value.failure.map_or_else( - || { - format!( - "append publication is {:?}; query or retry this same identity", - value.state - ) - }, - |failure| failure.message, - ); - return Err(unresolved( - options, - Some(value.state), - append_failure( - if value.state == OperationState::Running { - ErrorCode::Conflict - } else { - ErrorCode::OperationFailed - }, - message, + "append logical identity is already bound to a different intent or attempt", ), )); } - let Some(OperationResult::ArtifactPublish(result)) = value.result else { + Ok(()) +} + +fn append_receipt( + status: ClientCall, + options: &IdempotentAppendOptions, +) -> Result, ClientError> { + let Some(OperationResult::ArtifactAppend(result)) = status.value.result else { return Err(unresolved( options, - Some(value.state), - ClientError::ResponseMismatch( - "successful append has no publication receipt".to_owned(), - ), + Some(status.value.state), + ClientError::ResponseMismatch("committed append has no logical receipt".to_owned()), )); }; + let preparation = status + .value + .append_preparation + .expect("committed status has validated preparation"); if result.operation_id != options.operation_id - || result.artifact_revision_id != revision + || result.publication_operation_id != preparation.publication_operation_id + || result.artifact_revision_id != preparation.artifact_revision_id + || result.workspace_incarnation_id != options.expected_workspace_incarnation_id || result.target != options.target { return Err(unresolved( options, - Some(value.state), + Some(status.value.state), ClientError::ResponseMismatch( - "append receipt differs from its bound intent".to_owned(), + "append receipt differs from its durable preparation".to_owned(), ), )); } @@ -218,13 +542,45 @@ fn append_receipt( }) } +fn pending_append(options: &IdempotentAppendOptions, status: &OperationStatus) -> ClientError { + let recovery = append_operation_recovery(status); + let message = match recovery { + Ok(recovery) => format!( + "append is {}; next action: {}; retain the same logical identity", + recovery.state.as_str(), + recovery.next_action.as_str() + ), + Err(error) => return unresolved(options, Some(status.state), error), + }; + unresolved( + options, + Some(status.state), + append_failure( + if status.state == OperationState::Quarantined { + ErrorCode::OperationFailed + } else { + ErrorCode::Conflict + }, + message, + ), + ) +} + fn unresolved( options: &IdempotentAppendOptions, state: Option, source: ClientError, +) -> ClientError { + append_unresolved(options.operation_id, state, source) +} + +fn append_unresolved( + operation_id: OperationIdentity, + state: Option, + source: ClientError, ) -> ClientError { ClientError::AppendUnresolved { - operation_id: options.operation_id, + operation_id, state, source: Box::new(source), } @@ -251,7 +607,7 @@ fn append_intent_digest( let delta_len = u64::try_from(delta.len()) .map_err(|_| ClientError::InvalidOptions("append delta length exceeds u64".to_owned()))?; let mut hasher = Sha256::new(); - hasher.update(b"nokv.append.intent.v1\0"); + hasher.update(b"nokv.append.intent.v2\0"); hasher.update(root.0); hasher.update(options.operation_id.0); hasher.update(options.expected_workspace_incarnation_id.0); @@ -284,25 +640,20 @@ fn append_intent_digest( Ok(Digest(hasher.finalize().into())) } -fn append_revision_identity( - root: RootIdentity, - options: &IdempotentAppendOptions, -) -> ArtifactRevisionIdentity { - let mut hasher = Sha256::new(); - hasher.update(b"nokv.append.stable-revision.v1\0"); - hasher.update(root.0); - hasher.update(options.operation_id.0); - let digest = hasher.finalize(); - let mut revision = [0; 16]; - revision.copy_from_slice(&digest[..16]); - ArtifactRevisionIdentity(revision) -} - #[cfg(test)] mod tests { use super::*; + use crate::{ClientOptions, StaticRouteResolver, TransportError}; + use nokv_object::MemoryArtifactStore; use nokv_protocol::{ - OperationProgress, OperationToken, PublishPreparation, RelativePath, WorkbenchName, + decode_request, encode_response, AppendPreparation, LogicalShardIdentity, + ObjectNamespaceIdentity, OperationProgress, OperationToken, RelativePath, RootRoute, + WorkbenchName, WorkspaceRequest, WorkspaceResult, WorkspaceRpcOutcome, + WorkspaceRpcResponse, + }; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, }; fn options() -> IdempotentAppendOptions { @@ -315,168 +666,358 @@ mod tests { WorkspaceIdentity([2; 16]), ContentType::new("text/plain").unwrap(), ) + .with_block_size(1024) } - fn successful_status( - options: &IdempotentAppendOptions, - delta: &[u8], - ) -> ClientCall { - let root = RootIdentity([3; 16]); - let revision = append_revision_identity(root, options); - ClientCall { - commit_version: None, - replayed: false, - value: OperationStatus { - token: OperationToken { - operation_id: options.operation_id, - state_digest: Digest([9; 32]), - }, - kind: OperationKind::ArtifactPublish, - publish_preparation: Some(Box::new(PublishPreparation { - append_intent_digest: Some(append_intent_digest(root, options, delta).unwrap()), - target: options.target.clone(), - workspace_incarnation_id: options.expected_workspace_incarnation_id, - artifact_revision_id: revision, - })), - commit_preparation: None, - restore_preparation: None, - state: OperationState::Succeeded, - progress: OperationProgress { - completed_rows: 1, - total_rows: Some(1), - completed_bytes: delta.len() as u64, - total_bytes: Some(delta.len() as u64), - }, - result: Some(OperationResult::ArtifactPublish(PublishResult { - operation_id: options.operation_id, - target: options.target.clone(), - workspace_revision: 7, - generation: 4, - artifact_revision_id: revision, - logical_size: 100, - body_digest: nokv_protocol::sha256_digest_uri(Digest([4; 32])), - })), - failure: None, + fn committed_status(options: &IdempotentAppendOptions) -> OperationStatus { + let (publication_operation_id, artifact_revision_id) = + stable_append_attempt_identities(RootIdentity([3; 16]), options.operation_id, 4); + OperationStatus { + token: OperationToken { + operation_id: options.operation_id, + state_digest: Digest([9; 32]), + }, + kind: OperationKind::ArtifactAppend, + append_preparation: Some(Box::new(AppendPreparation { + intent_digest: append_intent_digest(RootIdentity([3; 16]), options, b"event\n") + .unwrap(), + target: options.target.clone(), + workspace_incarnation_id: options.expected_workspace_incarnation_id, + attempt: 4, + publication_operation_id, + artifact_revision_id, + attempt_phase: AppendAttemptPhase::Published, + attempt_failure: None, + activity_deadline_ms: 1234, + })), + publish_preparation: None, + commit_preparation: None, + restore_preparation: None, + state: OperationState::Succeeded, + progress: OperationProgress { + completed_rows: 1, + total_rows: Some(1), + completed_bytes: 100, + total_bytes: Some(100), }, + result: Some(OperationResult::ArtifactAppend(AppendResult { + operation_id: options.operation_id, + publication_operation_id, + target: options.target.clone(), + workspace_incarnation_id: options.expected_workspace_incarnation_id, + workspace_revision: 7, + generation: 4, + artifact_revision_id, + logical_size: 100, + body_digest: nokv_protocol::sha256_digest_uri(Digest([4; 32])), + })), + failure: None, + } + } + + struct StatusOnlyTransport { + status: OperationStatus, + calls: Arc, + } + + impl RpcTransport for StatusOnlyTransport { + fn round_trip( + &self, + _endpoint: std::net::SocketAddr, + bytes: &[u8], + ) -> Result, TransportError> { + let request = decode_request(bytes).unwrap(); + // A live-name lookup, metadata read or publication would be an + // incorrect side effect for a historical receipt or pending query. + assert!(matches!( + request.operation, + WorkspaceRequest::GetOperation(_) + )); + self.calls.fetch_add(1, Ordering::SeqCst); + encode_response(&WorkspaceRpcResponse { + route: request.route, + request_id: request.request_id, + commit_version: None, + replayed: false, + outcome: WorkspaceRpcOutcome::Success(Box::new(WorkspaceResult::Operation( + self.status.clone(), + ))), + }) + .map_err(|error| TransportError::new(error.to_string(), false)) } } + fn client( + status: OperationStatus, + ) -> ( + WorkspaceClient, + Arc, + ) { + let root_id = RootIdentity([3; 16]); + let calls = Arc::new(AtomicUsize::new(0)); + let route = RootRoute { + root_id, + logical_shard_id: LogicalShardIdentity([4; 16]), + object_namespace_id: ObjectNamespaceIdentity([5; 16]), + placement_generation: 1, + owner_epoch: 1, + }; + ( + WorkspaceClient::new( + root_id, + StatusOnlyTransport { + status, + calls: calls.clone(), + }, + StaticRouteResolver::new(route, ([127, 0, 0, 1], 4100).into()).unwrap(), + ClientOptions::default(), + ) + .unwrap(), + calls, + ) + } + #[test] - fn persisted_append_identity_has_frozen_v1_encoding() { - // Existing durable intents must remain recoverable after SDK changes. - // This vector fixes option tags, string lengths, byte order and domains. - let options = options(); - let root = RootIdentity([3; 16]); + fn durable_intent_has_frozen_v2_encoding() { + let actual = append_intent_digest(RootIdentity([3; 16]), &options(), b"event\n").unwrap(); assert_eq!( - append_intent_digest(root, &options, b"event\n").unwrap(), + actual, Digest([ - 252, 30, 220, 186, 179, 149, 245, 9, 148, 196, 201, 191, 224, 64, 103, 147, 49, - 197, 154, 223, 18, 234, 168, 87, 99, 0, 52, 10, 45, 175, 183, 3, + 16, 47, 57, 28, 213, 16, 82, 161, 242, 238, 209, 106, 39, 5, 195, 215, 97, 168, + 107, 135, 101, 59, 63, 149, 226, 65, 176, 208, 127, 126, 63, 51 ]) ); + let mut implicit = options(); + implicit.max_logical_size = None; + validate_append_inputs(&mut implicit, b"event\n").unwrap(); assert_eq!( - append_revision_identity(root, &options), - ArtifactRevisionIdentity([ - 138, 32, 231, 5, 12, 133, 71, 175, 10, 237, 44, 92, 128, 156, 4, 168, - ]) + actual, + append_intent_digest(RootIdentity([3; 16]), &implicit, b"event\n").unwrap() ); } #[test] - fn historical_receipt_preserves_original_result_without_live_metadata() { + fn committed_replay_and_incarnation_resolution_need_only_operation_metadata() { let options = options(); - let status = successful_status(&options, b"event\n"); - let expected = status.value.result.clone(); - let receipt = append_receipt( - status, - &options, - append_intent_digest(RootIdentity([3; 16]), &options, b"event\n").unwrap(), - append_revision_identity(RootIdentity([3; 16]), &options), - ) - .unwrap(); + let status = committed_status(&options); + let expected = status.result.clone(); + let (client, calls) = client(status); + assert_eq!( + client + .resolve_append_workspace_incarnation(options.operation_id, &options.target, None) + .unwrap(), + options.expected_workspace_incarnation_id + ); + let receipt = client + .append_artifact_idempotent(&MemoryArtifactStore::new(), options, b"event\n") + .unwrap(); assert!(receipt.replayed); assert_eq!( - Some(OperationResult::ArtifactPublish(receipt.value)), + Some(OperationResult::ArtifactAppend(receipt.value)), expected ); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[test] + fn receipt_preflight_needs_no_provider_and_normalizes_limits() { + let mut options = options(); + let status = committed_status(&options); + let expected = status.result.clone(); + let (client, calls) = client(status); + options.max_logical_size = None; + let receipt = client + .recover_append_receipt(&mut options, b"event\n") + .unwrap() + .unwrap(); + assert_eq!( + Some(OperationResult::ArtifactAppend(receipt.value)), + expected + ); + assert_eq!( + options.max_logical_size, + Some(DEFAULT_APPEND_MAX_LOGICAL_SIZE) + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn only_cleaned_is_a_safe_successor_boundary() { + let options = options(); + for phase in [ + AppendAttemptPhase::Uploading, + AppendAttemptPhase::Finalizing, + AppendAttemptPhase::Aborting, + AppendAttemptPhase::Cleaning, + AppendAttemptPhase::Cleaned, + AppendAttemptPhase::Quarantined, + ] { + let mut status = committed_status(&options); + status.state = if phase == AppendAttemptPhase::Quarantined { + OperationState::Quarantined + } else { + OperationState::Running + }; + status.result = None; + let preparation = status.append_preparation.as_mut().unwrap(); + preparation.attempt_phase = phase; + if matches!( + phase, + AppendAttemptPhase::Cleaned | AppendAttemptPhase::Quarantined + ) { + preparation.attempt_failure = Some(RpcFailure { + code: if phase == AppendAttemptPhase::Quarantined { + ErrorCode::Quarantined + } else { + ErrorCode::OperationFailed + }, + message: "previous publication attempt was safely stopped".to_owned(), + retryable: false, + conflict: Some(ConflictKind::OperationState), + current_generation: None, + route_hint: None, + }); + } + let recovery = append_operation_recovery(&status).unwrap(); + assert_eq!( + recovery.next_action == AppendNextAction::ResubmitSame, + phase == AppendAttemptPhase::Cleaned + ); + if phase != AppendAttemptPhase::Cleaned { + let (client, calls) = client(status); + let error = client + .append_artifact_idempotent( + &MemoryArtifactStore::new(), + options.clone(), + b"event\n", + ) + .unwrap_err(); + assert!( + matches!(error, ClientError::AppendUnresolved {operation_id, ..} if operation_id == options.operation_id) + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + } + } + + #[test] + fn inconsistent_parent_and_child_success_is_never_a_receipt() { + let mut status = committed_status(&options()); + status.append_preparation.as_mut().unwrap().attempt_phase = AppendAttemptPhase::Finalizing; + assert!(matches!( + append_operation_recovery(&status), + Err(ClientError::ResponseMismatch(_)) + )); + status.append_preparation.as_mut().unwrap().attempt_phase = AppendAttemptPhase::Published; + status.state = OperationState::Running; + status.result = None; + assert!(matches!( + append_operation_recovery(&status), + Err(ClientError::ResponseMismatch(_)) + )); } #[test] - fn existing_identity_rejects_every_changed_caller_constraint() { - let original = options().with_max_logical_size(1024); - let status = successful_status(&original, b"event\n"); - let mut changed = Vec::new(); + fn every_bound_input_and_child_identity_is_authenticated() { + let original = options(); + let status = committed_status(&original); + let root = RootIdentity([3; 16]); + let mut variants = Vec::new(); let mut value = original.clone(); - value.target.path = RelativePath::new("logs/other.jsonl").unwrap(); - changed.push(value); + value.target.path = RelativePath::new("other").unwrap(); + variants.push(value); let mut value = original.clone(); - value.target.workbench = WorkbenchName::new("run-43").unwrap(); - changed.push(value); + value.target.workbench = WorkbenchName::new("other").unwrap(); + variants.push(value); let mut value = original.clone(); - value.expected_workspace_incarnation_id = WorkspaceIdentity([5; 16]); - changed.push(value); + value.expected_workspace_incarnation_id = WorkspaceIdentity([8; 16]); + variants.push(value); let mut value = original.clone(); value.create_content_type = ContentType::new("application/json").unwrap(); - changed.push(value); - changed.push( + variants.push(value); + variants.push( original .clone() .with_content_type(ContentType::new("text/plain").unwrap()), ); - changed.push(original.clone().with_block_size(64)); - changed.push(original.clone().with_max_logical_size(2048)); - let mut value = original.clone(); - value.max_logical_size = None; - changed.push(value); - for options in changed { - let error = append_receipt( - status.clone(), - &options, - append_intent_digest(RootIdentity([3; 16]), &options, b"event\n").unwrap(), - append_revision_identity(RootIdentity([3; 16]), &options), + variants.push(original.clone().with_block_size(2048)); + variants.push(original.clone().with_max_logical_size(32 * 1024 * 1024)); + for value in variants { + let intent = append_intent_digest(root, &value, b"event\n").unwrap(); + assert_eq!( + validate_append_intent(root, &status, &value, intent) + .unwrap_err() + .rpc_code(), + Some(ErrorCode::RequestReplayMismatch) + ); + } + for (scope, delta) in [ + (root, b"other\n".as_slice()), + (RootIdentity([6; 16]), b"event\n".as_slice()), + ] { + let intent = append_intent_digest(scope, &original, delta).unwrap(); + assert_eq!( + validate_append_intent(scope, &status, &original, intent) + .unwrap_err() + .rpc_code(), + Some(ErrorCode::RequestReplayMismatch) + ); + } + let intent = append_intent_digest(root, &original, b"event\n").unwrap(); + let mut corrupt = status; + corrupt.append_preparation.as_mut().unwrap().attempt += 1; + assert_eq!( + validate_append_intent(root, &corrupt, &original, intent) + .unwrap_err() + .rpc_code(), + Some(ErrorCode::RequestReplayMismatch) + ); + } + + #[test] + fn explicit_incarnation_and_target_mismatches_keep_logical_id_in_errors() { + let options = options(); + let (client, _) = client(committed_status(&options)); + let error = client + .resolve_append_workspace_incarnation( + options.operation_id, + &options.target, + Some(WorkspaceIdentity([8; 16])), ) .unwrap_err(); - assert_eq!(error.rpc_code(), Some(ErrorCode::RequestReplayMismatch)); - assert!(!error.retryable()); - } - let error = append_receipt( - status, - &original, - append_intent_digest(RootIdentity([3; 16]), &original, b"other\n").unwrap(), - append_revision_identity(RootIdentity([3; 16]), &original), - ) - .unwrap_err(); + assert_eq!(error.rpc_code(), Some(ErrorCode::Conflict)); + assert!( + matches!(error, ClientError::AppendUnresolved {operation_id, ..} if operation_id == options.operation_id) + ); + let mut target = options.target; + target.path = RelativePath::new("changed").unwrap(); + let error = client + .resolve_append_workspace_incarnation(options.operation_id, &target, None) + .unwrap_err(); assert_eq!(error.rpc_code(), Some(ErrorCode::RequestReplayMismatch)); } #[test] - fn unfinished_or_failed_operation_never_becomes_a_new_attempt() { + fn oversized_input_is_rejected_before_rpc_or_provider_admission() { let options = options(); - for state in [ - OperationState::Running, - OperationState::Aborting, - OperationState::Failed, - OperationState::Quarantined, - ] { - let mut status = successful_status(&options, b"event\n"); - status.value.state = state; - status.value.result = None; - let error = append_receipt( - status, - &options, - append_intent_digest(RootIdentity([3; 16]), &options, b"event\n").unwrap(), - append_revision_identity(RootIdentity([3; 16]), &options), + let (client, calls) = client(committed_status(&options)); + let error = client + .append_artifact_idempotent( + &MemoryArtifactStore::new(), + options.clone(), + &vec![0; MAX_APPEND_DELTA_BYTES + 1], ) .unwrap_err(); - assert!(matches!( - error, - ClientError::AppendUnresolved { - operation_id, - state: Some(observed), - .. - } if observed == state && operation_id == options.operation_id - )); - assert!(!error.retryable()); - } + assert!(matches!(error, ClientError::InvalidOptions(_))); + assert_eq!(calls.load(Ordering::SeqCst), 0); + let error = client + .append_artifact_idempotent( + &MemoryArtifactStore::new(), + options.with_max_logical_size(0), + b"x", + ) + .unwrap_err(); + assert!(matches!(error, ClientError::InvalidOptions(_))); + assert_eq!(calls.load(Ordering::SeqCst), 0); } } diff --git a/crates/nokv-client/src/lib.rs b/crates/nokv-client/src/lib.rs index 596710249..76762d7f9 100644 --- a/crates/nokv-client/src/lib.rs +++ b/crates/nokv-client/src/lib.rs @@ -31,7 +31,10 @@ pub use error::{ArtifactPublishStage, ClientError, TransportError}; pub use generic_index::{ GenericIndexAbortOutcome, GenericIndexRegistrationOutcome, GenericIndexRegistrationPlan, }; -pub use idempotent_append::IdempotentAppendOptions; +pub use idempotent_append::{ + append_operation_recovery, AppendNextAction, AppendRecovery, AppendRecoveryState, + IdempotentAppendOptions, DEFAULT_APPEND_MAX_LOGICAL_SIZE, MAX_APPEND_DELTA_BYTES, +}; #[cfg(feature = "control")] pub use route::ControlRouteResolver; #[cfg(feature = "etcd")] diff --git a/crates/nokv-client/src/workbench_workflow.rs b/crates/nokv-client/src/workbench_workflow.rs index d66e27f49..7984edc46 100644 --- a/crates/nokv-client/src/workbench_workflow.rs +++ b/crates/nokv-client/src/workbench_workflow.rs @@ -2115,6 +2115,7 @@ mod tests { state_digest: Digest([8; 32]), }, kind: OperationKind::Commit, + append_preparation: None, publish_preparation: None, commit_preparation: Some(Box::new(preparation)), restore_preparation: None, @@ -2179,8 +2180,10 @@ mod tests { state_digest: Digest([0x18; 32]), }, kind: OperationKind::ArtifactPublish, + append_preparation: None, publish_preparation: Some(Box::new(nokv_protocol::PublishPreparation { append_intent_digest: None, + append_attempt: None, target: options.manifest_target.clone(), workspace_incarnation_id: binding.workspace_incarnation_id, artifact_revision_id: binding.artifact_revision_id, @@ -2438,6 +2441,7 @@ mod tests { state_digest: Digest([10; 32]), }, kind: OperationKind::Restore, + append_preparation: None, publish_preparation: None, commit_preparation: None, restore_preparation: Some(Box::new(restore_operation_preparation( @@ -2463,6 +2467,7 @@ mod tests { state_digest: Digest([10; 32]), }, kind: OperationKind::Restore, + append_preparation: None, publish_preparation: None, commit_preparation: None, restore_preparation: Some(Box::new(restore_operation_preparation( diff --git a/crates/nokv-meta/src/workspace/codec.rs b/crates/nokv-meta/src/workspace/codec.rs index b0e6b0bb4..b5cccde93 100644 --- a/crates/nokv-meta/src/workspace/codec.rs +++ b/crates/nokv-meta/src/workspace/codec.rs @@ -24,7 +24,7 @@ const GENERIC_INDEX_APPEND_RECEIPT_DISCRIMINATOR: u8 = 3; const FIXED_ID_BYTES: usize = 16; pub(crate) const SYSTEM_SCHEMA_KEY: &[u8] = b"schema"; -const SYSTEM_FORMAT_VERSION: u32 = 11; +const SYSTEM_FORMAT_VERSION: u32 = 12; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum SchemaMarkerVersion { @@ -1308,7 +1308,7 @@ mod tests { #[test] fn schema_marker_is_exact_and_versioned() { - assert_eq!(SYSTEM_FORMAT_VERSION, 11); + assert_eq!(SYSTEM_FORMAT_VERSION, 12); let marker = encode_schema_marker(); assert_eq!(marker[0], VALUE_FORMAT_VERSION); assert_eq!( diff --git a/crates/nokv-meta/src/workspace/commit.rs b/crates/nokv-meta/src/workspace/commit.rs index 18c9b24fc..909733400 100644 --- a/crates/nokv-meta/src/workspace/commit.rs +++ b/crates/nokv-meta/src/workspace/commit.rs @@ -39,7 +39,7 @@ use super::commit_records::{ }; use super::engine::{ CommandFit, CommandMutation, CommandPredicate, EventProjection, HistoryProjection, MetaError, - MetaShard, MetadataCommand, MetadataCommandResult, RootFenceAction, + MetaShard, MetadataCommand, MetadataCommandResult, RootFenceAction, MAX_COMMAND_ITEMS, }; use super::event_projection::change_event_projection; use super::generic_index::{ @@ -71,7 +71,6 @@ use nokv_types::{ FIXED_ID_BYTES, SHA256_BYTES, }; -const MAX_COMMAND_ITEMS: usize = 256; /// Canonical Workbench projection installed only with its typed commit head. pub const RUN_MANIFEST_PATH: &str = "metadata/run_manifest.json"; /// Member batches reserve three command items per path plus the operation row. @@ -544,7 +543,11 @@ impl<'a> CommitService<'a> { let mut plan = CommandPlan::default(); // Exclude publish/restore identities atomically with this build's // admission so GetOperation can never become ambiguous after success. - for kind in [OperationKind::Publish, OperationKind::Restore] { + for kind in [ + OperationKind::Publish, + OperationKind::Restore, + OperationKind::Append, + ] { plan.assert_value( MetadataFamily::Operation, super::codec::operation_key(request.context.root_id, kind, request.operation_id), @@ -4366,7 +4369,7 @@ mod tests { ) .unwrap() .len(), - 256 + 260 ); detach_head(&store, &mut counter, commit(1)); diff --git a/crates/nokv-meta/src/workspace/engine.rs b/crates/nokv-meta/src/workspace/engine.rs index 71113cb6c..5dc3db732 100644 --- a/crates/nokv-meta/src/workspace/engine.rs +++ b/crates/nokv-meta/src/workspace/engine.rs @@ -49,7 +49,8 @@ const SYSTEM_RECOVERY_CHAIN_DIGEST_KEY: &[u8] = b"recovery_chain_digest"; const SYSTEM_VALUE_FORMAT_VERSION: u8 = 1; const INITIAL_COMMIT_VERSION: u64 = 1; -const MAX_COMMAND_ITEMS: usize = 256; +/// Shared admission and recovery-codec item budget for one metadata command. +pub(super) const MAX_COMMAND_ITEMS: usize = 260; const MAX_DELIMITED_SCAN_ITEMS: usize = MAX_COMMAND_ITEMS * 2; const MAX_HISTORICAL_SCAN_PAGE_ROWS: usize = MAX_COMMAND_ITEMS; const MAX_HISTORICAL_SCAN_ATTEMPTS: usize = 4; diff --git a/crates/nokv-meta/src/workspace/mod.rs b/crates/nokv-meta/src/workspace/mod.rs index a4fd8e67b..0d80dcc43 100644 --- a/crates/nokv-meta/src/workspace/mod.rs +++ b/crates/nokv-meta/src/workspace/mod.rs @@ -161,11 +161,11 @@ pub use publication_records::{ MAX_QUARANTINE_EVIDENCE_BYTES, PUBLICATION_VALUE_FORMAT_VERSION, }; pub use publish_operation_records::{ - AppendSegment, ArtifactManifestRow, ManifestPosition, PublishAuthority, PublishClaim, - PublishOperationRecord, PublishRecordError, PublishResult, PublishTerminalError, - PublishTerminalErrorKind, PublishTransition, StagedObjectRecord, MAX_APPEND_SEGMENTS, - MAX_MANIFEST_ROWS, MAX_MULTIPART_ID_BYTES, MAX_STAGED_OBJECTS, MAX_TERMINAL_ERROR_BYTES, - PUBLISH_VALUE_FORMAT_VERSION, + AppendAttemptBinding, AppendOperationRecord, AppendSegment, ArtifactManifestRow, + ManifestPosition, PublishAuthority, PublishClaim, PublishOperationRecord, PublishRecordError, + PublishResult, PublishTerminalError, PublishTerminalErrorKind, PublishTransition, + StagedObjectRecord, MAX_APPEND_SEGMENTS, MAX_MANIFEST_ROWS, MAX_MULTIPART_ID_BYTES, + MAX_STAGED_OBJECTS, MAX_TERMINAL_ERROR_BYTES, PUBLISH_VALUE_FORMAT_VERSION, }; pub use query::{ aggregate_paths_at, catalog_fields_at, find_workspaces_at, get_workspace_at, read_changes_at, diff --git a/crates/nokv-meta/src/workspace/publication.rs b/crates/nokv-meta/src/workspace/publication.rs index 5a7906792..7e88cf79b 100644 --- a/crates/nokv-meta/src/workspace/publication.rs +++ b/crates/nokv-meta/src/workspace/publication.rs @@ -32,7 +32,7 @@ use super::codec::{ use super::commit::RUN_MANIFEST_PATH; use super::engine::{ CommandMutation, CommandPredicate, EventProjection, HistoryProjection, MetaError, MetaShard, - MetadataCommand, MetadataCommandResult, RootFenceAction, + MetadataCommand, MetadataCommandResult, RootFenceAction, MAX_COMMAND_ITEMS, }; use super::event_projection::change_event_projection; use super::keyspace::MetadataFamily; @@ -41,10 +41,10 @@ use super::publication_records::{ PublicationRecordCodecError, RevisionRefRecord, WorkspaceRecord, }; use super::publish_operation_records::{ - ArtifactManifestRow, ManifestPosition, PublishAuthority, PublishClaim, PublishOperationRecord, - PublishRecordError, PublishResult, PublishTerminalError, PublishTerminalErrorKind, - PublishTransition, StagedObjectRecord, MAX_DEPENDENCY_COUNT, MAX_MANIFEST_ROWS, - MAX_STAGED_OBJECTS, + AppendOperationRecord, ArtifactManifestRow, ManifestPosition, PublishAuthority, PublishClaim, + PublishOperationRecord, PublishRecordError, PublishResult, PublishTerminalError, + PublishTerminalErrorKind, PublishTransition, StagedObjectRecord, MAX_DEPENDENCY_COUNT, + MAX_MANIFEST_ROWS, MAX_STAGED_OBJECTS, }; use super::query_records::{ secondary_index_key, ChangeEventKind, ChangeEventRecord, QueryRecordError, @@ -57,12 +57,11 @@ use super::restore_records::{ RESTORE_MANIFEST_CONTENT_TYPE, }; -const MAX_COMMAND_ITEMS: usize = 256; /// Maximum rows admitted by one recoverable publication batch. pub const MAX_PUBLICATION_BATCH_ROWS: usize = 192; const _: () = assert!(MAX_PUBLICATION_BATCH_ROWS + 2 <= MAX_COMMAND_ITEMS); const _: () = assert!( - 2 * super::query_records::MAX_TYPED_PROJECTION_FIELDS + 2 * MAX_DEPENDENCY_COUNT as usize + 8 + 2 * super::query_records::MAX_TYPED_PROJECTION_FIELDS + 2 * MAX_DEPENDENCY_COUNT as usize + 10 <= MAX_COMMAND_ITEMS ); @@ -180,6 +179,9 @@ pub enum QuarantineReconcileResolution { /// was never published. Reconciliation releases the revision identity for /// a fresh begin. RevisionUnpublished, + /// Every failed append key is permanently sealed against delayed immutable + /// creates. The unpublished revision identity remains reserved forever. + RevisionUnpublishedSealed, /// The artifact revision is already published, so the staged provider /// keys are the published revision's live objects and must not be touched. /// Only this operation's private bookkeeping rows are removed. @@ -953,6 +955,7 @@ fn validate_operation_seals(operation: &PublishOperationRecord) -> Result<(), Pu fn validate_cleanup_update( expected: &StagedObjectRecord, next: &StagedObjectRecord, + append: bool, ) -> Result<(), PublicationError> { expected.validate()?; next.validate()?; @@ -963,7 +966,12 @@ fn validate_cleanup_update( || expected.expected_length != next.expected_length || expected.expected_digest_uri != next.expected_digest_uri || next.provider_state != StagedProviderState::Aborted - || next.cleanup_state != StagedCleanupState::Deleted + || next.cleanup_state + != if append { + StagedCleanupState::Sealed + } else { + StagedCleanupState::Deleted + } { return Err(PublicationError::InvalidStagedObjectTransition { sequence: expected.object_sequence, @@ -1239,7 +1247,7 @@ fn next_generation( fn publish_identity_digest(operation: &PublishOperationRecord) -> [u8; SHA256_BYTES] { let mut hasher = Sha256::new(); - hasher.update(b"nokv.publish.operation.identity.v4\0"); + hasher.update(b"nokv.publish.operation.identity.v5\0"); hasher.update(operation.operation_id.as_bytes()); match operation.append_intent_digest { Some(digest) => { @@ -1248,6 +1256,14 @@ fn publish_identity_digest(operation: &PublishOperationRecord) -> [u8; SHA256_BY } None => hasher.update([0]), } + match operation.append_attempt { + None => hasher.update([0]), + Some(binding) => { + hasher.update([1]); + hasher.update(binding.operation_id.as_bytes()); + hasher.update(binding.attempt.to_be_bytes()); + } + } hasher.update(operation.initiating_owner_epoch.get().to_be_bytes()); match operation.authority { PublishAuthority::Visible => hasher.update([1]), @@ -1303,7 +1319,7 @@ fn publish_identity_digest(operation: &PublishOperationRecord) -> [u8; SHA256_BY fn publish_initialization_digest(operation: &PublishOperationRecord) -> [u8; SHA256_BYTES] { let mut hasher = Sha256::new(); - hasher.update(b"nokv.publish.operation.initialization.v4\0"); + hasher.update(b"nokv.publish.operation.initialization.v5\0"); match operation.append_intent_digest { Some(digest) => { hasher.update([1]); @@ -1311,6 +1327,14 @@ fn publish_initialization_digest(operation: &PublishOperationRecord) -> [u8; SHA } None => hasher.update([0]), } + match operation.append_attempt { + None => hasher.update([0]), + Some(binding) => { + hasher.update([1]); + hasher.update(binding.operation_id.as_bytes()); + hasher.update(binding.attempt.to_be_bytes()); + } + } hasher.update(operation.initiating_owner_epoch.get().to_be_bytes()); match operation.authority { PublishAuthority::Visible => hasher.update([1]), @@ -1391,6 +1415,7 @@ fn finalization_takeover_absence_proof( fn reconcile_resolution_label(resolution: QuarantineReconcileResolution) -> &'static str { match resolution { QuarantineReconcileResolution::RevisionUnpublished => "revision-unpublished", + QuarantineReconcileResolution::RevisionUnpublishedSealed => "revision-unpublished-sealed", QuarantineReconcileResolution::RevisionPublished => "revision-published", } } @@ -1413,6 +1438,7 @@ fn reconcile_evidence_digest( hasher.update(b"nokv.publish.reconcile-evidence.v1\0"); hasher.update([match resolution { QuarantineReconcileResolution::RevisionUnpublished => 1, + QuarantineReconcileResolution::RevisionUnpublishedSealed => 3, QuarantineReconcileResolution::RevisionPublished => 2, }]); match original_evidence_digest { @@ -1640,6 +1666,20 @@ impl PublicationService<'_> { // successful initialization/complete must remain replayable only // when the live path and exact destination binding still agree. self.validate_existing_publish_replay_authority(request.context, &operation)?; + if let Some(binding) = operation.append_attempt { + let key = super::codec::operation_key( + request.context.root_id, + OperationKind::Append, + binding.operation_id, + ); + let payload = self + .read_payload(request.context, MetadataFamily::Operation, &key)? + .ok_or(PublicationError::OperationInputMismatch)?; + let parent = AppendOperationRecord::decode(&payload)?; + if !parent.matches_publication(&operation) || parent.result != operation.result { + return Err(PublicationError::OperationInputMismatch); + } + } return Ok(PublishCommandOutcome { commit_version: CommitVersion::new(request.context.read_version.get()) .expect("publication contexts always have a non-zero read version"), @@ -1677,9 +1717,13 @@ impl PublicationService<'_> { let operation_payload = request.operation.encode()?; let mut plan = CommandPlan::default(); // Public operation lookup has one root-scoped identity across all - // three lifecycle kinds. Fence competing admissions in this same + // four public lifecycle kinds. Fence competing admissions in this same // transaction, rather than trusting a prior lookup by the client. - for kind in [OperationKind::BuildCommit, OperationKind::Restore] { + for kind in [ + OperationKind::BuildCommit, + OperationKind::Restore, + OperationKind::Append, + ] { plan.assert_value( MetadataFamily::Operation, super::codec::operation_key( @@ -1690,6 +1734,7 @@ impl PublicationService<'_> { None, )?; } + self.prepare_append_admission(request.context, &request.operation, &mut plan)?; plan.put_absent( MetadataFamily::Operation, operation_key, @@ -2220,16 +2265,23 @@ impl PublicationService<'_> { expected_operation_payload, next_operation_payload.clone(), )?; - // Cleanup completion is the terminal owner transition for an aborted - // operation: its provider objects are durably gone, so the revision - // identity becomes claimable again in the same command. + // Failed append keys remain permanently sealed. Retain their revision + // reservation so no later publisher can reuse the sealed key namespace. if matches!(request.transition, PublishTransition::FinishCleanup) { - self.release_revision_claim( - request.context, - request.expected_operation.artifact_revision_id, - request.expected_operation.operation_id, - &mut plan, - )?; + if request.expected_operation.append_attempt.is_some() { + self.predicate_append_revision_reservation( + request.context, + &request.expected_operation, + &mut plan, + )?; + } else { + self.release_revision_claim( + request.context, + request.expected_operation.artifact_revision_id, + request.expected_operation.operation_id, + &mut plan, + )?; + } } let result = self.execute_plan( @@ -2388,6 +2440,77 @@ impl PublicationService<'_> { decode_operation_outcome(result, next_operation.operation_id) } + /// End an append attempt whose frozen path claim is definitively invalid. + /// The exact Finalizing CAS, absent revision, and current logical parent + /// jointly prove that this attempt cannot publish after cleanup wins. + pub fn fail_append_conflict( + &self, + context: PublicationContext, + operation: PublishOperationRecord, + ) -> Result { + validate_operation_seals(&operation)?; + if operation.append_attempt.is_none() || operation.phase != PublishPhase::Finalizing { + return Err(PublicationError::InvalidOperationPhase { + expected: PublishPhase::Finalizing, + actual: operation.phase, + }); + } + self.require_current_operation(context, &operation)?; + let path_key = path_current_key( + context.root_id, + operation.workspace_incarnation_id, + &operation.path, + ); + let current = self.read_publication_record( + context, + MetadataFamily::PathCurrent, + &path_key, + PathEntry::decode, + )?; + match validate_path_claim(&operation.claim, current.as_ref()) { + Err( + PublicationError::PathAlreadyExists + | PublicationError::PathNotFound + | PublicationError::PathGenerationMismatch { .. } + | PublicationError::AppendBaseRevisionMismatch, + ) => {} + Err(error) => return Err(error), + Ok(_) => return Err(PublicationError::ConcurrentMutation), + } + let expected_payload = operation.encode()?; + let proof = finalization_takeover_absence_proof(context, &expected_payload)?; + let mut next = operation; + next.take_over_finalization( + proof, + PublishTerminalError { + kind: PublishTerminalErrorKind::PublishConflict, + message: "append attempt lost its frozen path claim".to_owned(), + evidence_digest: None, + }, + )?; + let payload = next.encode()?; + let mut plan = CommandPlan::default(); + plan.replace( + MetadataFamily::Operation, + operation_key(context.root_id, OperationKind::Publish, next.operation_id), + expected_payload, + payload.clone(), + )?; + plan.assert_value( + MetadataFamily::ArtifactRevision, + artifact_revision_key(context.root_id, next.artifact_revision_id), + None, + )?; + plan.assert_value( + MetadataFamily::PathCurrent, + path_key, + current.map(|loaded| loaded.payload), + )?; + let result = + self.execute_plan(context, plan, payload, PublicationAuthorityPurpose::Cleanup)?; + decode_operation_outcome(result, next.operation_id) + } + pub fn cleanup_publish_batch( &self, request: CleanupPublishBatchRequest, @@ -2425,7 +2548,11 @@ impl PublicationService<'_> { count: request.staged_object_updates.len(), planned: next_operation.staged_object_cursor, })?; - validate_cleanup_update(&update.expected, &update.next)?; + validate_cleanup_update( + &update.expected, + &update.next, + next_operation.append_attempt.is_some(), + )?; if update.expected.object_sequence != sequence { return Err(PublicationError::StagedObjectSequenceMismatch { expected: sequence, @@ -2694,8 +2821,9 @@ impl PublicationService<'_> { /// Atomically resolve one fully-swept quarantined operation: CAS the /// operator terminal error in, transition `Quarantined -> Cleaned`, and - /// release the revision claim when this operation owns it. The verdict's - /// revision predicate is pinned by the same command. + /// release a generic publication's revision claim. Failed append claims + /// stay reserved because their provider keys remain permanently sealed. + /// The verdict's revision predicate is pinned by the same command. pub fn finish_reconcile_quarantined_publish( &self, request: FinishReconcileQuarantinedPublishRequest, @@ -2740,15 +2868,20 @@ impl PublicationService<'_> { expected_payload, next_payload.clone(), )?; - // Reconciliation is the operator path the quarantine fail-closed - // default was waiting for: with the staged liability durably resolved, - // the revision identity becomes claimable again in the same command. - self.release_revision_claim( - request.context, - request.expected_operation.artifact_revision_id, - request.expected_operation.operation_id, - &mut plan, - )?; + if request.expected_operation.append_attempt.is_some() { + self.predicate_append_revision_reservation( + request.context, + &request.expected_operation, + &mut plan, + )?; + } else { + self.release_revision_claim( + request.context, + request.expected_operation.artifact_revision_id, + request.expected_operation.operation_id, + &mut plan, + )?; + } let result = self.execute_reconcile_plan(request.context, plan, next_payload)?; decode_operation_outcome(result, next_operation.operation_id) } @@ -2766,8 +2899,23 @@ impl PublicationService<'_> { let revision_key = artifact_revision_key(context.root_id, operation.artifact_revision_id); let payload = self.read_payload(context, MetadataFamily::ArtifactRevision, &revision_key)?; + if operation.append_attempt.is_some() + != matches!( + resolution, + QuarantineReconcileResolution::RevisionUnpublishedSealed + ) + { + return Err(PublicationError::ReconcileResolutionMismatch { + resolution, + revision_published: payload.is_some(), + }); + } match (resolution, payload) { - (QuarantineReconcileResolution::RevisionUnpublished, None) => { + ( + QuarantineReconcileResolution::RevisionUnpublished + | QuarantineReconcileResolution::RevisionUnpublishedSealed, + None, + ) => { plan.assert_value(MetadataFamily::ArtifactRevision, revision_key, None)?; } (QuarantineReconcileResolution::RevisionPublished, Some(payload)) => { @@ -3065,9 +3213,12 @@ impl PublicationService<'_> { fn execute_reconcile_plan( &self, context: PublicationContext, - plan: CommandPlan, + mut plan: CommandPlan, deterministic_result: Vec, ) -> Result { + let operation = PublishOperationRecord::decode(&deterministic_result)?; + self.predicate_append_authority(context, &operation, &mut plan)?; + self.synchronize_active_publication(context, &operation, &mut plan)?; plan.validate_bounds()?; let command = MetadataCommand { schema_id: SCHEMA_ID.to_owned(), @@ -3284,6 +3435,382 @@ impl PublicationService<'_> { Ok(()) } + /// Keep failed append object keys owned by the original immutable child. + fn predicate_append_revision_reservation( + &self, + context: PublicationContext, + operation: &PublishOperationRecord, + plan: &mut CommandPlan, + ) -> Result<(), PublicationError> { + let key = artifact_revision_claim_key(context.root_id, operation.artifact_revision_id); + let payload = self + .read_payload(context, MetadataFamily::ArtifactRevision, &key)? + .ok_or(PublicationError::InvalidOperationSeal { + seal: "append revision reservation", + })?; + if ArtifactRevisionClaimRecord::decode(&payload)?.operation_id != operation.operation_id { + return Err(PublicationError::InvalidOperationSeal { + seal: "append revision reservation", + }); + } + plan.assert_value(MetadataFamily::ArtifactRevision, key, Some(payload)) + } + + /// Check the complete publication index and logical-parent graph at one + /// read version. This diagnoses missing/orphan markers without repairing + /// state or inferring that an unindexed operation is safe to abandon. + pub fn verify_append_publication_state( + &self, + context: PublicationContext, + ) -> Result<(), PublicationError> { + let corrupt = |reason: &str| { + PublicationError::Meta(MetaError::CorruptRecord { + record: "append publication state", + reason: reason.to_owned(), + }) + }; + for kind in [ + OperationKind::Publish, + OperationKind::Append, + OperationKind::ActivePublish, + ] { + let prefix = super::codec::operation_prefix(context.root_id, kind); + let mut cursor = None; + loop { + let rows = self.store.scan_prefix_at( + context.root_id, + context.placement_generation, + context.owner_epoch, + MetadataFamily::Operation, + &prefix, + context.read_version, + cursor.as_deref(), + 256, + )?; + for row in &rows { + let identity = + super::codec::decode_operation_key(context.root_id, kind, &row.key) + .ok_or_else(|| corrupt("invalid operation key"))?; + match kind { + OperationKind::Publish => { + let operation = PublishOperationRecord::decode(&row.value)?; + validate_operation_seals(&operation)?; + if operation.operation_id != identity { + return Err(corrupt("publication key and identity disagree")); + } + let active = matches!( + operation.phase, + PublishPhase::Uploading + | PublishPhase::Finalizing + | PublishPhase::Aborting + | PublishPhase::Cleaning + ); + let marker = self.read_payload( + context, + MetadataFamily::Operation, + &operation_key( + context.root_id, + OperationKind::ActivePublish, + identity, + ), + )?; + if marker.as_deref() != active.then_some(&[1][..]) { + return Err(corrupt("publication activity marker is missing, unexpected, or malformed")); + } + if let Some(binding) = operation.append_attempt { + if operation.phase != PublishPhase::Published { + let claim = self + .read_payload( + context, + MetadataFamily::ArtifactRevision, + &artifact_revision_claim_key( + context.root_id, + operation.artifact_revision_id, + ), + )? + .ok_or_else(|| { + corrupt("append revision reservation is missing") + })?; + if ArtifactRevisionClaimRecord::decode(&claim)?.operation_id + != identity + { + return Err(corrupt("append revision reservation belongs to another operation")); + } + } + let payload = self + .read_payload( + context, + MetadataFamily::Operation, + &operation_key( + context.root_id, + OperationKind::Append, + binding.operation_id, + ), + )? + .ok_or_else(|| corrupt("append child has no logical parent"))?; + let mut parent = AppendOperationRecord::decode(&payload)?; + if parent.publication_operation_id != identity { + if operation.phase != PublishPhase::Cleaned + || parent.attempt <= binding.attempt + { + return Err(corrupt( + "superseded append child is not safely cleaned", + )); + } + parent.attempt = binding.attempt; + parent.publication_operation_id = identity; + parent.artifact_revision_id = operation.artifact_revision_id; + } + if !parent.matches_publication(&operation) { + return Err(corrupt( + "append child identity differs from its parent", + )); + } + } + } + OperationKind::Append => { + let parent = AppendOperationRecord::decode(&row.value)?; + if parent.operation_id != identity { + return Err(corrupt("append parent key and identity disagree")); + } + let payload = self + .read_payload( + context, + MetadataFamily::Operation, + &operation_key( + context.root_id, + OperationKind::Publish, + parent.publication_operation_id, + ), + )? + .ok_or_else(|| corrupt("append parent has no current child"))?; + let child = PublishOperationRecord::decode(&payload)?; + if !parent.matches_publication(&child) + || parent.result != child.result + || parent.result.is_some() + != (child.phase == PublishPhase::Published) + { + return Err(corrupt( + "append parent result and current child disagree", + )); + } + for other in [ + OperationKind::Publish, + OperationKind::BuildCommit, + OperationKind::Restore, + ] { + if self + .read_payload( + context, + MetadataFamily::Operation, + &operation_key(context.root_id, other, identity), + )? + .is_some() + { + return Err(corrupt( + "logical append identity is occupied by another lifecycle", + )); + } + } + } + OperationKind::ActivePublish => { + if row.value != [1] { + return Err(corrupt("unknown active publication marker")); + } + let payload = self + .read_payload( + context, + MetadataFamily::Operation, + &operation_key( + context.root_id, + OperationKind::Publish, + identity, + ), + )? + .ok_or_else(|| { + corrupt("active marker has no canonical publication") + })?; + let operation = PublishOperationRecord::decode(&payload)?; + if !matches!( + operation.phase, + PublishPhase::Uploading + | PublishPhase::Finalizing + | PublishPhase::Aborting + | PublishPhase::Cleaning + ) { + return Err(corrupt("terminal publication remains active")); + } + } + _ => unreachable!("only publication graph kinds are scanned"), + } + } + if rows.len() < 256 { + break; + } + cursor = rows.last().map(|row| row.key.clone()); + } + } + Ok(()) + } + + fn prepare_append_admission( + &self, + context: PublicationContext, + operation: &PublishOperationRecord, + plan: &mut CommandPlan, + ) -> Result<(), PublicationError> { + let Some(binding) = operation.append_attempt else { + return Ok(()); + }; + let intent_digest = operation + .append_intent_digest + .ok_or(PublicationError::OperationInputMismatch)?; + let key = operation_key(context.root_id, OperationKind::Append, binding.operation_id); + let parent = AppendOperationRecord { + operation_id: binding.operation_id, + intent_digest, + workbench_id: operation.workbench_id.clone(), + workspace_incarnation_id: operation.workspace_incarnation_id, + path: operation.path.clone(), + attempt: binding.attempt, + publication_operation_id: operation.operation_id, + artifact_revision_id: operation.artifact_revision_id, + result: None, + }; + match self.read_payload(context, MetadataFamily::Operation, &key)? { + None => { + if binding.attempt != 0 { + return Err(PublicationError::OperationInputMismatch); + } + for kind in [ + OperationKind::Publish, + OperationKind::BuildCommit, + OperationKind::Restore, + ] { + plan.assert_value( + MetadataFamily::Operation, + operation_key(context.root_id, kind, binding.operation_id), + None, + )?; + } + plan.put_absent(MetadataFamily::Operation, key, parent.encode()?)?; + } + Some(payload) => { + let previous = AppendOperationRecord::decode(&payload)?; + if previous.operation_id != binding.operation_id + || previous.intent_digest != intent_digest + || previous.workbench_id != operation.workbench_id + || previous.path != operation.path + || previous.workspace_incarnation_id != operation.workspace_incarnation_id + || previous.result.is_some() + || previous.attempt.checked_add(1) != Some(binding.attempt) + { + return Err(PublicationError::OperationInputMismatch); + } + let child_key = operation_key( + context.root_id, + OperationKind::Publish, + previous.publication_operation_id, + ); + let child_payload = self + .read_payload(context, MetadataFamily::Operation, &child_key)? + .ok_or(PublicationError::OperationInputMismatch)?; + let child = PublishOperationRecord::decode(&child_payload)?; + validate_operation_seals(&child)?; + if !previous.matches_publication(&child) + || child.phase != PublishPhase::Cleaned + || child.result.is_some() + { + return Err(PublicationError::ConcurrentMutation); + } + self.predicate_append_revision_reservation(context, &child, plan)?; + plan.assert_value(MetadataFamily::Operation, child_key, Some(child_payload))?; + plan.assert_value( + MetadataFamily::ArtifactRevision, + artifact_revision_key(context.root_id, previous.artifact_revision_id), + None, + )?; + plan.replace(MetadataFamily::Operation, key, payload, parent.encode()?)?; + } + } + Ok(()) + } + + fn predicate_append_authority( + &self, + context: PublicationContext, + operation: &PublishOperationRecord, + plan: &mut CommandPlan, + ) -> Result<(), PublicationError> { + let Some(binding) = operation.append_attempt else { + return Ok(()); + }; + let key = operation_key(context.root_id, OperationKind::Append, binding.operation_id); + // Begin and final publication replace parent and child together. Validate + // their planned parent; all other transitions fence the current parent. + let planned = plan.mutations.iter().find_map(|mutation| match mutation { + CommandMutation::Put { + family: MetadataFamily::Operation, + key: candidate, + value, + } if candidate == &key => Some(value), + _ => None, + }); + if let Some(payload) = planned { + if !AppendOperationRecord::decode(payload)?.matches_publication(operation) { + return Err(PublicationError::OperationInputMismatch); + } + } else { + let payload = self + .read_payload(context, MetadataFamily::Operation, &key)? + .ok_or(PublicationError::OperationInputMismatch)?; + let parent = AppendOperationRecord::decode(&payload)?; + if !parent.matches_publication(operation) || parent.result.is_some() { + return Err(PublicationError::ConcurrentMutation); + } + if !plan + .exact_keys + .contains(&(MetadataFamily::Operation, key.clone())) + { + plan.assert_value(MetadataFamily::Operation, key, Some(payload))?; + } + } + Ok(()) + } + + fn synchronize_active_publication( + &self, + context: PublicationContext, + operation: &PublishOperationRecord, + plan: &mut CommandPlan, + ) -> Result<(), PublicationError> { + let key = operation_key( + context.root_id, + OperationKind::ActivePublish, + operation.operation_id, + ); + let current = self.read_payload(context, MetadataFamily::Operation, &key)?; + if current.as_ref().is_some_and(|payload| payload != &[1]) { + return Err(PublicationError::OperationInputMismatch); + } + let active = matches!( + operation.phase, + PublishPhase::Uploading + | PublishPhase::Finalizing + | PublishPhase::Aborting + | PublishPhase::Cleaning + ); + match (active, current) { + (true, None) => plan.put_absent(MetadataFamily::Operation, key, vec![1])?, + (false, Some(payload)) => plan.delete(MetadataFamily::Operation, key, payload)?, + (_, Some(payload)) if payload != [1] => { + return Err(PublicationError::OperationInputMismatch) + } + _ => {} + } + Ok(()) + } + fn execute_plan( &self, context: PublicationContext, @@ -3328,6 +3855,8 @@ impl PublicationService<'_> { authority_purpose: PublicationAuthorityPurpose, ) -> Result { let operation = PublishOperationRecord::decode(&deterministic_result)?; + self.predicate_append_authority(context, &operation, &mut plan)?; + self.synchronize_active_publication(context, &operation, &mut plan)?; self.predicate_publication_authority(context, &operation, authority_purpose, &mut plan)?; plan.validate_bounds()?; Ok(MetadataCommand { @@ -3769,6 +4298,23 @@ impl PublicationService<'_> { expected_operation_payload, published_operation_payload.clone(), )?; + if let Some(binding) = published_operation.append_attempt { + let key = super::codec::operation_key( + request.context.root_id, + OperationKind::Append, + binding.operation_id, + ); + let payload = self + .read_payload(request.context, MetadataFamily::Operation, &key)? + .ok_or(PublicationError::OperationInputMismatch)?; + let mut parent = AppendOperationRecord::decode(&payload)?; + if !parent.matches_publication(&published_operation) || parent.result.is_some() { + return Err(PublicationError::OperationInputMismatch); + } + parent.result = Some(publish_result.clone()); + plan.replace(MetadataFamily::Operation, key, payload, parent.encode()?)?; + } + if let Some(next_workspace) = next_workspace { plan.replace( MetadataFamily::WorkspaceCurrent, @@ -4523,6 +5069,7 @@ mod tests { ) -> PublishOperationRecord { let dependencies = Vec::new(); let mut operation = PublishOperationRecord { + append_attempt: None, append_intent_digest: None, operation_id, identity_digest: [0; SHA256_BYTES], @@ -5252,8 +5799,15 @@ mod tests { claim: PublishClaim, dependencies: &[ArtifactRevisionId], projection: TypedProjection, + append_attempt: Option, ) -> FinalizePublishOutcome { - let staged = staged_rows(artifact_revision_id, 1); + let row_count = match claim { + PublishClaim::Append { append_offset, .. } => { + usize::try_from(append_offset + 1).unwrap() + } + _ => 1, + }; + let staged = staged_rows(artifact_revision_id, row_count); let manifest = manifest_rows(&staged); let mut operation = publish_operation( operation_id, @@ -5263,6 +5817,8 @@ mod tests { &staged, &manifest, ); + operation.append_attempt = append_attempt; + operation.append_intent_digest = append_attempt.map(|_| [0xb1; SHA256_BYTES]); operation.dependency_count = u8::try_from(dependencies.len()).unwrap(); operation.dependency_depth = u8::from(!dependencies.is_empty()); operation.dependency_digest = dependency_owner_digest(dependencies).unwrap(); @@ -5419,52 +5975,670 @@ mod tests { ); } - #[test] - fn stable_append_identity_binds_all_256_intent_bits_and_cannot_drop_binding() { - let mut counter = 1; - let store = ready_store(&mut counter); - let service = PublicationService::new(&store); - let revision_id = revision(91_090); - let staged = staged_rows(revision_id, 1); - let manifest = manifest_rows(&staged); - let mut initial = publish_operation( - operation_id(91_090), - revision_id, - path("outputs/intent.bin"), + fn logical_append_child(attempt: u64) -> PublishOperationRecord { + let mut operation = publish_operation( + operation_id(92_000 + u128::from(attempt)), + revision(92_000 + u128::from(attempt)), + path("outputs/logical-append.log"), PublishClaim::CreateOnly, - &staged, - &manifest, + &[], + &[], ); - initial.append_intent_digest = Some([0x61; SHA256_BYTES]); - seal_publish_operation(&mut initial); - let begun = begin_operation(&service, &store, &mut counter, initial.clone()); - assert_eq!(begun.append_intent_digest, initial.append_intent_digest); - let mut changed_digest = [0x61; SHA256_BYTES]; - changed_digest[SHA256_BYTES - 1] = 0x62; - for intent in [Some(changed_digest), None] { - let mut changed = initial.clone(); - changed.append_intent_digest = intent; - seal_publish_operation(&mut changed); - assert_ne!(changed.identity_digest, initial.identity_digest); - assert_ne!(changed.initialization_digest, initial.initialization_digest); - assert_eq!( - service.begin_publish(BeginPublishRequest { - context: publication_context(&store, &mut counter), - operation: changed, - }), - Err(PublicationError::OperationInputMismatch) - ); - } - let replay = service - .begin_publish(BeginPublishRequest { - context: publication_context(&store, &mut counter), - operation: initial, - }) - .unwrap(); - assert!(replay.replayed); - assert_eq!(replay.operation, begun); - } - + operation.append_attempt = Some(super::super::AppendAttemptBinding { + operation_id: operation_id(93_000), + attempt, + }); + operation.append_intent_digest = Some([0xa7; SHA256_BYTES]); + seal_publish_operation(&mut operation); + operation + } + + fn read_append_parent( + service: &PublicationService<'_>, + store: &MetaShard, + counter: &mut u128, + ) -> AppendOperationRecord { + let payload = service + .read_payload( + publication_context(store, counter), + MetadataFamily::Operation, + &operation_key(root(), OperationKind::Append, operation_id(93_000)), + ) + .unwrap() + .unwrap(); + AppendOperationRecord::decode(&payload).unwrap() + } + + #[test] + fn append_cleanup_requires_seals_and_reserves_failed_revision_after_successor_admission() { + for reconcile in [false, true] { + let mut counter = 1; + let store = ready_store(&mut counter); + let service = PublicationService::new(&store); + let template = logical_append_child(0); + let staged = staged_rows(template.artifact_revision_id, 1); + let manifest = manifest_rows(&staged); + let mut initial = publish_operation( + template.operation_id, + template.artifact_revision_id, + template.path.clone(), + PublishClaim::CreateOnly, + &staged, + &manifest, + ); + initial.append_attempt = template.append_attempt; + initial.append_intent_digest = template.append_intent_digest; + seal_publish_operation(&mut initial); + let child = begin_operation(&service, &store, &mut counter, initial); + let mut child = service + .stage_objects_batch(StageObjectsBatchRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + staged_objects: staged.clone(), + }) + .unwrap() + .operation; + for transition in [ + PublishTransition::BeginAbort { + terminal_error: PublishTerminalError { + kind: PublishTerminalErrorKind::ActivityLeaseExpired, + message: "abandoned append upload".to_owned(), + evidence_digest: None, + }, + }, + PublishTransition::BeginCleaning, + ] { + child = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + transition, + }) + .unwrap() + .operation; + } + if reconcile { + child = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + transition: PublishTransition::Quarantine { + terminal_error: PublishTerminalError { + kind: PublishTerminalErrorKind::CleanupFailed, + message: "seal acknowledgement lost".to_owned(), + evidence_digest: Some([0xc1; 32]), + }, + }, + }) + .unwrap() + .operation; + let before = store.current_read_version().unwrap(); + assert!(matches!( + service.reconcile_quarantined_publish_batch( + ReconcileQuarantinedPublishBatchRequest { + context: publication_context(&store, &mut counter), + expected_operation: child.clone(), + resolution: QuarantineReconcileResolution::RevisionUnpublished, + staged_object_rows: staged.clone(), + } + ), + Err(PublicationError::ReconcileResolutionMismatch { .. }) + )); + assert_eq!(store.current_read_version().unwrap(), before); + child = service + .reconcile_quarantined_publish_batch(ReconcileQuarantinedPublishBatchRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + resolution: QuarantineReconcileResolution::RevisionUnpublishedSealed, + staged_object_rows: staged.clone(), + }) + .unwrap() + .operation; + assert!(service + .finish_reconcile_quarantined_publish( + FinishReconcileQuarantinedPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child.clone(), + resolution: QuarantineReconcileResolution::RevisionUnpublished, + reason: "absent is insufficient".to_owned(), + operator_evidence_digest: [0xc2; 32], + } + ) + .is_err()); + child = service + .finish_reconcile_quarantined_publish( + FinishReconcileQuarantinedPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + resolution: QuarantineReconcileResolution::RevisionUnpublishedSealed, + reason: "each immutable key is permanently sealed".to_owned(), + operator_evidence_digest: [0xc3; 32], + }, + ) + .unwrap() + .operation; + } else { + let mut next = staged[0].clone(); + next.provider_state = StagedProviderState::Aborted; + next.cleanup_state = StagedCleanupState::Deleted; + let before = store.current_read_version().unwrap(); + assert!(matches!( + service.cleanup_publish_batch(CleanupPublishBatchRequest { + context: publication_context(&store, &mut counter), + expected_operation: child.clone(), + staged_object_updates: vec![StagedObjectUpdate { + expected: staged[0].clone(), + next: next.clone() + }], + }), + Err(PublicationError::InvalidStagedObjectTransition { .. }) + )); + assert_eq!(store.current_read_version().unwrap(), before); + next.cleanup_state = StagedCleanupState::Sealed; + child = service + .cleanup_publish_batch(CleanupPublishBatchRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + staged_object_updates: vec![StagedObjectUpdate { + expected: staged[0].clone(), + next, + }], + }) + .unwrap() + .operation; + child = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + transition: PublishTransition::FinishCleanup, + }) + .unwrap() + .operation; + } + assert_eq!(child.phase, PublishPhase::Cleaned); + let claim_key = artifact_revision_claim_key(root(), child.artifact_revision_id); + let claim = service + .read_payload( + publication_context(&store, &mut counter), + MetadataFamily::ArtifactRevision, + &claim_key, + ) + .unwrap() + .unwrap(); + assert_eq!( + ArtifactRevisionClaimRecord::decode(&claim) + .unwrap() + .operation_id, + child.operation_id + ); + let successor = + begin_operation(&service, &store, &mut counter, logical_append_child(1)); + assert_eq!( + read_append_parent(&service, &store, &mut counter).publication_operation_id, + successor.operation_id + ); + service + .verify_append_publication_state(publication_context(&store, &mut counter)) + .unwrap(); + let reused_revision = publish_operation( + operation_id(94_000), + child.artifact_revision_id, + path("outputs/reuse-must-fail"), + PublishClaim::CreateOnly, + &[], + &[], + ); + assert!(matches!( + service.begin_publish(BeginPublishRequest { + context: publication_context(&store, &mut counter), + operation: reused_revision, + }), + Err(PublicationError::RevisionClaimHeld { .. }) + )); + } + } + + #[test] + fn append_parent_and_child_admit_and_advance_atomically_without_empty_attempts() { + let mut counter = 1; + let store = ready_store(&mut counter); + let service = PublicationService::new(&store); + let first = begin_operation(&service, &store, &mut counter, logical_append_child(0)); + let parent = read_append_parent(&service, &store, &mut counter); + assert!(parent.matches_publication(&first)); + assert!(parent.result.is_none()); + let active_prefix = super::super::operation_prefix(root(), OperationKind::ActivePublish); + assert_eq!( + count_prefix(&store, MetadataFamily::Operation, &active_prefix), + 1 + ); + for attempt in [1, 2] { + assert!(service + .begin_publish(BeginPublishRequest { + context: publication_context(&store, &mut counter), + operation: logical_append_child(attempt) + }) + .is_err()); + assert_eq!(read_append_parent(&service, &store, &mut counter), parent); + assert!(service + .read_payload( + publication_context(&store, &mut counter), + MetadataFamily::Operation, + &operation_key( + root(), + OperationKind::Publish, + operation_id(92_000 + u128::from(attempt)) + ) + ) + .unwrap() + .is_none()); + } + let mut child = first.clone(); + for transition in [ + PublishTransition::BeginAbort { + terminal_error: PublishTerminalError { + kind: PublishTerminalErrorKind::AbortedByCaller, + message: "abandoned upload".to_owned(), + evidence_digest: None, + }, + }, + PublishTransition::BeginCleaning, + PublishTransition::FinishCleanup, + ] { + child = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + transition, + }) + .unwrap() + .operation; + } + assert_eq!(child.phase, PublishPhase::Cleaned); + assert_eq!( + count_prefix(&store, MetadataFamily::Operation, &active_prefix), + 0 + ); + let next = begin_operation(&service, &store, &mut counter, logical_append_child(1)); + let advanced = read_append_parent(&service, &store, &mut counter); + assert!(advanced.matches_publication(&next)); + assert_eq!(advanced.attempt, 1); + assert!(service + .begin_publish(BeginPublishRequest { + context: publication_context(&store, &mut counter), + operation: first + }) + .is_err()); + assert_eq!( + count_prefix(&store, MetadataFamily::Operation, &active_prefix), + 1 + ); + } + + #[test] + fn append_parent_receipt_commits_with_child_and_can_never_advance_after_success() { + let mut counter = 1; + let store = ready_store(&mut counter); + let service = PublicationService::new(&store); + let child = begin_operation(&service, &store, &mut counter, logical_append_child(0)); + let child = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + transition: PublishTransition::BeginFinalization, + }) + .unwrap() + .operation; + let outcome = service + .finalize_publish(FinalizePublishRequest { + context: publication_context(&store, &mut counter), + artifact: published_artifact(&child), + expected_operation: child, + dependency_owner_revision_ids: Vec::new(), + }) + .unwrap(); + let parent = read_append_parent(&service, &store, &mut counter); + assert!(parent.matches_publication(&outcome.operation)); + assert_eq!(parent.result, Some(outcome.result)); + assert_eq!( + count_prefix( + &store, + MetadataFamily::Operation, + &super::super::operation_prefix(root(), OperationKind::ActivePublish) + ), + 0 + ); + assert!(service + .begin_publish(BeginPublishRequest { + context: publication_context(&store, &mut counter), + operation: logical_append_child(1) + }) + .is_err()); + assert_eq!(read_append_parent(&service, &store, &mut counter), parent); + } + + #[test] + fn append_known_path_conflict_proves_absence_before_cleanup_without_lease_expiry() { + let mut counter = 1; + let store = ready_store(&mut counter); + let service = PublicationService::new(&store); + let child = begin_operation(&service, &store, &mut counter, logical_append_child(0)); + let child = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + transition: PublishTransition::BeginFinalization, + }) + .unwrap() + .operation; + // A valid frozen claim cannot be cancelled by this recovery command. + assert!(service + .fail_append_conflict(publication_context(&store, &mut counter), child.clone()) + .is_err()); + publish_full( + &service, + &store, + &mut counter, + operation_id(94_000), + revision(94_000), + child.path.clone(), + PublishClaim::CreateOnly, + 1, + ); + let aborted = service + .fail_append_conflict(publication_context(&store, &mut counter), child.clone()) + .unwrap() + .operation; + assert_eq!(aborted.phase, PublishPhase::Aborting); + assert!(aborted.publication_absence_proof.is_some()); + assert!(service + .finalize_publish(FinalizePublishRequest { + context: publication_context(&store, &mut counter), + artifact: published_artifact(&child), + expected_operation: child, + dependency_owner_revision_ids: Vec::new(), + }) + .is_err()); + assert!(read_append_parent(&service, &store, &mut counter) + .result + .is_none()); + } + + #[test] + fn logical_append_finalizing_and_atomic_receipt_survive_holt_reopen() { + let directory = tempdir().unwrap(); + let database_path = directory.path().join("append-finalizing-recovery"); + let mut counter = 1; + let store = ready_file_store(&database_path, &mut counter); + let service = PublicationService::new(&store); + let child = begin_operation(&service, &store, &mut counter, logical_append_child(0)); + let finalizing = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + transition: PublishTransition::BeginFinalization, + }) + .unwrap() + .operation; + assert!(read_append_parent(&service, &store, &mut counter) + .result + .is_none()); + drop(store); + + let store = crate::workspace::test_support::open_file(&database_path, shard()).unwrap(); + let service = PublicationService::new(&store); + let parent = read_append_parent(&service, &store, &mut counter); + assert!(parent.matches_publication(&finalizing)); + service + .verify_append_publication_state(publication_context(&store, &mut counter)) + .unwrap(); + let mut child = service + .take_over_orphaned_publish(TakeOverOrphanedPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: finalizing.clone(), + observed_now_ms: 1_000_000, + maximum_clock_skew_ms: 0, + terminal_error: PublishTerminalError { + kind: PublishTerminalErrorKind::ActivityLeaseExpired, + message: "caller exited at Finalizing".to_owned(), + evidence_digest: None, + }, + }) + .unwrap() + .operation; + assert!(child.publication_absence_proof.is_some()); + for transition in [ + PublishTransition::BeginCleaning, + PublishTransition::FinishCleanup, + ] { + child = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + transition, + }) + .unwrap() + .operation; + } + let mut successor = logical_append_child(1); + successor.activity_deadline_ms = 2_000_000; + seal_publish_operation(&mut successor); + let successor = begin_operation(&service, &store, &mut counter, successor); + let successor = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: successor, + transition: PublishTransition::BeginFinalization, + }) + .unwrap() + .operation; + let projection = TypedProjection::new( + [( + QueryFieldId::new("append.kind").unwrap(), + QueryScalar::String("recovered".to_owned()), + )] + .into_iter() + .collect(), + ) + .unwrap(); + let mut artifact = published_artifact(&successor); + artifact.typed_index_projection = projection.encode().unwrap(); + let published = service + .finalize_publish(FinalizePublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: successor, + artifact, + dependency_owner_revision_ids: Vec::new(), + }) + .unwrap(); + let receipt = read_append_parent(&service, &store, &mut counter); + assert_eq!(receipt.result.as_ref(), Some(&published.result)); + drop(store); + + let store = crate::workspace::test_support::open_file(&database_path, shard()).unwrap(); + let service = PublicationService::new(&store); + assert_eq!(read_append_parent(&service, &store, &mut counter), receipt); + service + .verify_append_publication_state(publication_context(&store, &mut counter)) + .unwrap(); + let version = store.current_read_version().unwrap(); + let visible = get_visible_path_at( + &store, + RootReadContext::current(&store, root(), placement(), owner()).unwrap(), + &workbench(), + &receipt.path, + ) + .unwrap() + .unwrap(); + assert_eq!(visible.artifact_revision_id, receipt.artifact_revision_id); + assert_eq!(visible.generation, published.result.path_generation); + let revision_record = ArtifactRevisionRecord::decode( + &payload_at( + &store, + MetadataFamily::ArtifactRevision, + &artifact_revision_key(root(), receipt.artifact_revision_id), + version, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(revision_record.strong_reference_count, 1); + assert!(payload_at( + &store, + MetadataFamily::RevisionRef, + &path_revision_ref_key( + root(), + receipt.workspace_incarnation_id, + &receipt.path, + receipt.artifact_revision_id + ), + version + ) + .is_some()); + for (key, value) in secondary_index_rows( + root(), + receipt.workspace_incarnation_id, + &receipt.path, + published.result.path_generation, + &projection, + ) + .unwrap() + { + assert_eq!( + payload_at(&store, MetadataFamily::SecondaryIndex, &key, version), + Some(value) + ); + } + assert_eq!( + count_prefix( + &store, + MetadataFamily::Operation, + &super::super::operation_prefix(root(), OperationKind::ActivePublish) + ), + 0 + ); + assert!(service + .finalize_publish(FinalizePublishRequest { + context: publication_context(&store, &mut counter), + artifact: published_artifact(&finalizing), + expected_operation: finalizing, + dependency_owner_revision_ids: Vec::new(), + }) + .is_err()); + } + + #[test] + fn append_publication_fsck_rejects_missing_or_orphan_activity_markers_without_repair() { + for orphan in [false, true] { + let mut counter = 1; + let store = ready_store(&mut counter); + let service = PublicationService::new(&store); + let child = begin_operation(&service, &store, &mut counter, logical_append_child(0)); + service + .verify_append_publication_state(publication_context(&store, &mut counter)) + .unwrap(); + let key = operation_key( + root(), + OperationKind::ActivePublish, + if orphan { + operation_id(99_000) + } else { + child.operation_id + }, + ); + let mut command = fence_command( + &store, + next_request(&mut counter), + RootFenceAction::RequireActive, + ); + command.predicates.push(CommandPredicate::Value { + family: MetadataFamily::Operation, + key: key.clone(), + expected: (!orphan).then(|| vec![1]), + }); + // Only deleting an existing marker replaces a current value. + // An absent-key insertion must not request a history projection. + if !orphan { + command.history_projection.push(HistoryProjection { + family: MetadataFamily::Operation, + key: key.clone(), + }); + } + command.mutations.push(if orphan { + CommandMutation::Put { + family: MetadataFamily::Operation, + key, + value: vec![1], + } + } else { + CommandMutation::Delete { + family: MetadataFamily::Operation, + key, + } + }); + store.execute(&command.seal()).unwrap(); + let before = store.current_read_version().unwrap(); + assert!(matches!( + service.verify_append_publication_state(publication_context(&store, &mut counter)), + Err(PublicationError::Meta(MetaError::CorruptRecord { .. })) + )); + assert_eq!(store.current_read_version().unwrap(), before); + } + } + + #[test] + fn stable_append_identity_binds_all_256_intent_bits_and_cannot_drop_binding() { + let mut counter = 1; + let store = ready_store(&mut counter); + let service = PublicationService::new(&store); + let revision_id = revision(91_090); + let staged = staged_rows(revision_id, 1); + let manifest = manifest_rows(&staged); + let mut initial = publish_operation( + operation_id(91_090), + revision_id, + path("outputs/intent.bin"), + PublishClaim::CreateOnly, + &staged, + &manifest, + ); + initial.append_intent_digest = Some([0x61; SHA256_BYTES]); + initial.append_attempt = Some(super::super::AppendAttemptBinding { + operation_id: operation_id(91_091), + attempt: 0, + }); + seal_publish_operation(&mut initial); + let begun = begin_operation(&service, &store, &mut counter, initial.clone()); + assert_eq!(begun.append_intent_digest, initial.append_intent_digest); + let mut changed_digest = [0x61; SHA256_BYTES]; + changed_digest[SHA256_BYTES - 1] = 0x62; + for intent in [Some(changed_digest), None] { + let mut changed = initial.clone(); + changed.append_intent_digest = intent; + if intent.is_none() { + changed.append_attempt = None; + } + seal_publish_operation(&mut changed); + assert_ne!(changed.identity_digest, initial.identity_digest); + assert_ne!(changed.initialization_digest, initial.initialization_digest); + assert_eq!( + service.begin_publish(BeginPublishRequest { + context: publication_context(&store, &mut counter), + operation: changed, + }), + Err(PublicationError::OperationInputMismatch) + ); + } + let replay = service + .begin_publish(BeginPublishRequest { + context: publication_context(&store, &mut counter), + operation: initial, + }) + .unwrap(); + assert!(replay.replayed); + assert_eq!(replay.operation, begun); + } + #[test] fn successor_epoch_cannot_borrow_visible_or_commit_staging_operation_identity() { for (index, authority) in [ @@ -6806,7 +7980,7 @@ mod tests { assert_eq!(replaced.result.path_generation, Generation::new(2).unwrap()); let replace_bytes = capture.with_last_commit(crate::workspace::test_support::transaction_bytes); - assert_eq!(replace_bytes, 11_799_442); + assert_eq!(replace_bytes, 11_799_783); let removed = remove_path( &store, @@ -6856,6 +8030,7 @@ mod tests { PublishClaim::CreateOnly, &dependencies, projection.clone(), + None, ); let removed_path = maximum_path(101); publish_with_dependencies_and_projection( @@ -6868,6 +8043,7 @@ mod tests { PublishClaim::CreateOnly, &dependencies, projection, + None, ); drop(store); @@ -6885,6 +8061,7 @@ mod tests { }, &[], TypedProjection::empty(), + None, ); assert_eq!(replaced.result.path_generation, Generation::new(2).unwrap()); let removed = remove_path( @@ -6909,6 +8086,85 @@ mod tests { assert_eq!(removed.removed_artifact_revision_id, revision(40_001)); } + #[test] + fn logical_append_parent_and_active_index_fit_maximum_path_fields_and_dependencies() { + let mut counter = 1; + let (store, capture) = ready_capturing_store(&mut counter); + let service = PublicationService::new(&store); + let dependencies = seed_maximum_dependencies(&service, &store, &mut counter); + let target = maximum_path(222); + let before = budget_projection_with_prefix("oldset", 400); + let after = budget_projection_with_prefix("newset", 400); + assert_eq!(before.fields().len(), 60); + assert_eq!(after.fields().len(), 60); + assert_eq!(dependencies.len(), 64); + publish_with_dependencies_and_projection( + &service, + &store, + &mut counter, + operation_id(95_000), + revision(95_000), + target.clone(), + PublishClaim::CreateOnly, + &dependencies, + before, + None, + ); + let logical_id = operation_id(95_002); + let published = publish_with_dependencies_and_projection( + &service, + &store, + &mut counter, + operation_id(95_001), + revision(95_001), + target.clone(), + PublishClaim::Append { + expected_generation: Generation::new(1).unwrap(), + base_revision_id: revision(95_000), + append_offset: 1, + }, + &dependencies, + after, + Some(super::super::AppendAttemptBinding { + operation_id: logical_id, + attempt: 0, + }), + ); + assert_eq!(published.result.path_generation.get(), 2); + let bytes = capture.with_last_commit(crate::workspace::test_support::transaction_bytes); + assert!( + bytes < super::super::store_limits().max_transaction_bytes, + "complete derived transaction uses {bytes} bytes" + ); + capture.with_last_commit(|transaction| { + assert!(transaction.checks.len() <= super::super::store_limits().max_checks); + assert!(transaction.mutations.len() <= super::super::store_limits().max_mutations); + }); + let parent = AppendOperationRecord::decode( + &payload_at( + &store, + MetadataFamily::Operation, + &operation_key(root(), OperationKind::Append, logical_id), + store.current_read_version().unwrap(), + ) + .unwrap(), + ) + .unwrap(); + assert!(parent.matches_publication(&published.operation)); + assert_eq!(parent.result, Some(published.result)); + assert_eq!( + count_prefix( + &store, + MetadataFamily::Operation, + &super::super::operation_prefix(root(), OperationKind::ActivePublish) + ), + 0 + ); + service + .verify_append_publication_state(publication_context(&store, &mut counter)) + .unwrap(); + } + #[test] fn maximum_event_sized_disjoint_sixty_field_republish_pins_transaction_bytes() { let mut counter = 1; @@ -6985,7 +8241,7 @@ mod tests { assert_eq!( capture.with_last_commit(crate::workspace::test_support::transaction_bytes), - 9_860_715 + 9_861_056 ); } diff --git a/crates/nokv-meta/src/workspace/publication_records.rs b/crates/nokv-meta/src/workspace/publication_records.rs index 4e5969fe7..7584869cd 100644 --- a/crates/nokv-meta/src/workspace/publication_records.rs +++ b/crates/nokv-meta/src/workspace/publication_records.rs @@ -55,12 +55,13 @@ pub struct WorkspaceIncarnationClaimRecord { pub workbench_id: WorkbenchId, } -/// In-flight exclusive publish ownership of one artifact revision identity. +/// Exclusive publish ownership of one artifact revision identity. /// /// Created atomically by `begin_publish` and deleted in the same command that -/// publishes the revision or finishes the owning operation's cleanup, so two -/// operations can never simultaneously own the revision's permanent object -/// keys. +/// publishes the revision or finishes generic publication cleanup. Failed +/// append cleanup retains this claim permanently to reserve its sealed object +/// keys, including after a successor succeeds. Two operations can never own +/// the same revision's permanent object keys. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ArtifactRevisionClaimRecord { pub operation_id: OperationId, diff --git a/crates/nokv-meta/src/workspace/publish_operation_records.rs b/crates/nokv-meta/src/workspace/publish_operation_records.rs index 2af2067e5..e3b5fc00a 100644 --- a/crates/nokv-meta/src/workspace/publish_operation_records.rs +++ b/crates/nokv-meta/src/workspace/publish_operation_records.rs @@ -19,7 +19,7 @@ use nokv_types::{ }; /// Durable value format for publication-owned payloads. -pub const PUBLISH_VALUE_FORMAT_VERSION: u8 = 5; +pub const PUBLISH_VALUE_FORMAT_VERSION: u8 = 6; /// Hard safety bound for one publish operation's staged-object ledger. pub const MAX_STAGED_OBJECTS: u32 = 1_048_576; @@ -140,6 +140,7 @@ pub struct PublishTerminalError { /// and an exact-identity initialization mismatch remain distinguishable. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PublishOperationRecord { + pub append_attempt: Option, /// Full caller intent commitment for one stable append publication attempt. pub append_intent_digest: Option<[u8; SHA256_BYTES]>, pub operation_id: OperationId, @@ -196,6 +197,122 @@ pub struct PublishOperationRecord { pub terminal_error: Option, } +/// A publication attempt belongs permanently to one logical append. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AppendAttemptBinding { + pub operation_id: OperationId, + pub attempt: u64, +} + +/// Root-scoped logical append. Its current publication always exists and its +/// receipt is installed atomically with the publication and visible path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AppendOperationRecord { + pub operation_id: OperationId, + pub intent_digest: [u8; SHA256_BYTES], + pub workbench_id: WorkbenchId, + pub workspace_incarnation_id: WorkspaceIncarnationId, + pub path: NormalizedRelativePath, + pub attempt: u64, + pub publication_operation_id: OperationId, + pub artifact_revision_id: ArtifactRevisionId, + pub result: Option, +} + +impl AppendOperationRecord { + pub fn matches_publication(&self, publication: &PublishOperationRecord) -> bool { + publication.append_attempt + == Some(AppendAttemptBinding { + operation_id: self.operation_id, + attempt: self.attempt, + }) + && publication.append_intent_digest == Some(self.intent_digest) + && publication.operation_id == self.publication_operation_id + && publication.artifact_revision_id == self.artifact_revision_id + && publication.workbench_id == self.workbench_id + && publication.workspace_incarnation_id == self.workspace_incarnation_id + && publication.path == self.path + && publication.authority == PublishAuthority::Visible + } + + pub fn encode(&self) -> Result, PublishRecordError> { + if self.operation_id == self.publication_operation_id { + return Err(PublishRecordError::InvalidPhasePayload { + phase: PublishPhase::Uploading, + reason: "logical and publication identities must differ", + }); + } + if let Some(result) = &self.result { + validate_publish_result(result)?; + } + let mut encoded = vec![1]; + encoded.extend_from_slice(self.operation_id.as_bytes()); + encoded.extend_from_slice(&self.intent_digest); + push_bytes(&mut encoded, "workbench_id", self.workbench_id.as_bytes())?; + encoded.extend_from_slice(self.workspace_incarnation_id.as_bytes()); + push_bytes(&mut encoded, "path", self.path.as_str().as_bytes())?; + encoded.extend_from_slice(&self.attempt.to_be_bytes()); + encoded.extend_from_slice(self.publication_operation_id.as_bytes()); + encoded.extend_from_slice(self.artifact_revision_id.as_bytes()); + push_optional_result(&mut encoded, self.result.as_ref())?; + require_record_size( + "AppendOperationRecord", + encoded.len(), + MAX_PUBLISH_OPERATION_RECORD_BYTES, + )?; + Ok(encoded) + } + + pub fn decode(encoded: &[u8]) -> Result { + require_record_size( + "AppendOperationRecord", + encoded.len(), + MAX_PUBLISH_OPERATION_RECORD_BYTES, + )?; + let mut decoder = Decoder::new(encoded); + let actual = decoder.u8("append.value_format_version")?; + if actual != 1 { + return Err(PublishRecordError::UnsupportedValueVersion { + actual, + expected: 1, + }); + } + let operation_id = OperationId::from_bytes(decoder.fixed("operation_id")?); + let intent_digest = decoder.fixed("intent_digest")?; + let workbench_id = WorkbenchId::new( + decoder.string("workbench_id", WorkbenchId::MAX_BYTES)?, + ) + .map_err(|error| PublishRecordError::InvalidWorkbenchId { + reason: error.to_string(), + })?; + let workspace_incarnation_id = + WorkspaceIncarnationId::from_bytes(decoder.fixed("workspace_incarnation_id")?); + let path = + NormalizedRelativePath::new(decoder.string("path", NormalizedRelativePath::MAX_BYTES)?) + .map_err(|error| PublishRecordError::InvalidPath { + reason: error.to_string(), + })?; + let record = Self { + operation_id, + intent_digest, + workbench_id, + workspace_incarnation_id, + path, + attempt: decoder.u64("attempt")?, + publication_operation_id: OperationId::from_bytes( + decoder.fixed("publication_operation_id")?, + ), + artifact_revision_id: ArtifactRevisionId::from_bytes( + decoder.fixed("artifact_revision_id")?, + ), + result: decode_optional_result(&mut decoder)?, + }; + decoder.finish()?; + record.encode()?; + Ok(record) + } +} + /// Last ordered manifest key included in the durable rolling closure. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct ManifestPosition { @@ -447,6 +564,21 @@ impl std::error::Error for PublishRecordError {} impl PublishOperationRecord { pub fn validate(&self) -> Result<(), PublishRecordError> { + if self.append_attempt.is_some() != self.append_intent_digest.is_some() { + return Err(PublishRecordError::InvalidPhasePayload { + phase: self.phase, + reason: "append attempt and intent must be present together", + }); + } + if self + .append_attempt + .is_some_and(|binding| binding.operation_id == self.operation_id) + { + return Err(PublishRecordError::InvalidPhasePayload { + phase: self.phase, + reason: "logical and publication identities must differ", + }); + } if self.append_intent_digest.is_some() && (!matches!(self.authority, PublishAuthority::Visible) || matches!(self.claim, PublishClaim::ReplaceOnly { .. })) @@ -699,6 +831,14 @@ impl PublishOperationRecord { encoded.extend_from_slice(&self.identity_digest); encoded.extend_from_slice(&self.initialization_digest); push_optional_fixed(&mut encoded, &self.append_intent_digest); + match self.append_attempt { + None => encoded.push(0), + Some(binding) => { + encoded.push(1); + encoded.extend_from_slice(binding.operation_id.as_bytes()); + encoded.extend_from_slice(&binding.attempt.to_be_bytes()); + } + } encoded.extend_from_slice(&self.initiating_owner_epoch.get().to_be_bytes()); encoded.extend_from_slice(&self.activity_deadline_ms.to_be_bytes()); match self.authority { @@ -767,6 +907,21 @@ impl PublishOperationRecord { let identity_digest = decoder.fixed("identity_digest")?; let initialization_digest = decoder.fixed("initialization_digest")?; let append_intent_digest = decoder.optional_fixed("append_intent_digest")?; + let append_attempt = match decoder.u8("append_attempt")? { + 0 => None, + 1 => Some(AppendAttemptBinding { + operation_id: OperationId::from_bytes( + decoder.fixed("append_attempt.operation_id")?, + ), + attempt: decoder.u64("append_attempt.attempt")?, + }), + value => { + return Err(PublishRecordError::InvalidOptionalTag { + field: "append_attempt", + value, + }) + } + }; let initiating_owner_epoch = OwnerEpoch::new(decoder.u64("initiating_owner_epoch")?) .map_err(|_| PublishRecordError::ZeroScalar { field: "initiating_owner_epoch", @@ -851,6 +1006,7 @@ impl PublishOperationRecord { operation_id, identity_digest, initialization_digest, + append_attempt, append_intent_digest, initiating_owner_epoch, activity_deadline_ms, @@ -1339,11 +1495,13 @@ fn validate_staged_state_pair( | StagedProviderState::Uploaded | StagedProviderState::AbortPending, StagedCleanupState::DeletePending - ) | (StagedProviderState::Aborted, StagedCleanupState::Deleted) - | ( - StagedProviderState::Ambiguous, - StagedCleanupState::Quarantined - ) + ) | ( + StagedProviderState::Aborted, + StagedCleanupState::Deleted | StagedCleanupState::Sealed + ) | ( + StagedProviderState::Ambiguous, + StagedCleanupState::Quarantined + ) ); if valid { Ok(()) @@ -1811,6 +1969,7 @@ mod tests { fn uploading_operation() -> PublishOperationRecord { PublishOperationRecord { + append_attempt: None, append_intent_digest: None, operation_id: OperationId::from_bytes([0x10; 16]), identity_digest: [0x11; SHA256_BYTES], @@ -1900,7 +2059,7 @@ mod tests { &[0x10; 16], &[0x11; SHA256_BYTES], &[0x12; SHA256_BYTES], - &[0], + &[0, 0], &5_u64.to_be_bytes(), &50_000_u64.to_be_bytes(), &[1], @@ -1954,6 +2113,10 @@ mod tests { let mut digest = [0x61; SHA256_BYTES]; digest[SHA256_BYTES - 1] = 0x62; record.append_intent_digest = Some(digest); + record.append_attempt = Some(AppendAttemptBinding { + operation_id: OperationId::from_bytes([0x63; 16]), + attempt: 0, + }); let mut encoded = record.encode().unwrap(); assert_eq!(PublishOperationRecord::decode(&encoded).unwrap(), record); encoded[0] = PUBLISH_VALUE_FORMAT_VERSION - 1; @@ -2409,6 +2572,7 @@ mod tests { + SHA256_BYTES + SHA256_BYTES + 1 // absent append_intent_digest tag + + 1 // absent append_attempt tag + 8 + 8 + 1 @@ -2474,6 +2638,7 @@ mod tests { + SHA256_BYTES + SHA256_BYTES + 1 // absent append_intent_digest tag + + 1 // absent append_attempt tag + 8 + 8 + 1 diff --git a/crates/nokv-meta/src/workspace/recovery.rs b/crates/nokv-meta/src/workspace/recovery.rs index d0dabafc4..335b57032 100644 --- a/crates/nokv-meta/src/workspace/recovery.rs +++ b/crates/nokv-meta/src/workspace/recovery.rs @@ -15,14 +15,13 @@ use sha2::{Digest, Sha256}; use super::codec::SCHEMA_ID; use super::engine::{ CommandMutation, CommandPredicate, EventProjection, HistoryProjection, MetadataCommand, - RootFenceAction, + RootFenceAction, MAX_COMMAND_ITEMS, }; use super::keyspace::MetadataFamily; pub const RECOVERY_OUTBOX_VALUE_FORMAT_VERSION: u8 = 3; const LEGACY_RECOVERY_OUTBOX_VALUE_FORMAT_VERSION: u8 = 2; pub const RECOVERY_CHAIN_DIGEST_BYTES: usize = 32; -const MAX_ITEMS: usize = 256; pub(crate) const MAX_RECOVERY_BYTES: usize = 64 * 1024 * 1024; pub const MAX_RECOVERY_SEGMENT_BYTES: usize = MAX_RECOVERY_BYTES; pub const MAX_RECOVERY_SEGMENT_RECORDS: usize = 1024; @@ -1488,11 +1487,11 @@ fn put_count( field: &'static str, count: usize, ) -> Result<(), RecoveryCodecError> { - if count > MAX_ITEMS { + if count > MAX_COMMAND_ITEMS { return Err(RecoveryCodecError::LengthBound { field, length: count, - max: MAX_ITEMS, + max: MAX_COMMAND_ITEMS, }); } bytes.extend_from_slice(&u32_len(field, count)?.to_be_bytes()); @@ -1566,11 +1565,11 @@ impl<'a> Decoder<'a> { fn count(&mut self, field: &'static str) -> Result { let count = self.u32(field)? as usize; - if count > MAX_ITEMS { + if count > MAX_COMMAND_ITEMS { return Err(RecoveryCodecError::LengthBound { field, length: count, - max: MAX_ITEMS, + max: MAX_COMMAND_ITEMS, }); } Ok(count) @@ -1706,6 +1705,65 @@ mod tests { .seal() } + #[test] + fn recovery_command_codec_shares_engine_item_limit_in_both_directions() { + let mut command = command(); + command.predicates = vec![command.predicates[0].clone(); MAX_COMMAND_ITEMS]; + command.mutations = vec![command.mutations[0].clone(); MAX_COMMAND_ITEMS]; + command.history_projection = vec![command.history_projection[0].clone(); MAX_COMMAND_ITEMS]; + command.event_projection = vec![command.event_projection[0].clone(); MAX_COMMAND_ITEMS]; + let mutation = RecoveryMutationV1::MetadataCommand { + command: Box::new(command.clone().seal()), + lease_deadline_ms: None, + }; + let encoded = mutation.encode_canonical().unwrap(); + assert_eq!( + RecoveryMutationV1::decode_canonical(&encoded).unwrap(), + mutation + ); + let fields = [ + "predicates", + "mutations", + "history_projection", + "event_projection", + ]; + for field in fields { + let mut oversized = command.clone(); + match field { + "predicates" => oversized.predicates.push(oversized.predicates[0].clone()), + "mutations" => oversized.mutations.push(oversized.mutations[0].clone()), + "history_projection" => oversized + .history_projection + .push(oversized.history_projection[0].clone()), + "event_projection" => oversized + .event_projection + .push(oversized.event_projection[0].clone()), + _ => unreachable!(), + } + let mutation = RecoveryMutationV1::MetadataCommand { + command: Box::new(oversized.seal()), + lease_deadline_ms: None, + }; + assert_eq!( + mutation.encode_canonical(), + Err(RecoveryCodecError::LengthBound { + field, + length: MAX_COMMAND_ITEMS + 1, + max: MAX_COMMAND_ITEMS, + }) + ); + } + let oversized_count = ((MAX_COMMAND_ITEMS + 1) as u32).to_be_bytes(); + assert_eq!( + Decoder::new(&oversized_count).count("predicates"), + Err(RecoveryCodecError::LengthBound { + field: "predicates", + length: MAX_COMMAND_ITEMS + 1, + max: MAX_COMMAND_ITEMS, + }) + ); + } + #[test] fn recovery_record_has_frozen_golden_and_strict_codec() { let mutation = RecoveryMutationV1::MetadataCommand { diff --git a/crates/nokv-meta/src/workspace/restore.rs b/crates/nokv-meta/src/workspace/restore.rs index 0cc36c1fc..fc353f0ed 100644 --- a/crates/nokv-meta/src/workspace/restore.rs +++ b/crates/nokv-meta/src/workspace/restore.rs @@ -46,7 +46,7 @@ use super::commit_records::{ }; use super::engine::{ CommandFit, CommandMutation, CommandPredicate, EventProjection, HistoryProjection, MetaError, - MetaShard, MetadataCommand, MetadataScanItem, RootFenceAction, + MetaShard, MetadataCommand, MetadataScanItem, RootFenceAction, MAX_COMMAND_ITEMS, }; use super::event_projection::change_event_projection; use super::generic_index::{ @@ -91,7 +91,6 @@ pub const MAX_RESTORE_BATCH_MEMBERS: usize = 48; /// Exact Workbench initialization path published before publication. pub const RESTORE_MANIFEST_PATH: &str = "metadata/restore_manifest.json"; const RESTORE_OUTCOME_FORMAT: u8 = 1; -const MAX_COMMAND_ITEMS: usize = 256; const CAPACITY_EXCEEDED_MESSAGE: &str = "restore source member exceeds the serving metadata transaction budget"; const CLEANUP_CAPACITY_EXCEEDED_MESSAGE: &str = @@ -1189,7 +1188,11 @@ pub fn begin_restore( // Claim the root-scoped public operation identity in the same command as // the hidden destination and source holds. A competing kind cannot make // the eventual receipt unqueryable. - for kind in [OperationKind::Publish, OperationKind::BuildCommit] { + for kind in [ + OperationKind::Publish, + OperationKind::BuildCommit, + OperationKind::Append, + ] { plan.assert_value( MetadataFamily::Operation, super::codec::operation_key(context.root_id, kind, operation_id), @@ -10766,6 +10769,7 @@ mod tests { manifest: &[ManifestRowInput], ) -> PublishOperationRecord { let mut operation = PublishOperationRecord { + append_attempt: None, append_intent_digest: None, operation_id, identity_digest: [0; SHA256_BYTES], diff --git a/crates/nokv-object/src/admission.rs b/crates/nokv-object/src/admission.rs index 24bbf2c6c..2914cea17 100644 --- a/crates/nokv-object/src/admission.rs +++ b/crates/nokv-object/src/admission.rs @@ -29,6 +29,7 @@ pub enum AdmittedCreateMode { pub enum ProviderAdmissionCapability { AtomicCreateIfAbsent, ExactRangeRead, + ImmutableSealing, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -43,6 +44,7 @@ pub enum ProviderAdmissionError { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ProviderAdmissionProfile { max_verified_object_bytes: usize, + append_sealing: bool, } #[derive(Debug, PartialEq, Eq)] @@ -51,6 +53,7 @@ pub struct ProviderAdmissionReceipt { admitted_create_mode: AdmittedCreateMode, max_verified_object_bytes: usize, range_read_verified: bool, + append_sealing_verified: bool, handle_identity: ProviderHandleIdentity, } @@ -72,9 +75,16 @@ impl ProviderAdmissionProfile { } Ok(Self { max_verified_object_bytes, + append_sealing: false, }) } + /// Additionally verify permanent empty-object fences for abandoned append attempts. + pub const fn with_append_sealing(mut self) -> Self { + self.append_sealing = true; + self + } + pub const fn max_verified_object_bytes(self) -> usize { self.max_verified_object_bytes } @@ -97,6 +107,10 @@ impl ProviderAdmissionReceipt { self.range_read_verified } + pub const fn append_sealing_verified(&self) -> bool { + self.append_sealing_verified + } + pub fn is_bound_to_store(&self, store: &dyn ArtifactObjectStore) -> bool { self.contract_version == PROVIDER_ADMISSION_CONTRACT_VERSION && self.admitted_create_mode == AdmittedCreateMode::SinglePutIfAbsent @@ -110,6 +124,10 @@ impl ProviderAdmissionReceipt { && block_size <= self.max_verified_object_bytes } + pub fn admits_append_store(&self, store: &dyn ArtifactObjectStore, block_size: usize) -> bool { + self.append_sealing_verified && self.admits_store(store, block_size) + } + pub(crate) const fn trusted_in_process( handle_identity: ProviderHandleIdentity, max_verified_object_bytes: usize, @@ -119,9 +137,15 @@ impl ProviderAdmissionReceipt { admitted_create_mode: AdmittedCreateMode::SinglePutIfAbsent, max_verified_object_bytes, range_read_verified: true, + append_sealing_verified: false, handle_identity, } } + + pub(crate) const fn with_trusted_append_sealing(mut self) -> Self { + self.append_sealing_verified = true; + self + } } impl ProviderHandleIdentity { @@ -166,6 +190,14 @@ impl fmt::Debug for ProviderHandleIdentity { } impl ProviderAdmissionError { + pub const fn code(self) -> &'static str { + match self { + Self::Rejected { .. } => "ProviderAdmissionRejected", + Self::Unavailable => "ProviderAdmissionUnavailable", + Self::Inconclusive => "ProviderAdmissionInconclusive", + } + } + pub const fn retryable(self) -> bool { matches!(self, Self::Unavailable) } @@ -194,6 +226,7 @@ impl ProviderAdmissionCapability { match self { Self::AtomicCreateIfAbsent => "atomic create-if-absent", Self::ExactRangeRead => "exact range-read", + Self::ImmutableSealing => "permanent immutable sealing", } } } @@ -202,6 +235,9 @@ impl ProviderAdmissionCapability { /// /// The receipt qualifies single-PUT objects no larger than the probe payload. /// Multipart completion is deliberately not qualified by this contract version. +/// Optional append sealing verifies empty fences for absent and existing objects, +/// including a concurrent create and a later rejected payload write. These +/// fences require permanent key reservations and no external deletion/expiration. pub fn admit_artifact_provider( store: &(dyn ArtifactObjectStore + Sync), profile: ProviderAdmissionProfile, @@ -283,11 +319,16 @@ pub fn admit_artifact_provider( Err(error) => return Err(classify_provider_error(&error)), } + if profile.append_sealing { + verify_append_sealing(store, &nonce, &a)?; + } + Ok(ProviderAdmissionReceipt { contract_version: PROVIDER_ADMISSION_CONTRACT_VERSION, admitted_create_mode: AdmittedCreateMode::SinglePutIfAbsent, max_verified_object_bytes: profile.max_verified_object_bytes, range_read_verified: true, + append_sealing_verified: profile.append_sealing, handle_identity: store.provider_handle_identity(), }) })(); @@ -300,6 +341,92 @@ pub fn admit_artifact_provider( result } +fn verify_append_sealing( + store: &(dyn ArtifactObjectStore + Sync), + nonce: &str, + payload: &[u8], +) -> Result<(), ProviderAdmissionError> { + let empty_key = probe_key(nonce, "seal-empty")?; + let payload_key = probe_key(nonce, "seal-payload")?; + let race_key = probe_key(nonce, "seal-race")?; + let result = (|| { + observe_seal(store, &empty_key)?; + verify_sealed_key(store, &empty_key, payload)?; + + if !observation_confirms_own_bytes(observe_create(store, &payload_key, payload)?) { + return Err(sealing_rejected()); + } + observe_seal(store, &payload_key)?; + verify_sealed_key(store, &payload_key, payload)?; + + let barrier = Arc::new(Barrier::new(3)); + let (create, seal) = std::thread::scope(|scope| { + let key = &race_key; + let create_barrier = Arc::clone(&barrier); + let create = scope.spawn(move || { + create_barrier.wait(); + observe_create(store, key, payload) + }); + let seal_barrier = Arc::clone(&barrier); + let seal = scope.spawn(move || { + seal_barrier.wait(); + observe_seal(store, key) + }); + barrier.wait(); + (create.join(), seal.join()) + }); + create.map_err(|_| ProviderAdmissionError::Inconclusive)??; + seal.map_err(|_| ProviderAdmissionError::Inconclusive)??; + verify_sealed_key(store, &race_key, payload) + })(); + + // Reserved system probe keys have no metadata claim or publication caller. + // Every probe worker has joined before these disposable keys are removed. + for key in [&empty_key, &payload_key, &race_key] { + let _ = store.delete(key); + } + result +} + +fn sealing_rejected() -> ProviderAdmissionError { + ProviderAdmissionError::Rejected { + capability: ProviderAdmissionCapability::ImmutableSealing, + } +} + +fn observe_seal( + store: &(dyn ArtifactObjectStore + Sync), + key: &ObjectKey, +) -> Result<(), ProviderAdmissionError> { + let ambiguous = match store.seal_immutable(key) { + Ok(_) => false, + Err(ObjectError::SealAmbiguous { .. }) => true, + Err(error) => return Err(classify_provider_error(&error)), + }; + match store.head(key) { + Ok(Some(info)) if info.size == 0 => Ok(()), + Ok(_) if ambiguous => Err(ProviderAdmissionError::Inconclusive), + Ok(_) => Err(sealing_rejected()), + Err(error) => Err(classify_provider_error(&error)), + } +} + +fn verify_sealed_key( + store: &(dyn ArtifactObjectStore + Sync), + key: &ObjectKey, + payload: &[u8], +) -> Result<(), ProviderAdmissionError> { + if !observation_sees_foreign_bytes(observe_create(store, key, payload)?) { + return Err(sealing_rejected()); + } + observe_seal(store, key)?; + match store.read(key, None) { + Ok(bytes) if bytes.is_empty() => Ok(()), + Ok(_) => Err(sealing_rejected()), + Err(error) => Err(classify_provider_error(&error)), + } +} + fn concurrent_create_has_one_winner( store: &(dyn ArtifactObjectStore + Sync), key: &ObjectKey, @@ -385,6 +512,7 @@ fn classify_provider_error(error: &ObjectError) -> ProviderAdmissionError { ObjectError::AtomicCreateUnsupported => ProviderAdmissionError::Rejected { capability: ProviderAdmissionCapability::AtomicCreateIfAbsent, }, + ObjectError::ImmutableSealingUnsupported => sealing_rejected(), ObjectError::Backend { retryable: true, .. } => ProviderAdmissionError::Unavailable, diff --git a/crates/nokv-object/src/lib.rs b/crates/nokv-object/src/lib.rs index 60bbe13fc..841711628 100644 --- a/crates/nokv-object/src/lib.rs +++ b/crates/nokv-object/src/lib.rs @@ -6,6 +6,15 @@ //! and in-memory durable stores, and provides local soft caches. Namespace //! visibility, revision reachability, metadata transactions, and GC policy live //! in `nokv-meta`. +//! +//! Failed stable-append attempts additionally use permanent empty-object seals. +//! Their metadata authority must fence publication and reserve revision keys +//! forever before calling `seal_immutable`. S3 sealing uses conditional PUTs, +//! never DELETE, so late immutable uploads cannot restore payload bytes. The +//! append-specific provider admission profile verifies this stronger boundary; +//! ordinary publication admission does not imply sealing support. Seal keys +//! must be excluded from external deletion and expiration. In versioned buckets, +//! sealing changes the current object only; it does not remove older versions. mod admission; mod artifact; @@ -59,7 +68,7 @@ pub use recovery_log_segment::{ pub use store::{ ArtifactObjectStore, ArtifactStoreCapabilities, ImmutableCreateOutcome, MemoryArtifactStore, MemoryArtifactStoreStats, ObjectDeleteOutcome, ObjectError, ObjectInfo, ObjectKey, ObjectRange, - S3ArtifactStore, S3ArtifactStoreOptions, DEFAULT_S3_MULTIPART_CONCURRENCY, + ObjectSealOutcome, S3ArtifactStore, S3ArtifactStoreOptions, DEFAULT_S3_MULTIPART_CONCURRENCY, DEFAULT_S3_MULTIPART_PART_SIZE, }; pub use tiered::{TieredArtifactStore, TieredArtifactStoreOptions, TieredArtifactStoreStats}; diff --git a/crates/nokv-object/src/namespace.rs b/crates/nokv-object/src/namespace.rs index af5f3c1d1..250c0aa0d 100644 --- a/crates/nokv-object/src/namespace.rs +++ b/crates/nokv-object/src/namespace.rs @@ -8,7 +8,7 @@ use nokv_types::{ObjectNamespaceId, FIXED_ID_BYTES}; use crate::{ ArtifactObjectStore, ArtifactStoreCapabilities, ImmutableCreateOutcome, ObjectDeleteOutcome, - ObjectError, ObjectInfo, ObjectKey, ObjectRange, ProviderAdmissionReceipt, + ObjectError, ObjectInfo, ObjectKey, ObjectRange, ObjectSealOutcome, ProviderAdmissionReceipt, ProviderHandleIdentity, }; @@ -85,6 +85,10 @@ where self.inner.head(key) } + fn seal_immutable(&self, key: &ObjectKey) -> Result { + self.inner.seal_immutable(key) + } + fn delete(&self, key: &ObjectKey) -> Result { self.inner.delete(key) } diff --git a/crates/nokv-object/src/provider_admission_test.rs b/crates/nokv-object/src/provider_admission_test.rs index 3bb0046d8..860d1e570 100644 --- a/crates/nokv-object/src/provider_admission_test.rs +++ b/crates/nokv-object/src/provider_admission_test.rs @@ -4,8 +4,9 @@ use std::sync::Mutex; use crate::{ admit_artifact_provider, ArtifactObjectStore, ArtifactStoreCapabilities, ImmutableCreateOutcome, ObjectDeleteOutcome, ObjectError, ObjectInfo, ObjectKey, ObjectRange, - ProviderAdmissionCapability, ProviderAdmissionError, ProviderAdmissionProfile, S3ArtifactStore, - S3ArtifactStoreOptions, DEFAULT_ARTIFACT_BLOCK_SIZE, PROVIDER_ADMISSION_CONTRACT_VERSION, + ObjectSealOutcome, ProviderAdmissionCapability, ProviderAdmissionError, + ProviderAdmissionProfile, S3ArtifactStore, S3ArtifactStoreOptions, DEFAULT_ARTIFACT_BLOCK_SIZE, + PROVIDER_ADMISSION_CONTRACT_VERSION, }; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -20,6 +21,10 @@ enum FakeBehavior { /// observed as `Replayed` (the S3 adapter's behavior under transient /// write errors). LandedResponseReplayed, + SealingUnsupported, + DeleteInsteadOfSeal, + RestoreSealedPayload, + LoseSealResponse, } #[derive(Debug)] @@ -74,7 +79,20 @@ impl ArtifactObjectStore for FakeProvider { bytes: &[u8], ) -> Result { match self.behavior { - FakeBehavior::Correct => self.apply_create(key, bytes), + FakeBehavior::Correct + | FakeBehavior::SealingUnsupported + | FakeBehavior::DeleteInsteadOfSeal + | FakeBehavior::LoseSealResponse => self.apply_create(key, bytes), + FakeBehavior::RestoreSealedPayload => { + let mut objects = self.objects.lock().unwrap(); + if objects.get(key).is_some_and(Vec::is_empty) { + objects.insert(key.clone(), bytes.to_vec()); + Ok(ImmutableCreateOutcome::Created) + } else { + drop(objects); + self.apply_create(key, bytes) + } + } FakeBehavior::RejectConditional => Err(ObjectError::AtomicCreateUnsupported), FakeBehavior::Unavailable => Err(ObjectError::backend_failure( "endpoint=https://sentinel.invalid is temporarily unavailable", @@ -131,12 +149,80 @@ impl ArtifactObjectStore for FakeProvider { ObjectDeleteOutcome::Absent }) } + + fn seal_immutable(&self, key: &ObjectKey) -> Result { + if self.behavior == FakeBehavior::SealingUnsupported { + return Err(ObjectError::ImmutableSealingUnsupported); + } + let mut objects = self.objects.lock().unwrap(); + if self.behavior == FakeBehavior::DeleteInsteadOfSeal { + objects.remove(key); + return Ok(ObjectSealOutcome::Sealed); + } + let previous = objects.insert(key.clone(), Vec::new()); + if self.behavior == FakeBehavior::LoseSealResponse { + return Err(ObjectError::SealAmbiguous { + key: key.clone(), + detail: "endpoint=https://sentinel.invalid lost sealing response".to_owned(), + }); + } + Ok(if previous.is_some_and(|bytes| bytes.is_empty()) { + ObjectSealOutcome::AlreadySealed + } else { + ObjectSealOutcome::Sealed + }) + } } fn profile() -> ProviderAdmissionProfile { ProviderAdmissionProfile::single_put(64).unwrap() } +#[test] +fn generic_admission_does_not_grant_append_sealing() { + let store = FakeProvider::new(FakeBehavior::SealingUnsupported); + let receipt = admit_artifact_provider(&store, profile()).unwrap(); + assert!(receipt.admits_store(&store, 64)); + assert!(!receipt.append_sealing_verified()); + assert!(!receipt.admits_append_store(&store, 64)); + assert_eq!( + admit_artifact_provider(&store, profile().with_append_sealing()), + Err(ProviderAdmissionError::Rejected { + capability: ProviderAdmissionCapability::ImmutableSealing, + }) + ); +} + +#[test] +fn append_admission_proves_monotonic_sealing_and_handle_binding() { + for behavior in [FakeBehavior::Correct, FakeBehavior::LoseSealResponse] { + let store = FakeProvider::new(behavior); + let other = FakeProvider::new(behavior); + let receipt = admit_artifact_provider(&store, profile().with_append_sealing()).unwrap(); + assert!(receipt.append_sealing_verified()); + assert!(receipt.admits_append_store(&store, 64)); + assert!(!receipt.admits_append_store(&other, 64)); + assert!(!receipt.admits_append_store(&store, 65)); + assert!(store.objects.lock().unwrap().is_empty()); + } +} + +#[test] +fn append_admission_rejects_absence_and_payload_resurrection() { + for behavior in [ + FakeBehavior::DeleteInsteadOfSeal, + FakeBehavior::RestoreSealedPayload, + ] { + let store = FakeProvider::new(behavior); + assert_eq!( + admit_artifact_provider(&store, profile().with_append_sealing()), + Err(ProviderAdmissionError::Rejected { + capability: ProviderAdmissionCapability::ImmutableSealing, + }) + ); + } +} + #[test] fn landed_creates_with_lost_responses_still_admit_atomic_create_if_absent() { // Every fresh create lands but is reported as `Replayed` after @@ -249,6 +335,10 @@ fn public_provider_errors_never_render_endpoint_bucket_or_key_details() { key: key.clone(), detail: "endpoint=https://sentinel.invalid bucket=sentinel-bucket".to_owned(), }, + ObjectError::SealAmbiguous { + key: key.clone(), + detail: "endpoint=https://sentinel.invalid bucket=sentinel-bucket".to_owned(), + }, ObjectError::backend_failure( "endpoint=https://sentinel.invalid bucket=sentinel-bucket", true, @@ -303,6 +393,18 @@ fn live_provider_admission_passes_the_single_put_profile() { assert!(receipt.admits_store(&store, DEFAULT_ARTIFACT_BLOCK_SIZE)); } +#[test] +#[ignore = "requires an explicitly configured live S3-compatible endpoint"] +fn live_provider_admission_proves_append_sealing() { + let store = live_s3_store(); + let profile = ProviderAdmissionProfile::single_put(DEFAULT_ARTIFACT_BLOCK_SIZE) + .unwrap() + .with_append_sealing(); + let receipt = admit_artifact_provider(&store, profile).expect("live append sealing admission"); + assert!(receipt.append_sealing_verified()); + assert!(receipt.admits_append_store(&store, DEFAULT_ARTIFACT_BLOCK_SIZE)); +} + #[test] #[ignore = "requires an explicitly configured incompatible S3-compatible endpoint"] fn live_provider_admission_fails_closed_with_a_redacted_typed_result() { diff --git a/crates/nokv-object/src/store.rs b/crates/nokv-object/src/store.rs index f81db64a7..2207bc295 100644 --- a/crates/nokv-object/src/store.rs +++ b/crates/nokv-object/src/store.rs @@ -50,6 +50,13 @@ pub enum ObjectDeleteOutcome { Absent, } +/// A permanently reserved key containing no payload bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ObjectSealOutcome { + Sealed, + AlreadySealed, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ArtifactStoreCapabilities { pub range_read: bool, @@ -98,6 +105,7 @@ pub enum ObjectError { actual: ObjectNamespaceId, }, AtomicCreateUnsupported, + ImmutableSealingUnsupported, ProviderAdmissionRequired, ProviderAdmissionBlockSizeExceeded { requested: usize, @@ -111,6 +119,10 @@ pub enum ObjectError { key: ObjectKey, detail: String, }, + SealAmbiguous { + key: ObjectKey, + detail: String, + }, Backend { detail: String, retryable: bool, @@ -156,6 +168,17 @@ pub trait ArtifactObjectStore: Send + Sync { fn head(&self, key: &ObjectKey) -> Result, ObjectError>; + /// Permanently replace an abandoned object's bytes with an empty object. + /// + /// Metadata authority must first fence publication and reserve this key + /// forever. The implementation must never delete the key, including while + /// retrying, and must prevent late immutable creates from restoring payload + /// bytes after success. Callers must not subsequently delete sealed keys. + /// A post-dispatch error with an unknown result is [`ObjectError::SealAmbiguous`]. + fn seal_immutable(&self, _key: &ObjectKey) -> Result { + Err(ObjectError::ImmutableSealingUnsupported) + } + /// Delete one immutable object. /// /// A backend error after dispatch must be returned as @@ -199,6 +222,10 @@ where (**self).head(key) } + fn seal_immutable(&self, key: &ObjectKey) -> Result { + (**self).seal_immutable(key) + } + fn delete(&self, key: &ObjectKey) -> Result { (**self).delete(key) } @@ -296,6 +323,7 @@ pub struct MemoryArtifactStoreStats { pub reads: u64, pub read_bytes: u64, pub deletes: u64, + pub seals: u64, } impl MemoryArtifactStore { @@ -317,10 +345,10 @@ impl Default for MemoryArtifactStore { Self { state: Arc::new(Mutex::new(MemoryArtifactStoreState::default())), handle_identity, - admission_receipt: Arc::new(ProviderAdmissionReceipt::trusted_in_process( - handle_identity, - usize::MAX, - )), + admission_receipt: Arc::new( + ProviderAdmissionReceipt::trusted_in_process(handle_identity, usize::MAX) + .with_trusted_append_sealing(), + ), } } } @@ -402,6 +430,24 @@ impl ArtifactObjectStore for MemoryArtifactStore { .saturating_sub(bytes.len() as u64); Ok(ObjectDeleteOutcome::Deleted) } + + fn seal_immutable(&self, key: &ObjectKey) -> Result { + let mut state = self.state.lock().map_err(ObjectError::poisoned)?; + match state.objects.insert(key.clone(), Vec::new()) { + Some(previous) if previous.is_empty() => return Ok(ObjectSealOutcome::AlreadySealed), + Some(previous) => { + state.stats.resident_bytes = state + .stats + .resident_bytes + .saturating_sub(previous.len() as u64); + } + None => { + state.stats.resident_objects = state.stats.resident_objects.saturating_add(1); + } + } + state.stats.seals = state.stats.seals.saturating_add(1); + Ok(ObjectSealOutcome::Sealed) + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -613,6 +659,70 @@ impl ArtifactObjectStore for S3ArtifactStore { } } + fn seal_immutable(&self, key: &ObjectKey) -> Result { + let capabilities = self.operator.info().full_capability(); + if !capabilities.write_with_if_match || !capabilities.write_with_if_not_exists { + return Err(ObjectError::ImmutableSealingUnsupported); + } + + // A key may move only from absent/payload to an empty permanent fence. + // Never DELETE: a delayed deletion could remove another owner's seal + // after metadata has already certified cleanup. Conditional losers + // reobserve the winner; all cleanup writers converge on the same fence. + for _ in 0..8 { + let options = match self.operator.stat(key.as_str()) { + Ok(metadata) if metadata.content_length() == 0 => { + return Ok(ObjectSealOutcome::AlreadySealed); + } + Ok(metadata) => WriteOptions { + if_match: Some( + metadata + .etag() + .filter(|etag| !etag.is_empty()) + .ok_or(ObjectError::ImmutableSealingUnsupported)? + .to_owned(), + ), + ..WriteOptions::default() + }, + Err(error) if error.kind() == ErrorKind::NotFound => WriteOptions { + if_not_exists: true, + ..WriteOptions::default() + }, + Err(error) => return Err(ObjectError::opendal_backend(error)), + }; + match self + .operator + .write_options(key.as_str(), Vec::::new(), options) + { + Ok(_) => return Ok(ObjectSealOutcome::Sealed), + Err(error) + if matches!( + error.kind(), + ErrorKind::AlreadyExists | ErrorKind::ConditionNotMatch + ) => {} + Err(error) => { + // A lost response is success only after observing the + // permanent empty object. Absence is never a seal proof. + if self + .operator + .stat(key.as_str()) + .is_ok_and(|metadata| metadata.content_length() == 0) + { + return Ok(ObjectSealOutcome::AlreadySealed); + } + return Err(ObjectError::SealAmbiguous { + key: key.clone(), + detail: error.to_string(), + }); + } + } + } + Err(ObjectError::SealAmbiguous { + key: key.clone(), + detail: "conditional seal did not converge within its bounded retry budget".to_owned(), + }) + } + fn delete(&self, key: &ObjectKey) -> Result { let existed = match self.head(key) { Ok(Some(_)) => true, @@ -765,6 +875,8 @@ impl fmt::Display for ObjectError { } Self::AtomicCreateUnsupported => formatter .write_str("artifact object provider does not support atomic create-if-absent"), + Self::ImmutableSealingUnsupported => formatter + .write_str("artifact object provider does not support permanent immutable sealing"), Self::ProviderAdmissionRequired => formatter .write_str("artifact object provider has no valid write-conformance admission"), Self::ProviderAdmissionBlockSizeExceeded { @@ -778,6 +890,9 @@ impl fmt::Display for ObjectError { formatter.write_str("immutable create outcome is ambiguous") } Self::DeleteAmbiguous { .. } => formatter.write_str("delete outcome is ambiguous"), + Self::SealAmbiguous { .. } => { + formatter.write_str("immutable seal outcome is ambiguous") + } Self::Backend { .. } => formatter.write_str("artifact object backend is unavailable"), } } diff --git a/crates/nokv-object/src/tests.rs b/crates/nokv-object/src/tests.rs index 4ede6e3a5..6572d05ff 100644 --- a/crates/nokv-object/src/tests.rs +++ b/crates/nokv-object/src/tests.rs @@ -106,6 +106,120 @@ fn immutable_create_accepts_only_exact_replay() { assert_eq!(stats.collisions, 1); } +#[test] +fn sealing_absent_and_uploaded_objects_permanently_rejects_late_payload() { + let store = MemoryArtifactStore::new(); + let absent = ObjectKey::new("nokv/artifacts/absent").unwrap(); + let uploaded = ObjectKey::new("nokv/artifacts/uploaded").unwrap(); + store + .create_immutable(&uploaded, b"abandoned payload") + .unwrap(); + for key in [&absent, &uploaded] { + assert_eq!( + store.seal_immutable(key).unwrap(), + ObjectSealOutcome::Sealed + ); + assert_eq!( + store.seal_immutable(key).unwrap(), + ObjectSealOutcome::AlreadySealed + ); + assert!(matches!( + store.create_immutable(key, b"abandoned payload"), + Err(ObjectError::ImmutableCollision { .. }) + )); + assert_eq!(store.head(key).unwrap().unwrap().size, 0); + assert!(store.read(key, None).unwrap().is_empty()); + } + let stats = store.stats().unwrap(); + assert_eq!(stats.resident_bytes, 0); + assert_eq!(stats.resident_objects, 2); + assert_eq!(stats.seals, 2); + assert_eq!(stats.deletes, 0); + assert!(store + .provider_admission_receipt() + .unwrap() + .admits_append_store(&store, 4)); +} + +#[test] +fn concurrent_sealers_and_late_writers_converge_without_deleting_the_key() { + let store = MemoryArtifactStore::new(); + let key = ObjectKey::new("nokv/artifacts/seal-race").unwrap(); + let barrier = std::sync::Barrier::new(9); + std::thread::scope(|scope| { + for index in 0..8 { + let store = &store; + let key = &key; + let barrier = &barrier; + scope.spawn(move || { + barrier.wait(); + if index % 2 == 0 { + store.seal_immutable(key).unwrap(); + } else { + let result = store.create_immutable(key, b"payload"); + assert!( + result.is_ok() + || matches!(result, Err(ObjectError::ImmutableCollision { .. })) + ); + } + }); + } + barrier.wait(); + }); + assert_eq!(store.head(&key).unwrap().unwrap().size, 0); + assert_eq!(store.stats().unwrap().deletes, 0); + assert!(matches!( + store.create_immutable(&key, b"late"), + Err(ObjectError::ImmutableCollision { .. }) + )); +} + +#[test] +fn tiered_sealing_uses_the_durable_fence_and_invalidates_old_cache_bytes() { + let hot = MemoryArtifactStore::new(); + let durable = MemoryArtifactStore::new(); + let store = TieredArtifactStore::new(hot.clone(), durable.clone(), Default::default()); + let key = ObjectKey::new("nokv/artifacts/tiered-seal").unwrap(); + store.create_immutable(&key, b"old payload").unwrap(); + assert_eq!(hot.read(&key, None).unwrap(), b"old payload"); + store.seal_immutable(&key).unwrap(); + assert!(hot.head(&key).unwrap().is_none()); + assert_eq!(durable.head(&key).unwrap().unwrap().size, 0); + assert!(store.read(&key, None).unwrap().is_empty()); + assert!(matches!( + store.create_immutable(&key, b"late payload"), + Err(ObjectError::ImmutableCollision { .. }) + )); + assert_eq!(durable.stats().unwrap().deletes, 0); +} + +#[test] +fn namespace_and_arc_wrappers_preserve_sealing_and_its_receipt() { + let inner = std::sync::Arc::new(MemoryArtifactStore::new()); + let namespace = nokv_types::ObjectNamespaceId::from_bytes([9; 16]); + ensure_object_namespace(&inner, namespace).unwrap(); + let store = BoundArtifactStore::open(inner.clone(), namespace).unwrap(); + let key = ObjectKey::new("nokv/artifacts/bound-seal").unwrap(); + store.seal_immutable(&key).unwrap(); + assert_eq!(inner.head(&key).unwrap().unwrap().size, 0); + assert!(store + .provider_admission_receipt() + .unwrap() + .admits_append_store(&store, 4)); +} + +#[test] +fn an_evictable_local_cache_cannot_certify_permanent_sealing() { + let temp = tempfile::tempdir().unwrap(); + let store = LocalHotTier::new(LocalHotTierOptions::new(temp.path(), 4)).unwrap(); + let key = ObjectKey::new("nokv/artifacts/cache-seal").unwrap(); + assert_eq!( + store.seal_immutable(&key), + Err(ObjectError::ImmutableSealingUnsupported) + ); + assert!(store.head(&key).unwrap().is_none()); +} + #[test] fn upload_range_read_and_exact_replay_share_one_manifest() { let store = MemoryArtifactStore::new(); diff --git a/crates/nokv-object/src/tiered.rs b/crates/nokv-object/src/tiered.rs index a309a04cb..137018871 100644 --- a/crates/nokv-object/src/tiered.rs +++ b/crates/nokv-object/src/tiered.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, Mutex}; use crate::{ ArtifactObjectStore, ArtifactStoreCapabilities, ImmutableCreateOutcome, ObjectDeleteOutcome, - ObjectError, ObjectInfo, ObjectKey, ObjectRange, ProviderAdmissionReceipt, + ObjectError, ObjectInfo, ObjectKey, ObjectRange, ObjectSealOutcome, ProviderAdmissionReceipt, ProviderHandleIdentity, }; @@ -33,6 +33,7 @@ pub struct TieredArtifactStoreStats { pub durable_read_bytes: u64, pub durable_deletes: u64, pub hot_delete_errors: u64, + pub durable_seals: u64, } #[derive(Clone, Debug)] @@ -176,6 +177,21 @@ where self.durable.head(key) } + fn seal_immutable(&self, key: &ObjectKey) -> Result { + let outcome = self.durable.seal_immutable(key)?; + self.record(|stats| { + stats.durable_seals = stats.durable_seals.saturating_add(1); + })?; + // The hot tier is disposable and cannot certify a permanent fence. + // Only the durable key is sealed; invalidate any cached old payload. + if self.hot.delete(key).is_err() { + self.record(|stats| { + stats.hot_delete_errors = stats.hot_delete_errors.saturating_add(1); + })?; + } + Ok(outcome) + } + fn delete(&self, key: &ObjectKey) -> Result { let outcome = self.durable.delete(key)?; if outcome == ObjectDeleteOutcome::Deleted { diff --git a/crates/nokv-protocol/src/append.rs b/crates/nokv-protocol/src/append.rs new file mode 100644 index 000000000..60763aca9 --- /dev/null +++ b/crates/nokv-protocol/src/append.rs @@ -0,0 +1,181 @@ +// Copyright 2024-2026 The NoKV Authors. +// SPDX-License-Identifier: Apache-2.0 + +//! Logical append identity, attempts, and historical receipts. + +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; + +use crate::{ + ArtifactRevisionIdentity, Digest, DigestUri, OperationIdentity, ProtocolError, RootIdentity, + RpcFailure, WorkspaceIdentity, WorkspacePath, +}; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AppendAttemptBinding { + pub operation_id: OperationIdentity, + pub attempt: u64, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AppendAttemptPhase { + Uploading, + Finalizing, + Published, + Aborting, + Cleaning, + Cleaned, + Quarantined, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AppendPreparation { + pub intent_digest: Digest, + pub target: WorkspacePath, + pub workspace_incarnation_id: WorkspaceIdentity, + pub attempt: u64, + pub publication_operation_id: OperationIdentity, + pub artifact_revision_id: ArtifactRevisionIdentity, + pub attempt_phase: AppendAttemptPhase, + pub activity_deadline_ms: u64, + /// Failure of the current publication attempt, independent of the logical outcome. + pub attempt_failure: Option, +} + +impl AppendPreparation { + pub(crate) fn validate(&self) -> Result<(), ProtocolError> { + crate::types::require_generation("append.activity_deadline_ms", self.activity_deadline_ms)?; + if let Some(failure) = &self.attempt_failure { + failure.validate()?; + if matches!( + self.attempt_phase, + AppendAttemptPhase::Uploading + | AppendAttemptPhase::Finalizing + | AppendAttemptPhase::Published + ) { + return Err(ProtocolError::invalid( + "append.attempt_failure", + "is not valid for an active or published attempt", + )); + } + } else if matches!( + self.attempt_phase, + AppendAttemptPhase::Cleaned | AppendAttemptPhase::Quarantined + ) { + return Err(ProtocolError::invalid( + "append.attempt_failure", + "is required for a cleaned or quarantined attempt", + )); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AppendResult { + /// The identity persisted by the caller before its first append. + pub operation_id: OperationIdentity, + /// The single publication attempt that committed this append. + pub publication_operation_id: OperationIdentity, + pub target: WorkspacePath, + pub workspace_incarnation_id: WorkspaceIdentity, + pub workspace_revision: u64, + pub generation: u64, + pub artifact_revision_id: ArtifactRevisionIdentity, + pub logical_size: u64, + pub body_digest: DigestUri, +} + +impl AppendResult { + pub(crate) fn validate(&self) -> Result<(), ProtocolError> { + crate::types::require_generation("append.workspace_revision", self.workspace_revision)?; + crate::types::require_generation("append.generation", self.generation)?; + if self.operation_id == self.publication_operation_id { + return Err(ProtocolError::invalid( + "append.publication_operation_id", + "must differ from the logical operation identity", + )); + } + Ok(()) + } +} + +/// Derive distinct immutable identities for a logical append's numbered attempt. +pub fn stable_append_attempt_identities( + root: RootIdentity, + operation_id: OperationIdentity, + attempt: u64, +) -> (OperationIdentity, ArtifactRevisionIdentity) { + let derive = |domain: &[u8]| { + let mut hasher = Sha256::new(); + hasher.update(domain); + hasher.update(root.0); + hasher.update(operation_id.0); + hasher.update(attempt.to_be_bytes()); + let digest = hasher.finalize(); + let mut identity = [0; 16]; + identity.copy_from_slice(&digest[..16]); + identity + }; + ( + OperationIdentity(derive(b"nokv.append.publication.v2\0")), + ArtifactRevisionIdentity(derive(b"nokv.append.revision.v2\0")), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn attempt_failure_is_separate_from_logical_outcome_and_validated_by_phase() { + let mut preparation = AppendPreparation { + intent_digest: Digest([1; 32]), + target: WorkspacePath { + workbench: crate::WorkbenchName::new("append-test").unwrap(), + path: crate::RelativePath::new("outputs/events.jsonl").unwrap(), + }, + workspace_incarnation_id: WorkspaceIdentity([2; 16]), + attempt: 0, + publication_operation_id: OperationIdentity([3; 16]), + artifact_revision_id: ArtifactRevisionIdentity([4; 16]), + attempt_phase: AppendAttemptPhase::Uploading, + activity_deadline_ms: 1_000, + attempt_failure: None, + }; + assert!(preparation.validate().is_ok()); + preparation.attempt_phase = AppendAttemptPhase::Cleaned; + assert!(preparation.validate().is_err()); + preparation.attempt_failure = Some(RpcFailure { + code: crate::ErrorCode::OperationFailed, + message: "publication activity lease expired".to_owned(), + retryable: false, + conflict: Some(crate::ConflictKind::OperationState), + current_generation: None, + route_hint: None, + }); + for phase in [ + AppendAttemptPhase::Aborting, + AppendAttemptPhase::Cleaning, + AppendAttemptPhase::Cleaned, + AppendAttemptPhase::Quarantined, + ] { + preparation.attempt_phase = phase; + assert!(preparation.validate().is_ok()); + } + preparation.attempt_phase = AppendAttemptPhase::Published; + assert!(preparation.validate().is_err()); + preparation.attempt_phase = AppendAttemptPhase::Cleaned; + preparation + .attempt_failure + .as_mut() + .unwrap() + .message + .clear(); + assert!(preparation.validate().is_err()); + } +} diff --git a/crates/nokv-protocol/src/artifact_publish.rs b/crates/nokv-protocol/src/artifact_publish.rs index a997ca777..be8017840 100644 --- a/crates/nokv-protocol/src/artifact_publish.rs +++ b/crates/nokv-protocol/src/artifact_publish.rs @@ -16,7 +16,7 @@ pub const MAX_ARTIFACT_PUBLISH_OBJECTS: usize = 1_048_576; /// Maximum provider-neutral manifest rows returned by one range-plan page. pub const MAX_ARTIFACT_READ_PLAN_ROWS: usize = 512; -const PUBLICATION_RECORD_ENCODING: u8 = 5; +const PUBLICATION_RECORD_ENCODING: u8 = 6; /// Canonical closure committed by `BeginArtifactPublish`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/nokv-protocol/src/codec.rs b/crates/nokv-protocol/src/codec.rs index 3646820ea..f9c6ad3c2 100644 --- a/crates/nokv-protocol/src/codec.rs +++ b/crates/nokv-protocol/src/codec.rs @@ -10,7 +10,7 @@ use crate::request::WorkspaceRpcRequest; use crate::response::WorkspaceRpcResponse; /// The exact and only accepted wire schema. -pub const WORKSPACE_PROTOCOL_SCHEMA: &str = "nokv.workspace.rpc.v10"; +pub const WORKSPACE_PROTOCOL_SCHEMA: &str = "nokv.workspace.rpc.v11"; /// Exact schema for the versioned workspace RPC preflight exchange. pub const WORKSPACE_PREFLIGHT_SCHEMA: &str = "nokv.workspace.rpc_preflight.v1"; /// Exact schema for the advertised workspace RPC capability set. @@ -238,6 +238,7 @@ mod tests { route: route(), request_id: RequestIdentity([3; 16]), operation: WorkspaceRequest::BeginArtifactPublish(crate::BeginArtifactPublishRequest { + append_attempt: None, append_intent_digest: None, operation_id: OperationIdentity([1; 16]), artifact_revision_id: crate::ArtifactRevisionIdentity([2; 16]), @@ -272,7 +273,7 @@ mod tests { #[test] fn request_round_trips_with_exact_schema() { - assert_eq!(WORKSPACE_PROTOCOL_SCHEMA, "nokv.workspace.rpc.v10"); + assert_eq!(WORKSPACE_PROTOCOL_SCHEMA, "nokv.workspace.rpc.v11"); let expected = request(); let encoded = encode_request(&expected).unwrap(); assert!(encoded @@ -282,7 +283,7 @@ mod tests { } #[test] - fn get_path_read_version_fence_has_one_exact_v10_encoding() { + fn get_path_read_version_fence_has_one_exact_v11_encoding() { let expected = WorkspaceRpcRequest { route: route(), request_id: RequestIdentity([0x41; 16]), @@ -304,15 +305,15 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(&encoded)), [ - 43, 107, 167, 115, 219, 115, 94, 117, 176, 8, 147, 194, 210, 216, 56, 161, 64, 151, - 61, 250, 74, 122, 42, 225, 63, 43, 174, 194, 243, 61, 141, 87 + 151, 149, 192, 126, 198, 107, 89, 133, 25, 106, 99, 70, 118, 191, 55, 180, 115, 9, + 187, 76, 58, 192, 124, 202, 204, 164, 144, 69, 134, 119, 155, 116 ], - "update only for an intentional GetPath v10 wire change" + "update only for an intentional GetPath v11 wire change" ); } #[test] - fn artifact_v1_query_and_catalog_keep_one_exact_v10_encoding() { + fn artifact_v1_query_and_catalog_keep_one_exact_v11_encoding() { let search = WorkspaceRpcRequest { route: route(), request_id: RequestIdentity([0x3a; 16]), @@ -430,10 +431,10 @@ mod tests { assert_eq!( <[u8; 32]>::from(golden.finalize()), [ - 81, 145, 52, 165, 175, 120, 94, 153, 211, 189, 77, 0, 195, 136, 57, 177, 110, 117, - 113, 179, 217, 251, 63, 40, 249, 32, 75, 95, 159, 40, 215, 86 + 224, 19, 176, 101, 32, 19, 101, 57, 154, 117, 159, 21, 186, 94, 186, 159, 240, 26, + 238, 69, 44, 255, 243, 228, 71, 30, 182, 159, 24, 169, 164, 44 ], - "update only for an intentional ArtifactV1 v10 wire change" + "update only for an intentional ArtifactV1 v11 wire change" ); assert_eq!(decode_request(&encoded_search).unwrap(), search); assert_eq!( @@ -447,7 +448,7 @@ mod tests { } #[test] - fn generic_exact_catalog_path_match_has_one_exact_v10_encoding() { + fn generic_exact_catalog_path_match_has_one_exact_v11_encoding() { let expected = WorkspaceRpcRequest { route: route(), request_id: RequestIdentity([0x42; 16]), @@ -474,15 +475,15 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(&encoded)), [ - 5, 71, 115, 115, 113, 40, 164, 143, 127, 183, 2, 223, 147, 69, 79, 150, 111, 60, - 236, 152, 105, 119, 195, 187, 140, 187, 68, 58, 27, 49, 232, 246 + 236, 170, 91, 173, 234, 150, 77, 102, 131, 72, 85, 116, 45, 218, 121, 129, 54, 46, + 38, 40, 52, 116, 161, 232, 249, 129, 85, 69, 223, 111, 102, 199 ], - "update only for an intentional Catalog Exact v10 wire change" + "update only for an intentional Catalog Exact v11 wire change" ); } #[test] - fn generic_registration_surface_has_one_exact_v10_encoding() { + fn generic_registration_surface_has_one_exact_v11_encoding() { let begin = generic_begin_request(); let append = generic_append_request(); let finalize = WorkspaceRpcRequest { @@ -592,10 +593,10 @@ mod tests { assert_eq!( <[u8; 32]>::from(golden.finalize()), [ - 13, 220, 44, 77, 152, 142, 98, 27, 59, 110, 212, 142, 89, 211, 249, 131, 116, 211, - 160, 182, 25, 228, 109, 196, 47, 129, 80, 211, 206, 153, 180, 240 + 185, 53, 175, 164, 133, 145, 72, 71, 43, 190, 231, 221, 240, 3, 18, 74, 81, 106, + 55, 71, 148, 46, 123, 205, 194, 11, 46, 44, 167, 235, 163, 164 ], - "update only for an intentional Generic registration v10 wire change" + "update only for an intentional Generic registration v11 wire change" ); for request in [&begin, &append, &finalize, &abort, &get] { assert_eq!( @@ -937,6 +938,7 @@ mod tests { ); let status = OperationStatus { + append_preparation: None, publish_preparation: None, token: OperationToken { operation_id: commit.operation_id, @@ -973,7 +975,7 @@ mod tests { } #[test] - fn restore_v10_late_binding_and_source_manifest_read_round_trip() { + fn restore_v11_late_binding_and_source_manifest_read_round_trip() { let restore_identity = RestoreManifestIdentity { publication_operation_id: OperationIdentity([0x21; 16]), artifact_revision_id: ArtifactRevisionIdentity([0x22; 16]), @@ -1001,10 +1003,10 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(&encoded_prepare)), [ - 101, 32, 165, 198, 170, 16, 166, 173, 232, 230, 81, 59, 208, 210, 146, 92, 30, 202, - 123, 26, 0, 200, 90, 148, 136, 164, 207, 94, 196, 99, 243, 84 + 165, 58, 202, 67, 148, 85, 50, 21, 84, 12, 61, 62, 78, 30, 153, 174, 182, 119, 214, + 16, 29, 98, 204, 66, 241, 225, 130, 254, 58, 121, 97, 83 ], - "update only for an intentional restore-v10 wire change" + "update only for an intentional restore-v11 wire change" ); let bind = WorkspaceRpcRequest { @@ -1116,7 +1118,7 @@ mod tests { source_manifest ); - let mut complete_v10_golden = Sha256::new(); + let mut complete_v11_golden = Sha256::new(); for encoded in [ encoded_prepare, encoded_bind, @@ -1124,16 +1126,16 @@ mod tests { encoded_prepared, encoded_source_manifest, ] { - complete_v10_golden.update((encoded.len() as u64).to_be_bytes()); - complete_v10_golden.update(encoded); + complete_v11_golden.update((encoded.len() as u64).to_be_bytes()); + complete_v11_golden.update(encoded); } assert_eq!( - <[u8; 32]>::from(complete_v10_golden.finalize()), + <[u8; 32]>::from(complete_v11_golden.finalize()), [ - 168, 93, 200, 28, 36, 10, 187, 93, 244, 3, 225, 144, 3, 71, 3, 129, 161, 143, 153, - 15, 144, 210, 198, 248, 244, 207, 80, 42, 204, 238, 190, 79 + 196, 15, 64, 182, 228, 178, 23, 245, 169, 244, 22, 242, 215, 154, 19, 155, 168, + 188, 201, 101, 214, 92, 215, 160, 221, 205, 51, 247, 186, 249, 83, 31 ], - "update only for an intentional restore-v10 wire change" + "update only for an intentional restore-v11 wire change" ); } @@ -1261,7 +1263,7 @@ mod tests { decode_request(&encoded), Err(ProtocolError::SchemaMismatch { actual: "nokv.workspace.rpc.v8".to_owned(), - expected: "nokv.workspace.rpc.v10", + expected: "nokv.workspace.rpc.v11", }) ); @@ -1314,7 +1316,7 @@ mod tests { decode_request(&actual_v4), Err(ProtocolError::SchemaMismatch { actual: "nokv.workspace.rpc.v4".to_owned(), - expected: "nokv.workspace.rpc.v10", + expected: "nokv.workspace.rpc.v11", }) ); @@ -1342,7 +1344,7 @@ mod tests { decode_response(&encoded), Err(ProtocolError::SchemaMismatch { actual: "nokv.workspace.rpc.v4".to_owned(), - expected: "nokv.workspace.rpc.v10", + expected: "nokv.workspace.rpc.v11", }) ); } @@ -1456,8 +1458,8 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(encode_request(&expected).unwrap())), [ - 133, 126, 69, 128, 33, 243, 208, 203, 199, 178, 159, 93, 19, 148, 84, 149, 172, 95, - 182, 51, 79, 160, 109, 23, 199, 183, 137, 14, 146, 112, 191, 126 + 1, 199, 172, 241, 75, 114, 194, 208, 36, 153, 86, 191, 63, 217, 252, 231, 179, 144, + 197, 77, 51, 2, 137, 102, 136, 205, 100, 119, 58, 134, 223, 72 ], "update only for an intentional capability-v2 wire change" ); diff --git a/crates/nokv-protocol/src/lib.rs b/crates/nokv-protocol/src/lib.rs index ad36bcf0e..264afb48b 100644 --- a/crates/nokv-protocol/src/lib.rs +++ b/crates/nokv-protocol/src/lib.rs @@ -9,6 +9,7 @@ //! `nokv-types`, and the wire surface exposes neither Holt layout nor //! object-provider configuration. +mod append; mod artifact_publish; mod codec; mod error; @@ -22,6 +23,10 @@ pub const MAX_ARTIFACT_DEPENDENCY_OWNERS: u32 = 64; /// Maximum sealed artifact revision dependency depth. pub const MAX_ARTIFACT_DEPENDENCY_DEPTH: u8 = 8; +pub use append::{ + stable_append_attempt_identities, AppendAttemptBinding, AppendAttemptPhase, AppendPreparation, + AppendResult, +}; pub use artifact_publish::{ parse_sha256_digest_uri, seal_artifact_publish_plan, sha256_digest_uri, ArtifactPublishPlanSeal, MAX_ARTIFACT_PUBLISH_BATCH_ROWS, MAX_ARTIFACT_PUBLISH_OBJECTS, diff --git a/crates/nokv-protocol/src/request.rs b/crates/nokv-protocol/src/request.rs index 1d11394f3..88ed95b64 100644 --- a/crates/nokv-protocol/src/request.rs +++ b/crates/nokv-protocol/src/request.rs @@ -425,6 +425,8 @@ impl RemovePathRequest { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct BeginArtifactPublishRequest { + /// Logical append identity and monotonic attempt owning this publication. + pub append_attempt: Option, /// Exact caller intent for a stable append; binds one publication attempt. pub append_intent_digest: Option, pub operation_id: OperationIdentity, @@ -451,6 +453,21 @@ pub struct BeginArtifactPublishRequest { impl BeginArtifactPublishRequest { fn validate(&self) -> Result<(), ProtocolError> { self.condition.validate()?; + if self.append_attempt.is_some() != self.append_intent_digest.is_some() { + return Err(ProtocolError::invalid( + "begin_artifact_publish.append_attempt", + "attempt binding and intent digest must be supplied together", + )); + } + if self + .append_attempt + .is_some_and(|binding| binding.operation_id == self.operation_id) + { + return Err(ProtocolError::invalid( + "begin_artifact_publish.append_attempt", + "logical and publication identities must differ", + )); + } if self.append_intent_digest.is_some() && (!matches!(self.authority, PublicationAuthority::Visible) || self.expected_workspace_incarnation_id.is_none() @@ -701,6 +718,8 @@ pub enum QuarantineResolution { /// revision was never published; the revision identity is released for a /// fresh publication. ProviderObjectsAbsent, + /// Failed append keys are permanently sealed against delayed immutable creates. + ProviderObjectsSealed, /// The artifact revision is already published; staged provider keys are /// the published revision's live objects and only this operation's /// private bookkeeping rows are removed. @@ -2181,6 +2200,7 @@ mod tests { condition: PublishCondition, ) -> BeginArtifactPublishRequest { BeginArtifactPublishRequest { + append_attempt: None, append_intent_digest: None, operation_id: OperationIdentity([1; 16]), artifact_revision_id: ArtifactRevisionIdentity([2; 16]), @@ -2235,6 +2255,10 @@ mod tests { PublishCondition::CreateOnly, ); request.append_intent_digest = Some(Digest([0x61; 32])); + request.append_attempt = Some(crate::AppendAttemptBinding { + operation_id: OperationIdentity([0x62; 16]), + attempt: 0, + }); assert!(request.validate().is_err()); request.expected_workspace_incarnation_id = Some(WorkspaceIdentity([9; 16])); request.validate().unwrap(); diff --git a/crates/nokv-protocol/src/response.rs b/crates/nokv-protocol/src/response.rs index 5613f4793..855609766 100644 --- a/crates/nokv-protocol/src/response.rs +++ b/crates/nokv-protocol/src/response.rs @@ -1504,6 +1504,7 @@ impl OperationProgress { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(tag = "kind", content = "result", rename_all = "snake_case")] pub enum OperationResult { + ArtifactAppend(crate::AppendResult), ArtifactPublish(PublishResult), Commit(CommitResult), Restore(RestoreResult), @@ -1512,6 +1513,7 @@ pub enum OperationResult { impl OperationResult { fn validate(&self) -> Result<(), ProtocolError> { match self { + Self::ArtifactAppend(result) => result.validate(), Self::ArtifactPublish(result) => result.validate(), Self::Commit(result) => result.validate(), Self::Restore(result) => result.validate(), @@ -1522,6 +1524,7 @@ impl OperationResult { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct PublishPreparation { + pub append_attempt: Option, /// Caller intent bound to one append publication attempt, when present. pub append_intent_digest: Option, pub target: WorkspacePath, @@ -1532,6 +1535,7 @@ pub struct PublishPreparation { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct OperationStatus { + pub append_preparation: Option>, pub token: OperationToken, pub kind: OperationKind, /// Present exactly for artifact-publication operations, including terminal ones. @@ -1549,6 +1553,37 @@ pub struct OperationStatus { impl OperationStatus { fn validate(&self) -> Result<(), ProtocolError> { self.progress.validate()?; + match (self.kind, self.append_preparation.as_ref()) { + (OperationKind::ArtifactAppend, Some(preparation)) => { + preparation.validate()?; + if let Some(OperationResult::ArtifactAppend(result)) = &self.result { + if result.target != preparation.target + || result.workspace_incarnation_id != preparation.workspace_incarnation_id + || result.publication_operation_id != preparation.publication_operation_id + || result.artifact_revision_id != preparation.artifact_revision_id + || preparation.attempt_phase != crate::AppendAttemptPhase::Published + { + return Err(ProtocolError::invalid( + "operation.append_preparation", + "does not match the durable append result", + )); + } + } + } + (OperationKind::ArtifactAppend, None) => { + return Err(ProtocolError::invalid( + "operation.append_preparation", + "is required for append operations", + )) + } + (_, Some(_)) => { + return Err(ProtocolError::invalid( + "operation.append_preparation", + "is only valid for append operations", + )) + } + (_, None) => {} + } match (self.kind, self.publish_preparation.as_ref()) { (OperationKind::ArtifactPublish, Some(preparation)) => { if let Some(OperationResult::ArtifactPublish(result)) = &self.result { @@ -1636,6 +1671,9 @@ impl OperationStatus { if let Some(result) = &self.result { result.validate()?; let (expected_kind, result_operation_id) = match result { + OperationResult::ArtifactAppend(result) => { + (OperationKind::ArtifactAppend, result.operation_id) + } OperationResult::ArtifactPublish(result) => { (OperationKind::ArtifactPublish, result.operation_id) } @@ -2336,8 +2374,10 @@ mod tests { fn running_status(kind: OperationKind) -> OperationStatus { OperationStatus { + append_preparation: None, publish_preparation: (kind == OperationKind::ArtifactPublish).then(|| { Box::new(PublishPreparation { + append_attempt: None, append_intent_digest: None, target: WorkspacePath { workbench: WorkbenchName::new("run").unwrap(), diff --git a/crates/nokv-protocol/src/types.rs b/crates/nokv-protocol/src/types.rs index 32606cc93..b38b7da1f 100644 --- a/crates/nokv-protocol/src/types.rs +++ b/crates/nokv-protocol/src/types.rs @@ -139,6 +139,7 @@ impl RootRoute { #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "snake_case")] pub enum WorkspaceCapability { + ArtifactAppendV1, ArtifactPublishV1, ArtifactRangeReadV1, ChangeFeedV1, @@ -152,7 +153,8 @@ pub enum WorkspaceCapability { } impl WorkspaceCapability { - pub const ALL: [Self; 10] = [ + pub const ALL: [Self; 11] = [ + Self::ArtifactAppendV1, Self::ArtifactPublishV1, Self::ArtifactRangeReadV1, Self::ChangeFeedV1, @@ -821,6 +823,7 @@ pub struct OperationToken { #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum OperationKind { + ArtifactAppend, ArtifactPublish, Commit, Restore, diff --git a/crates/nokv-python/README.md b/crates/nokv-python/README.md index f3d0daea1..36393bf01 100644 --- a/crates/nokv-python/README.md +++ b/crates/nokv-python/README.md @@ -62,26 +62,49 @@ not for installing the SDK. `expected`) with nothing written. Callers that omit the argument keep the 0.11.0 behaviour; a server older than 0.11.1 rejects a fenced request as an invalid argument instead of ignoring the fence. -- `Client.append_bytes(workbench, path, data, operation_id, ...)` appends under a - caller-owned stable identity. The workspace must exist. Pass - `expected_workspace_incarnation_id` to pin the intended workspace across - restarts; omission observes the existing incarnation and fences the owner - request with it. Reuse the same identity and inputs after a lost reply. - The result contains the original `operation_id`, `artifact_revision_id`, +- `Client.append_bytes(workbench, path, data, operation_id, ...)` accepts Python + `bytes` under a caller-owned stable logical identity. The first admission + requires an existing workspace. An omitted `expected_workspace_incarnation_id` + first resolves a recorded operation's original incarnation, then observes the + live workspace only when no operation was found. Explicit incarnation fences + are always checked. Save the identity before the first call and reuse the + exact inputs after a lost reply. + The result matches native `workspace-path append`: `status`, `operation`, + `state=committed`, `next_action=none`, logical `operation_id`, successful + `publication_operation_id`, `workbench_id`, `path`, `artifact_revision_id`, `generation`, `workspace_revision`, `logical_size`, `body_digest`, and - `workspace_incarnation_id`, not the latest path head. `replayed` and nullable - `commit_version` are call metadata rather than stable receipt fields. + `workspace_incarnation_id`. These describe the original publication. + `replayed` and nullable `commit_version` are call metadata. `content_type=None` inherits an existing artifact's type and uses `application/octet-stream` on creation; an explicit type applies to both. - `max_logical_size` optionally caps the resulting body. One identity owns one - attempt; CAS conflicts are not retried under automatically changed identities. - `AppendError` carries `operation_id`, `state`, `code`, and the expected - incarnation in `expected`. `cause_code` preserves an underlying RPC code - such as `RequestReplayMismatch`, or is `None` for failures without an RPC - cause; `retryable` is false. This distinguishes a permanently mismatched - intent from an uncertain reply without parsing the error message. An unknown - state does not prove nothing was published. Incarnation conflicts raise `WorkspaceIncarnationMismatch` with - the append's `operation_id` attached. Recover using the original identity. + Use an explicit matching type when replaying between Python and CLI text. + The delta limit is 16 MiB, checked before copying the Python bytes. + `max_logical_size=None` means a 16 MiB resulting-body limit; an explicit larger + bound is part of the intent and must be retained across retries. The default + block size is 4 MiB and maps to CLI `--block-size`. A retry may start a successor publication only after the + predecessor is fenced and cleaned; the logical identity never changes. +- `Client.operation_status(operation_id)` queries an append without resending + its payload or depending on a current workspace. `Client(root_id, routing)` + needs no object-store configuration; configured object stores are initialized + lazily on object operations, so a fresh status client works during an S3 + outage. The result reports `state` and `next_action` from the shared Rust SDK: + `committed/none` with the original `receipt`, `pending/poll`, + `ready_to_retry/resubmit_same`, or `quarantined/operator_reconcile`. It also + contains the logical and publication identities, attempt number and phase, + activity deadline, original target and incarnation, progress, `cause_code`, + `failure_message`, and `attempt_failure` for a failed physical attempt. The + logical operation may still be `ready_to_retry` while retaining that failure. + A query never advances an attempt. Retain or reproduce the original delta + until commitment; follow `resubmit_same` using `append_bytes`. +- Append and status failures carry `operation_id`, observed `state`, `code`, + expected incarnation in `expected`, `cause_code`, and `next_action=query_same`. + `publication_operation_id` is `None` until status supplies an observed + publication. `AppendError` is a `RuntimeError`; incarnation conflicts retain + the `WorkspaceIncarnationMismatch` subtype with the same recovery attributes. + `cause_code=RequestReplayMismatch` identifies changed inputs or another + lifecycle, so recover the original intent instead of blindly resubmitting. + `retryable` is false for generic retry handlers. An unknown result or a + `NotFound` observation never authorizes a replacement logical identity. - `WorkbenchFileSystem` is an fsspec compatibility adapter bound to one explicit Workbench. Paths must be one of `input`, `scripts`, `outputs`, `logs`, or `metadata`, optionally followed by an artifact-relative path. Sections and @@ -102,3 +125,9 @@ types are not part of version 1. Snapshots require a committed Workbench. Commit and restore must be driven by the canonical Workbench lifecycle facade; clients must not synthesize a run-manifest or duplicate the durable workflow locally. + +Stable append's size, retention, owner recovery, and release contract is specified +in [the append product specification](../../docs/development/append-product-spec.md). +The fsspec adapter's append mode remains outside this contract; retrying harnesses +should use native `nokv workspace-path append` first, or `Client.append_bytes` +when embedded Python execution is required. diff --git a/crates/nokv-python/python/nokv/__init__.py b/crates/nokv-python/python/nokv/__init__.py index 6e79310f2..77863d0c1 100644 --- a/crates/nokv-python/python/nokv/__init__.py +++ b/crates/nokv-python/python/nokv/__init__.py @@ -36,7 +36,10 @@ class AppendError(RuntimeError): ``state`` is the observed durable operation state, or ``None`` when it is unknown. Neither a timeout nor an unknown state proves that nothing was published. Retry the same intent with the same ``operation_id``; do not - generate a replacement identity to bypass an unresolved operation. + generate a replacement identity to bypass an unresolved operation. Query + ``Client.operation_status`` and follow its ``next_action``; the exception's + ``next_action`` is conservatively ``query_same``. A publication identity is + not inferred from an error and remains ``None`` until observed by a query. ``cause_code`` preserves the underlying RPC code, for example ``RequestReplayMismatch`` for a permanently mismatched intent, or ``None`` when there is no RPC failure. ``retryable`` is always false: generic retry @@ -51,6 +54,8 @@ def __init__( code: str, expected: str | None, cause_code: str | None = None, + next_action: str = "query_same", + publication_operation_id: str | None = None, ) -> None: super().__init__(message) self.operation_id = operation_id @@ -59,6 +64,8 @@ def __init__( self.expected = expected self.cause_code = cause_code self.retryable = False + self.next_action = next_action + self.publication_operation_id = publication_operation_id class WorkspaceIncarnationMismatch(RuntimeError): diff --git a/crates/nokv-python/src/client.rs b/crates/nokv-python/src/client.rs index 1862d8942..cc6ebbb2a 100644 --- a/crates/nokv-python/src/client.rs +++ b/crates/nokv-python/src/client.rs @@ -6,7 +6,7 @@ use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use nokv_client::{ @@ -29,7 +29,7 @@ use nokv_types::WorkbenchId; use nokv_workbench_projection::CanonicalWorkbenchProjection; use pyo3::exceptions::{PyFileExistsError, PyFileNotFoundError, PyRuntimeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyAnyMethods, PyDictMethods, PyListMethods}; +use pyo3::types::{PyAnyMethods, PyBytesMethods, PyDictMethods, PyListMethods}; use pyo3::types::{PyBytes, PyDict, PyList}; use serde_json::{Map as JsonMap, Value as JsonValue}; @@ -39,15 +39,16 @@ use crate::local_adapter::{ }; use crate::object_store::{ConfiguredObjectStore, PythonObjectStoreConfig}; use crate::python_value::{ - aggregate_result_to_py, catalog_result_to_py, find_workspaces_result_to_py, hex, - parse_aggregates, parse_field_specs, parse_fixed_hex, parse_predicates, parse_sort, - path_metadata_to_py, path_page_to_py, publish_outcome_to_py, publish_result_to_py, + aggregate_result_to_py, append_result_to_py, catalog_result_to_py, + find_workspaces_result_to_py, hex, parse_aggregates, parse_field_specs, parse_fixed_hex, + parse_predicates, parse_sort, path_metadata_to_py, path_page_to_py, publish_outcome_to_py, read_outcome_to_py, search_result_to_py, snapshot_result_to_py, workspace_summary_to_py, PythonAggregateSpec, PythonFieldSpec, PythonPredicateSpec, PythonSortSpec, }; use crate::routing::PythonRoutingConfig; type RustWorkspaceClient = WorkspaceClient>; +type BoundPythonObjectStore = Arc>; type PythonRangeBatchRequest = (String, Vec<(u64, u64)>, Option, Option); const DEFAULT_SNAPSHOT_LEASE_SECONDS: u64 = 7 * 24 * 60 * 60; @@ -70,7 +71,8 @@ struct CollectedFile { #[pyclass(name = "Client")] pub(crate) struct PythonWorkspaceClient { client: Arc, - objects: Arc>, + objects: Mutex>, + object_store: Option, /// Presentation root the durable run manifest records, for example /// `/agents//wb`. It is not addressing: artifacts resolve by /// workbench name either way. But it is hashed into the commit @@ -86,7 +88,7 @@ impl PythonWorkspaceClient { #[pyo3(signature = ( root_id, routing, - object_store, + object_store = None, max_attempts = 3, connect_timeout_ms = 5_000, read_timeout_ms = 30_000, @@ -99,7 +101,7 @@ impl PythonWorkspaceClient { py: Python<'_>, root_id: &str, routing: PyRef<'_, PythonRoutingConfig>, - object_store: PyRef<'_, PythonObjectStoreConfig>, + object_store: Option>, max_attempts: u32, connect_timeout_ms: u64, read_timeout_ms: u64, @@ -122,22 +124,12 @@ impl PythonWorkspaceClient { let client = WorkspaceClient::new(root_id, transport, resolver, ClientOptions { max_attempts }) .map_err(value_error)?; - let object_store = (*object_store).clone(); - let objects = py - .detach(move || object_store.build()) - .map_err(value_error)?; - let preflight = py - .detach(|| client.preflight(std::iter::empty())) + py.detach(|| client.preflight(std::iter::empty())) .map_err(runtime_error)?; - let namespace_id = preflight.value.route.object_namespace_id.into(); - if objects.is_memory() { - nokv_object::ensure_object_namespace(&objects, namespace_id).map_err(value_error)?; - } - let objects = - nokv_object::BoundArtifactStore::open(objects, namespace_id).map_err(value_error)?; Ok(Self { client: Arc::new(client), - objects: Arc::new(objects), + objects: Mutex::new(None), + object_store: object_store.map(|config| (*config).clone()), workbench_root, }) } @@ -180,7 +172,7 @@ impl PythonWorkspaceClient { replace, }; let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); + let objects = self.bound_objects(py)?; let max_artifact_bytes = LIFECYCLE_MAX_MANIFEST_BYTES; let outcome = py .detach(move || { @@ -256,7 +248,7 @@ impl PythonWorkspaceClient { destination_workbench_id, }; let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); + let objects = self.bound_objects(py)?; let outcome = py .detach(move || { let options = WorkbenchLifecycleOptions::new(LIFECYCLE_MAX_MANIFEST_BYTES) @@ -501,8 +493,9 @@ impl PythonWorkspaceClient { /// Append bytes once under a caller-owned operation identity. /// - /// The workspace must already exist. Retain the identity and incarnation - /// across process restarts. Replays return the original publication receipt, + /// The workspace must exist on first admission. Retain the logical identity + /// across process restarts; recovery resolves its original incarnation. + /// Replays return the original publication receipt, /// which may precede the current path head. Existing artifact metadata and /// content type are inherited unless `content_type` explicitly overrides it. #[pyo3(signature = ( @@ -521,7 +514,7 @@ impl PythonWorkspaceClient { py: Python<'py>, workbench: &str, path: &str, - data: Vec, + data: &Bound<'py, PyBytes>, operation_id: &str, content_type: Option<&str>, block_size: usize, @@ -529,11 +522,44 @@ impl PythonWorkspaceClient { expected_workspace_incarnation_id: Option<&str>, ) -> PyResult> { let operation_id = OperationIdentity(parse_fixed_hex("operation_id", operation_id)?); - let target = parse_workspace_path(workbench, path)?; + if data.as_bytes().len() > nokv_client::MAX_APPEND_DELTA_BYTES { + return Err(append_local_error( + py, + format!( + "append delta is {} bytes, maximum is {}", + data.as_bytes().len(), + nokv_client::MAX_APPEND_DELTA_BYTES, + ), + operation_id, + None, + "InvalidArgument", + None, + )); + } + let data = data.as_bytes().to_vec(); + let target = parse_workspace_path(workbench, path).map_err(|error| { + append_local_error( + py, + error.to_string(), + operation_id, + None, + "InvalidArgument", + None, + ) + })?; let content_type = content_type .map(|value| ContentType::new(value.to_owned())) .transpose() - .map_err(value_error)?; + .map_err(|error| { + append_local_error( + py, + error.to_string(), + operation_id, + None, + "InvalidArgument", + None, + ) + })?; let create_content_type = content_type .clone() .unwrap_or(ContentType::new("application/octet-stream").map_err(value_error)?); @@ -543,21 +569,11 @@ impl PythonWorkspaceClient { }) .transpose()?; let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); - let (call, incarnation) = py - .detach(move || { - let incarnation = match expected { - Some(incarnation) => incarnation, - None => { - client - .get_workspace(GetWorkspaceRequest { - workbench: target.workbench.clone(), - }) - .map_err(|error| (Box::new(error), None))? - .value - .workspace_incarnation_id - } - }; + let (options, recovered, incarnation) = py + .detach(|| { + let incarnation = client + .resolve_append_workspace_incarnation(operation_id, &target, expected) + .map_err(|error| (Box::new(error), expected))?; let mut options = IdempotentAppendOptions::new( operation_id, target, @@ -571,14 +587,131 @@ impl PythonWorkspaceClient { if let Some(max_logical_size) = max_logical_size { options = options.with_max_logical_size(max_logical_size); } - client - .append_artifact_idempotent(objects.as_ref(), options, &data) - .map(|call| (call, incarnation)) - .map_err(|error| (Box::new(error), Some(incarnation))) + let recovered = client + .recover_append_receipt(&mut options, &data) + .map_err(|error| (Box::new(error), Some(incarnation)))?; + Ok((options, recovered, incarnation)) }) .map_err(|(error, expected)| append_error(py, *error, operation_id, expected))?; - let dict = publish_result_to_py(py, &call.value)?; - dict.set_item("workspace_incarnation_id", hex(&incarnation.0))?; + let call = match recovered { + Some(call) => call, + None => { + let objects = self.bound_objects(py).map_err(|error| { + append_local_error( + py, + error.to_string(), + operation_id, + Some(incarnation), + "AppendFailed", + None, + ) + })?; + py.detach(|| objects.inner().validate_append_capabilities()) + .map_err(|error| { + append_local_error( + py, + error.to_string(), + operation_id, + Some(incarnation), + "AppendFailed", + Some(error.code()), + ) + })?; + py.detach(move || { + client.append_artifact_idempotent(objects.as_ref(), options, &data) + }) + .map_err(|error| append_error(py, error, operation_id, Some(incarnation)))? + } + }; + let dict = append_result_to_py(py, &call.value)?; + dict.set_item("status", "success")?; + dict.set_item("operation", "append")?; + dict.set_item("state", "committed")?; + dict.set_item("next_action", "none")?; + set_call_metadata(&dict, call.commit_version, call.replayed)?; + Ok(dict) + } + + /// Query a logical append without its payload, workspace, or object store. + /// + /// This only observes state. Follow `next_action`; a missing operation or + /// unavailable owner does not authorize replacement of the logical identity. + fn operation_status<'py>( + &self, + py: Python<'py>, + operation_id: &str, + ) -> PyResult> { + let operation_id = OperationIdentity(parse_fixed_hex("operation_id", operation_id)?); + let client = Arc::clone(&self.client); + let call = py + .detach(move || client.get_append_operation(operation_id)) + .map_err(|error| append_error(py, error, operation_id, None))?; + let recovery = nokv_client::append_operation_recovery(&call.value) + .map_err(|error| append_error(py, error, operation_id, None))?; + let status = &call.value; + let preparation = status + .append_preparation + .as_ref() + .ok_or_else(|| runtime_error("append status has no preparation"))?; + let dict = PyDict::new(py); + dict.set_item("status", "success")?; + dict.set_item("operation", "append")?; + dict.set_item("operation_id", hex(&status.token.operation_id.0))?; + dict.set_item("state", recovery.state.as_str())?; + dict.set_item("next_action", recovery.next_action.as_str())?; + dict.set_item("observed_state", format!("{:?}", status.state))?; + dict.set_item( + "publication_operation_id", + hex(&preparation.publication_operation_id.0), + )?; + dict.set_item("attempt", preparation.attempt)?; + dict.set_item( + "attempt_phase", + format!("{:?}", preparation.attempt_phase).to_ascii_lowercase(), + )?; + dict.set_item("activity_deadline_ms", preparation.activity_deadline_ms)?; + dict.set_item( + "workspace_incarnation_id", + hex(&preparation.workspace_incarnation_id.0), + )?; + dict.set_item("workbench_id", preparation.target.workbench.as_str())?; + dict.set_item("path", preparation.target.path.as_str())?; + let progress = PyDict::new(py); + progress.set_item("completed_rows", status.progress.completed_rows)?; + progress.set_item("total_rows", status.progress.total_rows)?; + progress.set_item("completed_bytes", status.progress.completed_bytes)?; + progress.set_item("total_bytes", status.progress.total_bytes)?; + dict.set_item("progress", progress)?; + let attempt_failure = preparation.attempt_failure.as_ref(); + dict.set_item( + "cause_code", + attempt_failure.map(|failure| format!("{:?}", failure.code)), + )?; + dict.set_item( + "failure_message", + attempt_failure.map(|failure| failure.message.as_str()), + )?; + match attempt_failure { + Some(failure) => { + let details = PyDict::new(py); + details.set_item("code", format!("{:?}", failure.code))?; + details.set_item("message", &failure.message)?; + details.set_item("retryable", failure.retryable)?; + details.set_item( + "conflict", + failure.conflict.map(|conflict| format!("{conflict:?}")), + )?; + details.set_item("current_generation", failure.current_generation)?; + dict.set_item("attempt_failure", details)?; + } + None => dict.set_item("attempt_failure", py.None())?, + } + match &status.result { + Some(nokv_protocol::OperationResult::ArtifactAppend(result)) => { + dict.set_item("receipt", append_result_to_py(py, result)?)?; + } + _ => dict.set_item("receipt", py.None())?, + } set_call_metadata(&dict, call.commit_version, call.replayed)?; Ok(dict) } @@ -631,7 +764,7 @@ impl PythonWorkspaceClient { expected_workspace_incarnation_id, )?; let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); + let objects = self.bound_objects(py)?; let outcome = py .detach(move || client.publish_artifact(objects.as_ref(), options, &data)) .map_err(|error| publish_error(py, error, expected_workspace_incarnation_id))?; @@ -685,7 +818,7 @@ impl PythonWorkspaceClient { )?; let local_file = PathBuf::from(local_file); let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); + let objects = self.bound_objects(py)?; let outcome = py .detach(move || { let bytes = read_regular_file(&local_file) @@ -714,7 +847,7 @@ impl PythonWorkspaceClient { let target = parse_workspace_path(workbench, path)?; let view = parse_read_view(snapshot_id)?; let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); + let objects = self.bound_objects(py)?; let outcome = py .detach(move || client.read_artifact(objects.as_ref(), None, target, view)) .map_err(client_error)?; @@ -734,7 +867,7 @@ impl PythonWorkspaceClient { let target = parse_workspace_path(workbench, path)?; let view = parse_read_view(snapshot_id)?; let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); + let objects = self.bound_objects(py)?; let outcome = py .detach(move || { client.read_artifact_range(objects.as_ref(), None, target, view, offset, length) @@ -772,7 +905,7 @@ impl PythonWorkspaceClient { }) .collect::>>()?; let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); + let objects = self.bound_objects(py)?; let outcome = py .detach(move || { client.read_artifact_ranges_batch(objects.as_ref(), None, requests, view) @@ -1044,7 +1177,7 @@ impl PythonWorkspaceClient { let view = parse_read_view(snapshot_id)?; let local_directory = PathBuf::from(local_directory); let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); + let objects = self.bound_objects(py)?; let files = py .detach(move || { materialize_workspace( @@ -1095,7 +1228,7 @@ impl PythonWorkspaceClient { let prefix = parse_optional_relative_path(prefix)?; let content_type = ContentType::new(content_type.to_owned()).map_err(value_error)?; let client = Arc::clone(&self.client); - let objects = Arc::clone(&self.objects); + let objects = self.bound_objects(py)?; let files = py .detach(move || { collect_workspace( @@ -1121,6 +1254,40 @@ impl PythonWorkspaceClient { } impl PythonWorkspaceClient { + fn bound_objects(&self, py: Python<'_>) -> PyResult { + py.detach(|| { + let mut cached = self + .objects + .lock() + .map_err(|_| runtime_error("object store initialization lock was poisoned"))?; + if let Some(objects) = cached.as_ref() { + return Ok(Arc::clone(objects)); + } + let config = self.object_store.as_ref().ok_or_else(|| { + value_error("this operation requires an object_store configuration") + })?; + let objects = config.build().map_err(value_error)?; + let namespace_id = self + .client + .preflight(std::iter::empty()) + .map_err(runtime_error)? + .value + .route + .object_namespace_id + .into(); + if objects.is_memory() { + nokv_object::ensure_object_namespace(&objects, namespace_id) + .map_err(value_error)?; + } + let objects = Arc::new( + nokv_object::BoundArtifactStore::open(objects, namespace_id) + .map_err(value_error)?, + ); + *cached = Some(Arc::clone(&objects)); + Ok(objects) + }) + } + fn lifecycle_workbench_path(&self, workbench_id: &WorkbenchId) -> PyResult { let root = self.workbench_root.as_deref().ok_or_else(|| { value_error( @@ -1572,15 +1739,6 @@ fn append_error( ) -> PyErr { let identity = hex(&operation_id.0); let expected = expected.map(|value| hex(&value.0)); - if expected.is_some() - && client_error_failure(&error).is_some_and(|failure| { - failure.conflict == Some(nokv_protocol::ConflictKind::WorkspaceIncarnation) - }) - { - let mapped = publish_error(py, error, expected.as_deref()); - let _ = mapped.value(py).setattr("operation_id", identity); - return mapped; - } let state = match &error { ClientError::AppendUnresolved { state, .. } => state.map(|state| format!("{state:?}")), _ => None, @@ -1588,11 +1746,30 @@ fn append_error( let cause_code = client_error_failure(&error).map(|failure| format!("{:?}", failure.code)); let code = if matches!(&error, ClientError::AppendUnresolved { .. }) { "AppendUnresolved".to_owned() + } else if matches!(&error, ClientError::InvalidOptions(_)) { + "InvalidArgument".to_owned() } else { client_error_failure(&error) .map(|failure| format!("{:?}", failure.code)) .unwrap_or_else(|| "AppendFailed".to_owned()) }; + if expected.is_some() + && client_error_failure(&error).is_some_and(|failure| { + failure.conflict == Some(nokv_protocol::ConflictKind::WorkspaceIncarnation) + }) + { + let mapped = publish_error(py, error, expected.as_deref()); + let _ = mapped.value(py).setattr("operation_id", identity); + let _ = mapped.value(py).setattr("state", state); + let _ = mapped.value(py).setattr("code", code); + let _ = mapped.value(py).setattr("cause_code", cause_code); + let _ = mapped.value(py).setattr("next_action", "query_same"); + let _ = mapped + .value(py) + .setattr("publication_operation_id", py.None()); + let _ = mapped.value(py).setattr("retryable", false); + return mapped; + } let message = error.to_string(); py.import("nokv") .and_then(|module| module.getattr("AppendError")) @@ -1603,6 +1780,30 @@ fn append_error( .unwrap_or_else(|_| PyRuntimeError::new_err(message)) } +fn append_local_error( + py: Python<'_>, + message: String, + operation_id: OperationIdentity, + expected: Option, + code: &str, + cause_code: Option<&str>, +) -> PyErr { + py.import("nokv") + .and_then(|module| module.getattr("AppendError")) + .and_then(|class| { + class.call1(( + message.clone(), + hex(&operation_id.0), + None::, + code, + expected.map(|value| hex(&value.0)), + cause_code, + )) + }) + .map(PyErr::from_value) + .unwrap_or_else(|_| PyRuntimeError::new_err(message)) +} + fn client_error_failure(error: &ClientError) -> Option<&nokv_protocol::RpcFailure> { match error { ClientError::Rpc(failure) => Some(failure), diff --git a/crates/nokv-python/src/object_store.rs b/crates/nokv-python/src/object_store.rs index 815dc4d46..6ec267c1e 100644 --- a/crates/nokv-python/src/object_store.rs +++ b/crates/nokv-python/src/object_store.rs @@ -4,12 +4,12 @@ */ use std::fmt; -use std::sync::Arc; +use std::sync::{Arc, Mutex, OnceLock}; use nokv_object::{ admit_artifact_provider, ArtifactObjectStore, ArtifactStoreCapabilities, ImmutableCreateOutcome, MemoryArtifactStore, ObjectDeleteOutcome, ObjectError, ObjectInfo, - ObjectKey, ObjectRange, ProviderAdmissionError, ProviderAdmissionProfile, + ObjectKey, ObjectRange, ObjectSealOutcome, ProviderAdmissionError, ProviderAdmissionProfile, ProviderAdmissionReceipt, ProviderHandleIdentity, S3ArtifactStore, S3ArtifactStoreOptions, DEFAULT_ARTIFACT_BLOCK_SIZE, }; @@ -22,6 +22,8 @@ pub(crate) enum ConfiguredObjectStore { S3 { store: S3ArtifactStore, admission: Arc, + append_admission: Arc>, + admission_probe: Arc>, }, } @@ -114,11 +116,49 @@ impl PythonObjectStoreConfig { } impl ConfiguredObjectStore { + pub(crate) fn validate_append_capabilities(&self) -> Result<(), ProviderAdmissionError> { + match self { + Self::Memory(_) => Ok(()), + Self::S3 { + store, + append_admission, + admission_probe, + .. + } => cache_successful_admission(append_admission, admission_probe, || { + admit_artifact_provider( + store, + ProviderAdmissionProfile::single_put(DEFAULT_ARTIFACT_BLOCK_SIZE)? + .with_append_sealing(), + ) + }), + } + } + pub(crate) fn is_memory(&self) -> bool { matches!(self, Self::Memory(_)) } } +fn cache_successful_admission( + admission: &OnceLock, + probe_lock: &Mutex<()>, + probe: impl FnOnce() -> Result, +) -> Result<(), ProviderAdmissionError> { + if admission.get().is_some() { + return Ok(()); + } + let _guard = probe_lock + .lock() + .map_err(|_| ProviderAdmissionError::Inconclusive)?; + if admission.get().is_none() { + // Long-lived embedded clients must recover after a provider outage. + // Only successful evidence is immutable; errors may be retried. + let receipt = probe()?; + admission.get_or_init(|| receipt); + } + Ok(()) +} + impl PythonObjectStoreConfig { pub(crate) fn build(&self) -> Result { match &self.options { @@ -132,6 +172,8 @@ impl PythonObjectStoreConfig { Ok(ConfiguredObjectStore::S3 { store, admission: Arc::new(admission), + append_admission: Arc::new(OnceLock::new()), + admission_probe: Arc::new(Mutex::new(())), }) } } @@ -179,7 +221,11 @@ impl ArtifactObjectStore for ConfiguredObjectStore { fn provider_admission_receipt(&self) -> Option<&ProviderAdmissionReceipt> { match self { Self::Memory(store) => store.provider_admission_receipt(), - Self::S3 { admission, .. } => Some(admission.as_ref()), + Self::S3 { + admission, + append_admission, + .. + } => append_admission.get().or(Some(admission.as_ref())), } } @@ -194,6 +240,13 @@ impl ArtifactObjectStore for ConfiguredObjectStore { } } + fn seal_immutable(&self, key: &ObjectKey) -> Result { + match self { + Self::Memory(store) => store.seal_immutable(key), + Self::S3 { store, .. } => store.seal_immutable(key), + } + } + fn read(&self, key: &ObjectKey, range: Option) -> Result, ObjectError> { match self { Self::Memory(store) => store.read(key, range), @@ -218,8 +271,77 @@ impl ArtifactObjectStore for ConfiguredObjectStore { #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use super::*; + #[test] + fn failed_append_admission_can_recover_on_the_same_cache() { + let admission = OnceLock::new(); + let probe_lock = Mutex::new(()); + let store = MemoryArtifactStore::new(); + let calls = AtomicUsize::new(0); + for error in [ + ProviderAdmissionError::Unavailable, + ProviderAdmissionError::Inconclusive, + ] { + assert_eq!( + cache_successful_admission(&admission, &probe_lock, || { + calls.fetch_add(1, Ordering::SeqCst); + Err(error) + }), + Err(error) + ); + assert!(admission.get().is_none()); + } + cache_successful_admission(&admission, &probe_lock, || { + calls.fetch_add(1, Ordering::SeqCst); + admit_artifact_provider( + &store, + ProviderAdmissionProfile::single_put(64) + .unwrap() + .with_append_sealing(), + ) + }) + .unwrap(); + cache_successful_admission(&admission, &probe_lock, || { + calls.fetch_add(1, Ordering::SeqCst); + Err(ProviderAdmissionError::Unavailable) + }) + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 3); + assert!(admission.get().unwrap().admits_append_store(&store, 64)); + } + + #[test] + fn concurrent_append_admission_publishes_one_successful_receipt() { + let admission = OnceLock::new(); + let probe_lock = Mutex::new(()); + let store = MemoryArtifactStore::new(); + let calls = AtomicUsize::new(0); + let barrier = std::sync::Barrier::new(5); + std::thread::scope(|scope| { + for _ in 0..4 { + scope.spawn(|| { + barrier.wait(); + cache_successful_admission(&admission, &probe_lock, || { + calls.fetch_add(1, Ordering::SeqCst); + admit_artifact_provider( + &store, + ProviderAdmissionProfile::single_put(64) + .unwrap() + .with_append_sealing(), + ) + }) + .unwrap(); + }); + } + barrier.wait(); + }); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!(admission.get().unwrap().admits_append_store(&store, 64)); + } + #[test] fn memory_configuration_builds_the_explicit_test_store() { let config = PythonObjectStoreConfig::memory(); @@ -259,6 +381,8 @@ mod tests { let store = ConfiguredObjectStore::S3 { store: S3ArtifactStore::new(S3ArtifactStoreOptions::new("unused-test-bucket")).unwrap(), admission: Arc::new(foreign), + append_admission: Arc::new(OnceLock::new()), + admission_probe: Arc::new(Mutex::new(())), }; assert!(!store diff --git a/crates/nokv-python/src/python_value.rs b/crates/nokv-python/src/python_value.rs index 4d1ce05ec..5e281ca8f 100644 --- a/crates/nokv-python/src/python_value.rs +++ b/crates/nokv-python/src/python_value.rs @@ -390,6 +390,30 @@ pub(crate) fn publish_result_to_py<'py>( Ok(dict) } +pub(crate) fn append_result_to_py<'py>( + py: Python<'py>, + result: &nokv_protocol::AppendResult, +) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("operation_id", hex(&result.operation_id.0))?; + dict.set_item( + "publication_operation_id", + hex(&result.publication_operation_id.0), + )?; + dict.set_item("workbench_id", result.target.workbench.as_str())?; + dict.set_item("path", result.target.path.as_str())?; + dict.set_item( + "workspace_incarnation_id", + hex(&result.workspace_incarnation_id.0), + )?; + dict.set_item("workspace_revision", result.workspace_revision)?; + dict.set_item("generation", result.generation)?; + dict.set_item("artifact_revision_id", hex(&result.artifact_revision_id.0))?; + dict.set_item("logical_size", result.logical_size)?; + dict.set_item("body_digest", result.body_digest.as_str())?; + Ok(dict) +} + pub(crate) fn publish_outcome_to_py<'py>( py: Python<'py>, outcome: &ArtifactPublishOutcome, diff --git a/crates/nokv-python/tests/test_api_surface.py b/crates/nokv-python/tests/test_api_surface.py index 2311d6060..1519f8bca 100644 --- a/crates/nokv-python/tests/test_api_surface.py +++ b/crates/nokv-python/tests/test_api_surface.py @@ -31,6 +31,11 @@ def test_versioned_workbench_surface(): assert hasattr(nokv.Client, "publish_bytes") assert hasattr(nokv.Client, "publish_file") assert hasattr(nokv.Client, "append_bytes") + assert hasattr(nokv.Client, "operation_status") + assert list(inspect.signature(nokv.Client.operation_status).parameters) == [ + "self", "operation_id" + ] + assert "object_store=None" in (nokv.Client.__text_signature__ or "") append = inspect.signature(nokv.Client.append_bytes).parameters assert append["operation_id"].default is inspect.Parameter.empty assert append["content_type"].default is None @@ -102,6 +107,8 @@ def test_append_error_keeps_identity_and_observed_state(): assert error.expected == "b" * 32 assert error.cause_code is None assert error.retryable is False + assert error.next_action == "query_same" + assert error.publication_operation_id is None assert str(error) == "reply lost" mismatch = nokv.AppendError( "different intent", "a" * 32, None, "AppendUnresolved", "b" * 32, @@ -110,3 +117,22 @@ def test_append_error_keeps_identity_and_observed_state(): assert mismatch.cause_code == "RequestReplayMismatch" assert mismatch.code == "AppendUnresolved" assert mismatch.retryable is False + + +def test_append_provider_admission_errors_preserve_recovery_contract(): + for cause in ( + "ProviderAdmissionRejected", + "ProviderAdmissionUnavailable", + "ProviderAdmissionInconclusive", + ): + error = nokv.AppendError( + "provider admission did not complete", "a" * 32, None, + "AppendFailed", "b" * 32, cause, + ) + assert error.code == "AppendFailed" + assert error.cause_code == cause + assert error.state is None + assert error.operation_id == "a" * 32 + assert error.next_action == "query_same" + assert error.publication_operation_id is None + assert error.retryable is False diff --git a/crates/nokv-server/src/bootstrap.rs b/crates/nokv-server/src/bootstrap.rs index c735f5a9c..79cdb727c 100644 --- a/crates/nokv-server/src/bootstrap.rs +++ b/crates/nokv-server/src/bootstrap.rs @@ -70,6 +70,8 @@ pub struct RootAttach { /// Complete input for opening and serving one logical metadata shard. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ShardBoot { + /// Activity lease for stable append attempts; 1000..=86400000 milliseconds. + pub append_activity_lease_ms: u64, /// Logical shard that owns the store and lease. pub shard_id: nokv_types::LogicalShardId, /// Exact physical-store open mode. @@ -461,6 +463,17 @@ pub fn bootstrap_shard( boot: ShardBoot, ) -> Result { validate_boot(&boot)?; + if !recovery_objects + .provider_admission_receipt() + .is_some_and(|receipt| { + receipt.is_bound_to_store(recovery_objects.as_ref()) + && receipt.append_sealing_verified() + }) + { + return Err(ServerError::InvalidBootstrap( + "shard owner requires a handle-bound provider receipt verifying permanent append sealing".to_owned(), + )); + } let expected_namespace = boot .roots .first() @@ -658,8 +671,10 @@ pub fn bootstrap_shard( )); } - let metadata_executor: Arc = - Arc::new(MetadataWorkspaceRequestExecutor::new(Arc::clone(&meta))); + let metadata_executor: Arc = Arc::new( + MetadataWorkspaceRequestExecutor::new(Arc::clone(&meta)) + .with_append_activity_lease_ms(boot.append_activity_lease_ms)?, + ); let executor: Arc = Arc::new(RecoveryPublishingExecutor::new( metadata_executor, Arc::clone(&recovery), @@ -755,6 +770,11 @@ pub fn bootstrap_shard( } fn validate_boot(boot: &ShardBoot) -> Result<(), ServerError> { + if !(1_000..=crate::MAX_APPEND_ACTIVITY_LEASE_MS).contains(&boot.append_activity_lease_ms) { + return Err(ServerError::InvalidBootstrap( + "append activity lease must be between 1000 and 86400000 milliseconds".to_owned(), + )); + } if boot.recovery_publication == RecoveryPublicationMode::LocalOnly && matches!(boot.open, OpenMode::RecoverLog(_)) { @@ -1128,11 +1148,18 @@ fn open_existing_meta( path: PathBuf, logical_shard_id: nokv_types::LogicalShardId, ) -> Result, ServerError> { - let holt = HoltStore::open(HoltOptions::file( - path, - metadata_catalog(), - meta::store_limits(), - ))?; + let options = HoltOptions::file(path, metadata_catalog(), meta::store_limits()); + // A writable Holt open may replay/truncate its WAL and checkpoint pages + // before the metadata schema gate runs. Validate the complete existing + // authority through Holt's read-only recovery first, so an incompatible or + // corrupt store is rejected without changing its persisted files. + { + let mut inspection_options = options.clone(); + inspection_options.config = inspection_options.config.read_only(); + let inspection = HoltStore::open(inspection_options)?; + meta::MetaShard::open(Arc::new(inspection), logical_shard_id)?; + } + let holt = HoltStore::open(options)?; let store: Arc = Arc::new(holt); Ok(Arc::new(meta::MetaShard::open(store, logical_shard_id)?)) } @@ -1543,6 +1570,13 @@ mod tests { self.inner.head(key) } + fn seal_immutable( + &self, + key: &ObjectKey, + ) -> Result { + self.inner.seal_immutable(key) + } + fn delete(&self, key: &ObjectKey) -> Result { self.inner.delete(key) } @@ -1592,6 +1626,13 @@ mod tests { self.inner.head(key) } + fn seal_immutable( + &self, + key: &ObjectKey, + ) -> Result { + self.inner.seal_immutable(key) + } + fn delete(&self, key: &ObjectKey) -> Result { let attempt = self.delete_attempts.fetch_add(1, Ordering::SeqCst) + 1; if attempt == self.fail_delete_at.load(Ordering::SeqCst) { @@ -2155,6 +2196,7 @@ mod tests { fn acquire_boot(path: PathBuf, roots: &[RootId]) -> ShardBoot { ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::New(path), @@ -2185,6 +2227,7 @@ mod tests { seed: u8, ) -> ShardBoot { ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(path), @@ -2416,6 +2459,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects, ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -2819,6 +2863,7 @@ mod tests { let registry = Arc::new(RootOwnerRegistry::new()); let boot = ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -2875,6 +2920,7 @@ mod tests { as_control(&control), Arc::new(RootOwnerRegistry::new()), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -2931,6 +2977,7 @@ mod tests { as_control(&control), Arc::clone(®istry), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database.clone()), @@ -2989,6 +3036,7 @@ mod tests { as_control(&control), Arc::clone(®istry), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -3045,6 +3093,7 @@ mod tests { as_control(&control), Arc::new(RootOwnerRegistry::new()), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -3100,6 +3149,7 @@ mod tests { as_control(&control), Arc::new(RootOwnerRegistry::new()), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -3156,6 +3206,7 @@ mod tests { as_control(&control), Arc::new(RootOwnerRegistry::new()), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -3221,6 +3272,7 @@ mod tests { as_control(&control), Arc::new(RootOwnerRegistry::new()), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -3242,6 +3294,72 @@ mod tests { assert_eq!(record.state, LogicalShardState::Unassigned); } + #[test] + fn incompatible_existing_store_is_rejected_without_physical_file_mutation() { + use nokv_meta_store::{Commit, Key, Mutation, ReadBatch, ReadOp, ReadResult, WriteTxn}; + + fn image(directory: &Path) -> std::collections::BTreeMap> { + let mut result = std::collections::BTreeMap::new(); + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + result.extend(image(&path)); + } else { + result.insert(path.clone(), std::fs::read(path).unwrap()); + } + } + result + } + + let temporary = TempDir::new().unwrap(); + let database = temporary.path().join("metadata"); + drop(initialize_meta(database.clone(), shard()).unwrap()); + let system = meta::keyspaces() + .iter() + .find(|entry| entry.name == "system") + .unwrap() + .id; + let raw = HoltStore::open(HoltOptions::file( + &database, + metadata_catalog(), + meta::store_limits(), + )) + .unwrap(); + let values = raw + .read(ReadBatch { + ops: vec![ReadOp::Get(Key::new(system, b"schema".to_vec()))], + }) + .unwrap(); + let ReadResult::Get(Some(mut marker)) = values.results.into_iter().next().unwrap() else { + panic!("schema marker must exist") + }; + let version_offset = marker.len() - std::mem::size_of::(); + marker[version_offset..].copy_from_slice(&11_u32.to_be_bytes()); + assert_eq!( + raw.commit(WriteTxn { + checks: vec![], + mutations: vec![Mutation::Put { + key: Key::new(system, b"schema".to_vec()), + value: marker, + }] + }) + .unwrap(), + Commit::Applied + ); + drop(raw); + let before = image(&database); + assert!(!before.is_empty()); + let error = open_existing_meta(database.clone(), shard()) + .err() + .expect("old format rejected"); + assert!(error.to_string().contains("schema"), "{error}"); + assert_eq!( + image(&database), + before, + "rejection must not replay or checkpoint the original files" + ); + } + #[test] fn legacy_unbound_root_fence_is_bound_once_during_successor_reopen() { use nokv_meta_store::{Commit, Key, Mutation, TxnStore, WriteTxn}; @@ -3296,6 +3414,7 @@ mod tests { as_control(&control), Arc::new(RootOwnerRegistry::new()), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -3348,6 +3467,90 @@ mod tests { assert!(!database.exists()); } + #[test] + fn unqualified_append_owner_is_rejected_before_control_files_or_object_io() { + struct UnqualifiedOwnerStore { + identity: ProviderHandleIdentity, + receipt: Option, + } + impl ArtifactObjectStore for UnqualifiedOwnerStore { + fn object_namespace(&self) -> Option { + Some(ObjectNamespaceId::from_bytes( + [10; nokv_types::FIXED_ID_BYTES], + )) + } + fn capabilities(&self) -> ArtifactStoreCapabilities { + ArtifactStoreCapabilities::default() + } + fn provider_handle_identity(&self) -> ProviderHandleIdentity { + self.identity + } + fn provider_admission_receipt(&self) -> Option<&ProviderAdmissionReceipt> { + self.receipt.as_ref() + } + fn create_immutable( + &self, + _: &ObjectKey, + _: &[u8], + ) -> Result { + panic!("unqualified bootstrap must not write objects") + } + fn read(&self, _: &ObjectKey, _: Option) -> Result, ObjectError> { + panic!("unqualified bootstrap must not read objects") + } + fn head(&self, _: &ObjectKey) -> Result, ObjectError> { + panic!("unqualified bootstrap must not head objects") + } + fn delete(&self, _: &ObjectKey) -> Result { + panic!("unqualified bootstrap must not delete objects") + } + fn seal_immutable( + &self, + _: &ObjectKey, + ) -> Result { + panic!("unqualified bootstrap must not seal objects") + } + } + // Missing receipt, genuine generic-only receipt, and strong receipt + // borrowed from another handle all fail before acquiring an epoch. + for mode in 0..3 { + let temporary = TempDir::new().unwrap(); + let root_id = root(1); + let control = active_control(&[root_id]); + let registry = Arc::new(RootOwnerRegistry::new()); + let database = temporary.path().join("metadata"); + let raw = MemoryArtifactStore::new(); + let mut profile = nokv_object::ProviderAdmissionProfile::single_put(4).unwrap(); + if mode == 2 { + profile = profile.with_append_sealing(); + } + let receipt = if mode == 0 { + None + } else { + Some(nokv_object::admit_artifact_provider(&raw, profile).unwrap()) + }; + let objects = UnqualifiedOwnerStore { + identity: if mode == 2 { + ProviderHandleIdentity::new() + } else { + raw.provider_handle_identity() + }, + receipt, + }; + let before = control.get_logical_shard(&shard()).unwrap(); + let outcome = super::bootstrap_shard( + as_control(&control), + Arc::clone(®istry), + Arc::new(objects), + acquire_boot(database.clone(), &[root_id]), + ); + assert!(matches!(outcome, Err(ServerError::InvalidBootstrap(_)))); + assert_eq!(control.get_logical_shard(&shard()).unwrap(), before); + assert_eq!(registry.installed_root_count().unwrap(), 0); + assert!(!database.exists()); + } + } + #[test] fn unverified_requested_recovery_frontier_is_rejected_before_acquire() { let temporary = TempDir::new().unwrap(); @@ -3384,6 +3587,7 @@ mod tests { control.release_owner(&lease).unwrap(); let boot = ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::New(temporary.path().join("metadata")), @@ -3599,6 +3803,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects.clone(), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::RecoverLog(target), @@ -3661,6 +3866,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects, ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::RecoverLog(target), @@ -3725,6 +3931,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects.clone(), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(behind_path.clone()), @@ -3787,6 +3994,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects.clone(), ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(source_path.clone()), @@ -3838,6 +4046,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects, ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::RecoverLog(target), @@ -3897,6 +4106,7 @@ mod tests { objects.fail_delete_at(2); let recovered_path = temporary.path().join("recovered"); let boot = ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::RecoverLog(recovered_path.clone()), @@ -3973,6 +4183,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects, ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::RecoverLog(temporary.path().join("recovered")), @@ -4044,6 +4255,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects, ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::RecoverLog(target), @@ -4107,6 +4319,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects, ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::RecoverLog(target), @@ -4163,6 +4376,7 @@ mod tests { Arc::new(RootOwnerRegistry::new()), objects, ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::RecoverLog(temporary.path().join("recovered")), @@ -4315,6 +4529,7 @@ mod tests { let registry = Arc::new(RootOwnerRegistry::new()); let boot = ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), @@ -4852,6 +5067,7 @@ mod tests { )); let registry = Arc::new(RootOwnerRegistry::new()); let boot = ShardBoot { + append_activity_lease_ms: 1_800_000, recovery_publication: RecoveryPublicationMode::Shared, shard_id: shard(), open: OpenMode::Existing(database), diff --git a/crates/nokv-server/src/executor.rs b/crates/nokv-server/src/executor.rs index 776467419..ba9be5520 100644 --- a/crates/nokv-server/src/executor.rs +++ b/crates/nokv-server/src/executor.rs @@ -21,6 +21,8 @@ const MANIFEST_PLAN_CURSOR_BYTES: usize = 1 + types::FIXED_ID_BYTES * 2 + 8 * 3; // of spending the client's much smaller transport retry budget. const MAX_INTERNAL_METADATA_ATTEMPTS: u32 = 8; const PUBLISH_ACTIVITY_LEASE_MS: u64 = 30 * 60 * 1_000; +/// Maximum configured inactivity budget before an append attempt can be recovered. +pub const MAX_APPEND_ACTIVITY_LEASE_MS: u64 = 24 * 60 * 60 * 1_000; const RUN_MANIFEST_PATH: &str = "metadata/run_manifest.json"; const _: () = assert!(protocol::MAX_ARTIFACT_DEPENDENCY_OWNERS == meta::MAX_REVISION_DEPENDENCIES); const _: () = assert!(MAX_INTERNAL_METADATA_ATTEMPTS > 0); @@ -34,7 +36,8 @@ const _: () = assert!(protocol::MAX_QUERY_PAGE_LIMIT as usize == meta::MAX_QUERY // reject the legacy 61..=64 gap at DTO-to-domain conversion as main does. const _: () = assert!(protocol::ArtifactDescriptor::MAX_INDEX_FIELDS >= meta::MAX_TYPED_PROJECTION_FIELDS); -const SUPPORTED_WORKSPACE_CAPABILITIES: [protocol::WorkspaceCapability; 10] = [ +const SUPPORTED_WORKSPACE_CAPABILITIES: [protocol::WorkspaceCapability; 11] = [ + protocol::WorkspaceCapability::ArtifactAppendV1, protocol::WorkspaceCapability::ArtifactPublishV1, protocol::WorkspaceCapability::ArtifactRangeReadV1, protocol::WorkspaceCapability::ChangeFeedV1, @@ -158,6 +161,7 @@ impl RestorePreparationCoordinator { #[derive(Clone)] pub struct MetadataWorkspaceRequestExecutor { meta: Arc, + append_activity_lease_ms: u64, restore_preparations: Arc, #[cfg(feature = "restore-crash-test-support")] restore_initialization_barrier: Option, @@ -169,6 +173,7 @@ impl MetadataWorkspaceRequestExecutor { pub fn new(meta: Arc) -> Self { Self { meta, + append_activity_lease_ms: PUBLISH_ACTIVITY_LEASE_MS, restore_preparations: Arc::new(RestorePreparationCoordinator::default()), #[cfg(feature = "restore-crash-test-support")] restore_initialization_barrier: None, @@ -177,6 +182,19 @@ impl MetadataWorkspaceRequestExecutor { } } + pub fn with_append_activity_lease_ms( + mut self, + milliseconds: u64, + ) -> Result { + if !(1_000..=MAX_APPEND_ACTIVITY_LEASE_MS).contains(&milliseconds) { + return Err(crate::ServerError::InvalidOptions( + "append activity lease must be between 1000 and 86400000 milliseconds".to_owned(), + )); + } + self.append_activity_lease_ms = milliseconds; + Ok(self) + } + #[cfg(feature = "restore-crash-test-support")] pub fn with_restore_initialization_barrier( mut self, @@ -1889,6 +1907,18 @@ impl MetadataWorkspaceRequestExecutor { rpc: &protocol::WorkspaceRpcRequest, request: &protocol::BeginArtifactPublishRequest, ) -> Result { + if let Some(binding) = request.append_attempt { + let (operation_id, revision_id) = protocol::stable_append_attempt_identities( + rpc.route.root_id, + binding.operation_id, + binding.attempt, + ); + if request.operation_id != operation_id || request.artifact_revision_id != revision_id { + return Err(invalid_argument( + "append publication identities must match the logical operation and attempt", + )); + } + } self.claim_mutation(rpc)?; let step_id = derived_request_id(rpc.request_id, b"publish-begin", 0); if let Some(outcome) = self.replayed_publish(rpc.route, step_id)? { @@ -1906,8 +1936,15 @@ impl MetadataWorkspaceRequestExecutor { self.find_publish_operation(rpc.route, read.read_version, request.operation_id)? { if existing.append_intent_digest.is_some() || request.append_intent_digest.is_some() { - if existing.append_intent_digest - != request.append_intent_digest.map(|value| value.0) + if existing + .append_attempt + .map(|binding| protocol::AppendAttemptBinding { + operation_id: binding.operation_id.into(), + attempt: binding.attempt, + }) + != request.append_attempt + || existing.append_intent_digest + != request.append_intent_digest.map(|value| value.0) || existing.authority != meta::PublishAuthority::Visible || request.authority != protocol::PublicationAuthority::Visible || existing.workbench_id != workbench_id(&request.target.workbench)? @@ -2092,12 +2129,25 @@ impl MetadataWorkspaceRequestExecutor { } }; let mut operation = meta::PublishOperationRecord { + append_attempt: request + .append_attempt + .map(|binding| meta::AppendAttemptBinding { + operation_id: binding.operation_id.into(), + attempt: binding.attempt, + }), append_intent_digest: request.append_intent_digest.map(|value| value.0), operation_id: request.operation_id.into(), identity_digest: [0; types::SHA256_BYTES], initialization_digest: [0; types::SHA256_BYTES], initiating_owner_epoch: context.owner_epoch, - activity_deadline_ms: publish_activity_deadline_ms(&self.meta)?, + activity_deadline_ms: publish_activity_deadline_ms( + &self.meta, + if request.append_attempt.is_some() { + self.append_activity_lease_ms + } else { + PUBLISH_ACTIVITY_LEASE_MS + }, + )?, authority, workbench_id: workbench, workspace_incarnation_id, @@ -2362,14 +2412,37 @@ impl MetadataWorkspaceRequestExecutor { manifest_id: request.artifact.manifest_identity.clone(), typed_index_projection: encode_index_fields(&request.artifact.index_fields)?, }; - let outcome = meta::PublicationService::new(&self.meta) - .finalize_publish(meta::FinalizePublishRequest { - context, - expected_operation: finalizing_operation, - artifact, - dependency_owner_revision_ids, - }) - .map_err(publication_failure)?; + let service = meta::PublicationService::new(&self.meta); + let outcome = match service.finalize_publish(meta::FinalizePublishRequest { + context, + expected_operation: finalizing_operation.clone(), + artifact, + dependency_owner_revision_ids, + }) { + Ok(outcome) => outcome, + Err(error) => { + if finalizing_operation.append_attempt.is_some() + && matches!( + error, + meta::PublicationError::PathAlreadyExists + | meta::PublicationError::PathNotFound + | meta::PublicationError::PathGenerationMismatch { .. } + | meta::PublicationError::AppendBaseRevisionMismatch + ) + { + let cleanup_id = derived_request_id(rpc.request_id, b"append-path-conflict", 0); + let cleanup_context = self.publication_context(rpc.route, cleanup_id)?; + // Only the same exact Finalizing state may become Aborting. + // A concurrent winner remains authoritative and is queried by + // the client; unknown finalization outcomes never enter here. + match service.fail_append_conflict(cleanup_context, finalizing_operation) { + Ok(_) | Err(meta::PublicationError::ConcurrentMutation) => {} + Err(cleanup_error) => return Err(publication_failure(cleanup_error)), + } + } + return Err(publication_failure(error)); + } + }; published_response( outcome.operation, outcome.result, @@ -2429,6 +2502,9 @@ impl MetadataWorkspaceRequestExecutor { protocol::QuarantineResolution::ProviderObjectsAbsent => { meta::QuarantineReconcileResolution::RevisionUnpublished } + protocol::QuarantineResolution::ProviderObjectsSealed => { + meta::QuarantineReconcileResolution::RevisionUnpublishedSealed + } protocol::QuarantineResolution::RevisionPublished => { meta::QuarantineReconcileResolution::RevisionPublished } @@ -2729,6 +2805,7 @@ impl MetadataWorkspaceRequestExecutor { types::OperationKind::Publish, types::OperationKind::BuildCommit, types::OperationKind::Restore, + types::OperationKind::Append, ] { let key = meta::operation_key(context.root_id, kind, operation_id); if let Some(payload) = self @@ -2755,6 +2832,19 @@ impl MetadataWorkspaceRequestExecutor { return Err(not_found("operation does not exist")); }; let status = match kind { + types::OperationKind::Append => { + let parent = meta::AppendOperationRecord::decode(&payload) + .map_err(|error| internal(format!("invalid append operation: {error}")))?; + if parent.operation_id != operation_id { + return Err(internal("append operation key and payload disagree")); + } + let child = self.load_publish_operation( + rpc.route, + context.read_version, + parent.publication_operation_id.into(), + )?; + append_operation_status(&parent, &child)? + } types::OperationKind::Publish => { let operation = meta::PublishOperationRecord::decode(&payload) .map_err(|error| internal(format!("invalid publish operation: {error}")))?; @@ -3277,7 +3367,14 @@ impl MetadataWorkspaceRequestExecutor { token.operation_id, )?; require_publish_token(&operation, token)?; - let activity_deadline_ms = publish_activity_deadline_ms(&self.meta)?; + let activity_deadline_ms = publish_activity_deadline_ms( + &self.meta, + if operation.append_attempt.is_some() { + self.append_activity_lease_ms + } else { + PUBLISH_ACTIVITY_LEASE_MS + }, + )?; if activity_deadline_ms <= operation.activity_deadline_ms { return Ok(operation); } @@ -3947,7 +4044,10 @@ fn decode_manifest_plan_cursor( )) } -fn publish_activity_deadline_ms(meta: &meta::MetaShard) -> Result { +fn publish_activity_deadline_ms( + meta: &meta::MetaShard, + lease_ms: u64, +) -> Result { let wall_clock_ms = u64::try_from( SystemTime::now() .duration_since(UNIX_EPOCH) @@ -3958,7 +4058,7 @@ fn publish_activity_deadline_ms(meta: &meta::MetaShard) -> Result Result { + if !parent.matches_publication(child) + || parent.result != child.result + || parent.result.is_some() != (child.phase == types::PublishPhase::Published) + { + return Err(internal("append parent and current publication disagree")); + } + let child_status = publish_operation_status(child)?; + let phase = match child.phase { + types::PublishPhase::Uploading => protocol::AppendAttemptPhase::Uploading, + types::PublishPhase::Finalizing => protocol::AppendAttemptPhase::Finalizing, + types::PublishPhase::Published => protocol::AppendAttemptPhase::Published, + types::PublishPhase::Aborting => protocol::AppendAttemptPhase::Aborting, + types::PublishPhase::Cleaning => protocol::AppendAttemptPhase::Cleaning, + types::PublishPhase::Cleaned => protocol::AppendAttemptPhase::Cleaned, + types::PublishPhase::Quarantined => protocol::AppendAttemptPhase::Quarantined, + }; + let target = workspace_path(&parent.workbench_id, &parent.path)?; + let result = parent + .result + .as_ref() + .map(|published| -> Result<_, protocol::RpcFailure> { + Ok(protocol::OperationResult::ArtifactAppend( + protocol::AppendResult { + operation_id: parent.operation_id.into(), + publication_operation_id: parent.publication_operation_id.into(), + target: target.clone(), + workspace_incarnation_id: parent.workspace_incarnation_id.into(), + workspace_revision: published.workspace_revision.get(), + generation: published.path_generation.get(), + artifact_revision_id: parent.artifact_revision_id.into(), + logical_size: published.logical_size, + body_digest: protocol::DigestUri::new(published.body_digest_uri.clone()) + .map_err(|error| internal(error.to_string()))?, + }, + )) + }) + .transpose()?; + let state = if result.is_some() { + protocol::OperationState::Succeeded + } else if child.phase == types::PublishPhase::Quarantined { + protocol::OperationState::Quarantined + } else { + protocol::OperationState::Running + }; + let mut hasher = Sha256::new(); + hasher.update(b"nokv.append.state.v1\0"); + hasher.update( + parent + .encode() + .map_err(|error| internal(error.to_string()))?, + ); + hasher.update(child_status.token.state_digest.0); + Ok(protocol::OperationStatus { + append_preparation: Some(Box::new(protocol::AppendPreparation { + intent_digest: protocol::Digest(parent.intent_digest), + target, + workspace_incarnation_id: parent.workspace_incarnation_id.into(), + attempt: parent.attempt, + publication_operation_id: parent.publication_operation_id.into(), + artifact_revision_id: parent.artifact_revision_id.into(), + attempt_phase: phase, + activity_deadline_ms: child.activity_deadline_ms, + attempt_failure: child_status.failure.clone().or_else(|| { + child + .terminal_error + .as_ref() + .map(|error| operation_terminal_failure(&error.message)) + }), + })), + token: protocol::OperationToken { + operation_id: parent.operation_id.into(), + state_digest: protocol::Digest(hasher.finalize().into()), + }, + kind: protocol::OperationKind::ArtifactAppend, + publish_preparation: None, + commit_preparation: None, + restore_preparation: None, + state, + progress: child_status.progress, + result, + failure: (state == protocol::OperationState::Quarantined) + .then_some(child_status.failure) + .flatten(), + }) +} + fn publish_operation_status( operation: &meta::PublishOperationRecord, ) -> Result { @@ -4231,7 +4421,14 @@ fn publish_operation_status( ), }; Ok(protocol::OperationStatus { + append_preparation: None, publish_preparation: Some(Box::new(protocol::PublishPreparation { + append_attempt: operation.append_attempt.map(|binding| { + protocol::AppendAttemptBinding { + operation_id: binding.operation_id.into(), + attempt: binding.attempt, + } + }), append_intent_digest: operation.append_intent_digest.map(protocol::Digest), target: workspace_path(&operation.workbench_id, &operation.path)?, workspace_incarnation_id: operation.workspace_incarnation_id.into(), @@ -4311,6 +4508,7 @@ fn build_commit_operation_status( ), }; Ok(protocol::OperationStatus { + append_preparation: None, publish_preparation: None, token: protocol::OperationToken { operation_id: operation.operation_id.into(), @@ -5145,6 +5343,7 @@ fn restore_operation_status( ), }; Ok(protocol::OperationStatus { + append_preparation: None, publish_preparation: None, token: protocol::OperationToken { operation_id: operation.operation_id.into(), @@ -8211,6 +8410,7 @@ mod tests { first_request_fill, protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_attempt: None, append_intent_digest: None, operation_id, artifact_revision_id, @@ -9397,6 +9597,7 @@ mod tests { request_id: protocol::RequestIdentity([0x53; types::FIXED_ID_BYTES]), operation: protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_attempt: None, append_intent_digest: None, operation_id, artifact_revision_id, @@ -10145,6 +10346,7 @@ mod tests { request_fill, protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_attempt: None, append_intent_digest: None, operation_id, artifact_revision_id, @@ -10263,12 +10465,20 @@ mod tests { } fn stable_append_begin_request() -> protocol::BeginArtifactPublishRequest { - let artifact_revision_id = - protocol::ArtifactRevisionIdentity([0x84; types::FIXED_ID_BYTES]); + let binding = protocol::AppendAttemptBinding { + operation_id: protocol::OperationIdentity([0x83; types::FIXED_ID_BYTES]), + attempt: 0, + }; + let (operation_id, artifact_revision_id) = protocol::stable_append_attempt_identities( + root().into(), + binding.operation_id, + binding.attempt, + ); let seals = protocol::seal_artifact_publish_plan(artifact_revision_id, &[], &[]).unwrap(); protocol::BeginArtifactPublishRequest { + append_attempt: Some(binding), append_intent_digest: Some(protocol::Digest([0x85; types::SHA256_BYTES])), - operation_id: protocol::OperationIdentity([0x83; types::FIXED_ID_BYTES]), + operation_id, artifact_revision_id, target: protocol::WorkspacePath { workbench: protocol::WorkbenchName::new("stable-append").unwrap(), @@ -10379,6 +10589,9 @@ mod tests { { let mut changed = begin.clone(); changed.append_intent_digest = digest; + if digest.is_none() { + changed.append_attempt = None; + } let mismatch = executor .execute(&restore_rpc( 0x88 + index as u8, @@ -10545,6 +10758,7 @@ mod tests { request_id: protocol::RequestIdentity([0x64; types::FIXED_ID_BYTES]), operation: protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_attempt: None, append_intent_digest: None, operation_id, artifact_revision_id, @@ -10824,6 +11038,7 @@ mod tests { 0x74, protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_attempt: None, append_intent_digest: None, operation_id: operation_identity, artifact_revision_id: revision_identity, @@ -11058,6 +11273,7 @@ mod tests { 0x7C, protocol::WorkspaceRequest::BeginArtifactPublish( protocol::BeginArtifactPublishRequest { + append_attempt: None, append_intent_digest: None, operation_id: protocol::OperationIdentity( [0x7D; types::FIXED_ID_BYTES], diff --git a/crates/nokv-server/src/lib.rs b/crates/nokv-server/src/lib.rs index 017bc755b..39e6fcec7 100644 --- a/crates/nokv-server/src/lib.rs +++ b/crates/nokv-server/src/lib.rs @@ -24,7 +24,7 @@ mod test_support; pub use bootstrap::{bootstrap_shard, LeaseMode, OpenMode, RootAttach, ShardBoot, ShardOwner}; pub use error::ServerError; -pub use executor::MetadataWorkspaceRequestExecutor; +pub use executor::{MetadataWorkspaceRequestExecutor, MAX_APPEND_ACTIVITY_LEASE_MS}; #[cfg(feature = "restore-crash-test-support")] pub use executor::{ RestoreInitializationBarrier, RestoreInitializationBarrierEvidence, @@ -32,10 +32,10 @@ pub use executor::{ RestoreManifestPublicationEvidence, }; pub use lifecycle::{ - ArtifactLifecycleDeleter, LifecycleAbsenceProof, LifecycleCycleReport, - LifecycleDeleteDisposition, LifecycleDeleteError, LifecycleDeletePurpose, - LifecycleDeleteRequest, LifecycleDurabilityBarrier, LifecycleError, LifecycleObjectDeleter, - LifecycleRunner, LifecycleRunnerOptions, + ArtifactLifecycleCleaner, LifecycleCleanupDisposition, LifecycleCleanupError, + LifecycleCleanupProof, LifecycleCleanupPurpose, LifecycleCleanupRequest, LifecycleCycleReport, + LifecycleDurabilityBarrier, LifecycleError, LifecycleObjectCleaner, LifecycleRunner, + LifecycleRunnerOptions, }; pub use recovery_installer::{ install_recovery_log, validate_local_recovery_prefix, LocalRecoveryPrefixReport, diff --git a/crates/nokv-server/src/lifecycle.rs b/crates/nokv-server/src/lifecycle.rs index 9d45df358..3534ceacd 100644 --- a/crates/nokv-server/src/lifecycle.rs +++ b/crates/nokv-server/src/lifecycle.rs @@ -17,7 +17,7 @@ use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use nokv_meta::workspace as meta; -use nokv_object::{ArtifactObjectStore, ObjectDeleteOutcome, ObjectKey}; +use nokv_object::{ArtifactObjectStore, ObjectDeleteOutcome, ObjectKey, ObjectSealOutcome}; use nokv_protocol::RootRoute; use nokv_types::{ ArtifactRevisionId, BuildCommitPhase, CommitRetirePhase, CommitState, GcClaimState, GcPhase, @@ -34,49 +34,59 @@ static NEVER_SHUTDOWN: AtomicBool = AtomicBool::new(false); /// Why an authoritative metadata row requires provider deletion. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum LifecycleDeletePurpose { +pub enum LifecycleCleanupPurpose { AbortedPublication, + AbortedAppend, RevisionGarbageCollection, } -/// Provider-neutral deletion request. For an aborted multipart publication the -/// provider implementation must abort the named upload and prove the final key -/// absent before returning success. +/// Provider-neutral cleanup request. Failed append keys must be permanently +/// sealed; ordinary publication and revision GC require absence. Multipart +/// cleanup must also fence the named upload before returning success. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct LifecycleDeleteRequest { - pub purpose: LifecycleDeletePurpose, +pub struct LifecycleCleanupRequest { + pub purpose: LifecycleCleanupPurpose, pub object_key: String, pub multipart_upload_id: Option>, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum LifecycleDeleteDisposition { +pub enum LifecycleCleanupDisposition { Deleted, AlreadyAbsent, + /// The key remains occupied permanently and rejects delayed immutable creates. + Sealed, } /// Stable provider evidence consumed by the durable metadata state machine. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct LifecycleAbsenceProof { - pub disposition: LifecycleDeleteDisposition, +pub struct LifecycleCleanupProof { + pub disposition: LifecycleCleanupDisposition, pub digest: [u8; SHA256_BYTES], } -impl LifecycleAbsenceProof { - /// Build stable provider evidence for one authoritative delete request. +impl LifecycleCleanupProof { + fn matches_request(&self, request: &LifecycleCleanupRequest) -> bool { + let sealed = self.disposition == LifecycleCleanupDisposition::Sealed; + sealed == (request.purpose == LifecycleCleanupPurpose::AbortedAppend) + && *self == Self::from_cleanup_request(request, self.disposition) + } + + /// Build stable provider evidence for one authoritative cleanup request. /// The caller must pass the canonical object key received from the /// lifecycle worker; the digest binds the metadata schema, proof domain, - /// delete purpose, exact key, optional multipart id, and disposition. - pub fn from_delete_request( - request: &LifecycleDeleteRequest, - disposition: LifecycleDeleteDisposition, + /// cleanup purpose, exact key, optional multipart id, and disposition. + pub fn from_cleanup_request( + request: &LifecycleCleanupRequest, + disposition: LifecycleCleanupDisposition, ) -> Self { let mut hasher = Sha256::new(); - hasher.update(b"nokv.lifecycle.object-absence-proof.v1\0"); + hasher.update(b"nokv.lifecycle.object-cleanup-proof.v2\0"); hash_part(&mut hasher, meta::SCHEMA_ID.as_bytes()); hasher.update([match request.purpose { - LifecycleDeletePurpose::AbortedPublication => 1, - LifecycleDeletePurpose::RevisionGarbageCollection => 2, + LifecycleCleanupPurpose::AbortedPublication => 1, + LifecycleCleanupPurpose::RevisionGarbageCollection => 2, + LifecycleCleanupPurpose::AbortedAppend => 3, }]); hash_part(&mut hasher, request.object_key.as_bytes()); match request.multipart_upload_id.as_deref() { @@ -87,8 +97,9 @@ impl LifecycleAbsenceProof { } } hasher.update([match disposition { - LifecycleDeleteDisposition::Deleted => 1, - LifecycleDeleteDisposition::AlreadyAbsent => 2, + LifecycleCleanupDisposition::Deleted => 1, + LifecycleCleanupDisposition::AlreadyAbsent => 2, + LifecycleCleanupDisposition::Sealed => 3, }]); Self { disposition, @@ -98,21 +109,21 @@ impl LifecycleAbsenceProof { } /// Provider failure classification. `Retryable` is legal only when the -/// implementation knows deletion was not dispatched. Every uncertain outcome +/// implementation knows cleanup was not dispatched. Every uncertain outcome /// must be `Ambiguous` so the owning metadata operation is quarantined. #[derive(Clone, Debug, PartialEq, Eq)] -pub enum LifecycleDeleteError { +pub enum LifecycleCleanupError { Retryable { detail: String }, Ambiguous { evidence: Vec }, } /// Narrow provider boundary used by lifecycle workers. It deliberately has no /// list operation. -pub trait LifecycleObjectDeleter: Send + Sync { - fn delete( +pub trait LifecycleObjectCleaner: Send + Sync { + fn cleanup( &self, - request: &LifecycleDeleteRequest, - ) -> Result; + request: &LifecycleCleanupRequest, + ) -> Result; } /// Required recovery barrier for background metadata mutations. A lifecycle @@ -128,53 +139,68 @@ impl LifecycleDurabilityBarrier for RecoveryPublisher { } } -/// Server-owned adapter from immutable object-store primitives to the durable -/// lifecycle deletion contract. +/// Server-owned adapter from object-store primitives to durable cleanup. +/// Failed append cleanup permanently seals keys and never dispatches DELETE; +/// ordinary publication and revision GC retain physical deletion semantics. /// /// The object boundary cannot prove that a failed destructive call was not /// dispatched, and it currently exposes no multipart-abort operation. Both /// cases therefore fail closed as ambiguous so the metadata operation is /// quarantined instead of retrying an uncertain deletion. #[derive(Clone)] -pub struct ArtifactLifecycleDeleter { +pub struct ArtifactLifecycleCleaner { store: Arc, } -impl ArtifactLifecycleDeleter { +impl ArtifactLifecycleCleaner { pub fn new(store: Arc) -> Self { Self { store } } } -impl LifecycleObjectDeleter for ArtifactLifecycleDeleter +impl LifecycleObjectCleaner for ArtifactLifecycleCleaner where Store: ArtifactObjectStore + Send + Sync + ?Sized, { - fn delete( + fn cleanup( &self, - request: &LifecycleDeleteRequest, - ) -> Result { + request: &LifecycleCleanupRequest, + ) -> Result { if request.multipart_upload_id.is_some() { - return Err(LifecycleDeleteError::Ambiguous { + return Err(LifecycleCleanupError::Ambiguous { evidence: b"configured object provider has no multipart-abort boundary".to_vec(), }); } let key = ObjectKey::new(request.object_key.clone()).map_err(|error| { - LifecycleDeleteError::Ambiguous { + LifecycleCleanupError::Ambiguous { evidence: format!("authoritative lifecycle object key is invalid: {error}") .into_bytes(), } })?; - let disposition = match self.store.delete(&key) { - Ok(ObjectDeleteOutcome::Deleted) => LifecycleDeleteDisposition::Deleted, - Ok(ObjectDeleteOutcome::Absent) => LifecycleDeleteDisposition::AlreadyAbsent, - Err(error) => { - return Err(LifecycleDeleteError::Ambiguous { - evidence: format!("object delete outcome is uncertain: {error}").into_bytes(), - }); + let disposition = if request.purpose == LifecycleCleanupPurpose::AbortedAppend { + match self.store.seal_immutable(&key) { + Ok(ObjectSealOutcome::Sealed | ObjectSealOutcome::AlreadySealed) => { + LifecycleCleanupDisposition::Sealed + } + Err(error) => { + return Err(LifecycleCleanupError::Ambiguous { + evidence: format!("object seal outcome is uncertain: {error}").into_bytes(), + }) + } + } + } else { + match self.store.delete(&key) { + Ok(ObjectDeleteOutcome::Deleted) => LifecycleCleanupDisposition::Deleted, + Ok(ObjectDeleteOutcome::Absent) => LifecycleCleanupDisposition::AlreadyAbsent, + Err(error) => { + return Err(LifecycleCleanupError::Ambiguous { + evidence: format!("object delete outcome is uncertain: {error}") + .into_bytes(), + }) + } } }; - Ok(LifecycleAbsenceProof::from_delete_request( + Ok(LifecycleCleanupProof::from_cleanup_request( request, disposition, )) @@ -232,6 +258,7 @@ impl LifecycleRunnerOptions { pub struct LifecycleCycleReport { pub metadata_transitions: u64, pub provider_deletions: u64, + pub provider_seals: u64, pub quarantined_operations: u64, pub deferred_operations: u64, } @@ -302,6 +329,7 @@ impl From for LifecycleError { #[derive(Default)] struct LifecycleCursors { + publication_index_verified: bool, publish_operation: Option>, restore_operation: Option>, build_operation: Option>, @@ -320,7 +348,7 @@ pub struct LifecycleRunner { registry: Arc, route: RootRoute, owner_loss: OwnerLossSignal, - objects: Arc, + objects: Arc, durability: Arc, options: LifecycleRunnerOptions, cursors: Mutex, @@ -334,7 +362,7 @@ impl LifecycleRunner { registry: Arc, route: RootRoute, owner_loss: OwnerLossSignal, - objects: Arc, + objects: Arc, durability: Arc, options: LifecycleRunnerOptions, ) -> Result { @@ -401,6 +429,13 @@ impl LifecycleRunner { .lock() .map_err(|_| LifecycleError::WorkerLockPoisoned)?; let mut report = LifecycleCycleReport::default(); + if !cursors.publication_index_verified { + let context = self.publication_context(b"verify-append-publication-state", &[])?; + meta::PublicationService::new(&self.meta) + .verify_append_publication_state(context) + .map_err(|error| state("verify append publication state", error))?; + cursors.publication_index_verified = true; + } self.recover_publications(&mut cursors, observed_now_ms, &mut report)?; self.reap_snapshots(&mut cursors, observed_now_ms, &mut report)?; self.recover_restores(&mut cursors, &mut report)?; @@ -511,7 +546,7 @@ impl LifecycleRunner { observed_now_ms: u64, report: &mut LifecycleCycleReport, ) -> Result<(), LifecycleError> { - let prefix = meta::operation_prefix(self.root_id(), OperationKind::Publish); + let prefix = meta::operation_prefix(self.root_id(), OperationKind::ActivePublish); let rows = self.scan_page( meta::MetadataFamily::Operation, &prefix, @@ -524,9 +559,25 @@ impl LifecycleRunner { ); for item in rows { let operation_id = - meta::decode_operation_key(self.root_id(), OperationKind::Publish, &item.key) + meta::decode_operation_key(self.root_id(), OperationKind::ActivePublish, &item.key) .ok_or_else(|| corrupt("publish operation key", "malformed root/kind key"))?; - let operation = meta::PublishOperationRecord::decode(&item.value) + if item.value != [1] { + return Err(corrupt("active publish index", "unknown marker encoding")); + } + let context = self.read_context()?; + let key = meta::operation_key(self.root_id(), OperationKind::Publish, operation_id); + let payload = self + .meta + .read_at( + context.root_id, + context.placement_generation, + context.owner_epoch, + meta::MetadataFamily::Operation, + &key, + context.read_version, + )? + .ok_or_else(|| corrupt("active publish index", "canonical operation is absent"))?; + let operation = meta::PublishOperationRecord::decode(&payload) .map_err(|error| corrupt("publish operation", error.to_string()))?; if operation.operation_id != operation_id { return Err(corrupt( @@ -689,34 +740,57 @@ impl LifecycleRunner { report, ); } - if !matches!( - (expected.provider_state, expected.cleanup_state), - (StagedProviderState::Aborted, StagedCleanupState::Deleted) - ) { + let append = operation.append_attempt.is_some(); + let completed_cleanup = if append { + StagedCleanupState::Sealed + } else { + StagedCleanupState::Deleted + }; + if expected.provider_state != StagedProviderState::Aborted + || expected.cleanup_state != completed_cleanup + { self.require_current_owner()?; self.publish_current()?; #[cfg(test)] self.run_before_provider_delete_test_hook(); let _operation_permit = self.enter_destructive_operation()?; - let request = LifecycleDeleteRequest { - purpose: LifecycleDeletePurpose::AbortedPublication, + let request = LifecycleCleanupRequest { + purpose: if append { + LifecycleCleanupPurpose::AbortedAppend + } else { + LifecycleCleanupPurpose::AbortedPublication + }, object_key: expected.object_key.clone(), multipart_upload_id: expected.multipart_upload_id.clone(), }; - match self.objects.delete(&request) { - Ok(_) => report.provider_deletions += 1, - Err(LifecycleDeleteError::Retryable { .. }) => { + match self.objects.cleanup(&request) { + Ok(proof) if proof.matches_request(&request) => { + if append { + report.provider_seals += 1; + } else { + report.provider_deletions += 1; + } + } + Ok(_) => { + return self.quarantine_publication( + operation, + b"provider cleanup proof does not match the requested purpose" + .to_vec(), + report, + ) + } + Err(LifecycleCleanupError::Retryable { .. }) => { report.deferred_operations += 1; return Ok(()); } - Err(LifecycleDeleteError::Ambiguous { evidence }) => { + Err(LifecycleCleanupError::Ambiguous { evidence }) => { return self.quarantine_publication(operation, evidence, report); } } } let mut next = expected.clone(); next.provider_state = StagedProviderState::Aborted; - next.cleanup_state = StagedCleanupState::Deleted; + next.cleanup_state = completed_cleanup; updates.push(meta::StagedObjectUpdate { expected, next }); } let encoded = operation @@ -1401,21 +1475,29 @@ impl LifecycleRunner { #[cfg(test)] self.run_before_provider_delete_test_hook(); let _operation_permit = self.enter_destructive_operation()?; - let request = LifecycleDeleteRequest { - purpose: LifecycleDeletePurpose::RevisionGarbageCollection, + let request = LifecycleCleanupRequest { + purpose: LifecycleCleanupPurpose::RevisionGarbageCollection, object_key: entry.row.object_key.clone(), multipart_upload_id: None, }; - match self.objects.delete(&request) { - Ok(proof) => { + match self.objects.cleanup(&request) { + Ok(proof) if proof.matches_request(&request) => { report.provider_deletions += 1; Some(proof.digest) } - Err(LifecycleDeleteError::Retryable { .. }) => { + Ok(_) => { + return self.quarantine_gc( + operation, + b"provider cleanup proof does not prove revision object absence" + .to_vec(), + report, + ) + } + Err(LifecycleCleanupError::Retryable { .. }) => { report.deferred_operations += 1; return Ok(()); } - Err(LifecycleDeleteError::Ambiguous { evidence }) => { + Err(LifecycleCleanupError::Ambiguous { evidence }) => { return self.quarantine_gc(operation, evidence, report); } } @@ -1886,31 +1968,71 @@ mod tests { } } - fn lifecycle_delete_request() -> LifecycleDeleteRequest { - LifecycleDeleteRequest { - purpose: LifecycleDeletePurpose::RevisionGarbageCollection, + fn lifecycle_delete_request() -> LifecycleCleanupRequest { + LifecycleCleanupRequest { + purpose: LifecycleCleanupPurpose::RevisionGarbageCollection, object_key: "nokv/artifacts/object".to_owned(), multipart_upload_id: None, } } + #[test] + fn aborted_append_cleanup_seals_without_delete_and_rejects_late_create() { + for already_uploaded in [false, true] { + let store = Arc::new(nokv_object::MemoryArtifactStore::new()); + let key = ObjectKey::new("test/failed-append-key".to_owned()).unwrap(); + if already_uploaded { + store.create_immutable(&key, b"delayed payload").unwrap(); + } + let cleaner = ArtifactLifecycleCleaner::new(Arc::clone(&store)); + let request = LifecycleCleanupRequest { + purpose: LifecycleCleanupPurpose::AbortedAppend, + object_key: key.as_str().to_owned(), + multipart_upload_id: None, + }; + for _ in 0..2 { + let proof = cleaner.cleanup(&request).unwrap(); + assert_eq!(proof.disposition, LifecycleCleanupDisposition::Sealed); + assert!(proof.matches_request(&request)); + assert_eq!(store.head(&key).unwrap().unwrap().size, 0); + assert!(matches!( + store.create_immutable(&key, b"delayed payload"), + Err(ObjectError::ImmutableCollision { .. }) + )); + } + assert_eq!(store.stats().unwrap().deletes, 0); + let absence = LifecycleCleanupProof::from_cleanup_request( + &request, + LifecycleCleanupDisposition::AlreadyAbsent, + ); + assert!(!absence.matches_request(&request)); + let mut gc = request.clone(); + gc.purpose = LifecycleCleanupPurpose::RevisionGarbageCollection; + let sealed = LifecycleCleanupProof::from_cleanup_request( + &gc, + LifecycleCleanupDisposition::Sealed, + ); + assert!(!sealed.matches_request(&gc)); + } + } + #[test] fn artifact_lifecycle_deleter_proves_deleted_object_absent() { let store = Arc::new(FakeArtifactStore { delete_result: Ok(ObjectDeleteOutcome::Deleted), delete_calls: AtomicUsize::new(0), }); - let deleter = ArtifactLifecycleDeleter::new(Arc::clone(&store)); + let deleter = ArtifactLifecycleCleaner::new(Arc::clone(&store)); let request = lifecycle_delete_request(); - let proof = deleter.delete(&request).unwrap(); + let proof = deleter.cleanup(&request).unwrap(); - assert_eq!(proof.disposition, LifecycleDeleteDisposition::Deleted); + assert_eq!(proof.disposition, LifecycleCleanupDisposition::Deleted); assert_eq!( proof, - LifecycleAbsenceProof::from_delete_request( + LifecycleCleanupProof::from_cleanup_request( &request, - LifecycleDeleteDisposition::Deleted, + LifecycleCleanupDisposition::Deleted, ) ); assert_eq!(store.delete_calls.load(Ordering::SeqCst), 1); @@ -1922,12 +2044,15 @@ mod tests { delete_result: Ok(ObjectDeleteOutcome::Absent), delete_calls: AtomicUsize::new(0), }); - let deleter = ArtifactLifecycleDeleter::new(Arc::clone(&store)); + let deleter = ArtifactLifecycleCleaner::new(Arc::clone(&store)); let request = lifecycle_delete_request(); - let proof = deleter.delete(&request).unwrap(); + let proof = deleter.cleanup(&request).unwrap(); - assert_eq!(proof.disposition, LifecycleDeleteDisposition::AlreadyAbsent); + assert_eq!( + proof.disposition, + LifecycleCleanupDisposition::AlreadyAbsent + ); assert_eq!(store.delete_calls.load(Ordering::SeqCst), 1); } @@ -1937,13 +2062,13 @@ mod tests { delete_result: Ok(ObjectDeleteOutcome::Deleted), delete_calls: AtomicUsize::new(0), }); - let deleter = ArtifactLifecycleDeleter::new(Arc::clone(&store)); + let deleter = ArtifactLifecycleCleaner::new(Arc::clone(&store)); let mut request = lifecycle_delete_request(); request.multipart_upload_id = Some(b"upload-id".to_vec()); - let error = deleter.delete(&request).unwrap_err(); + let error = deleter.cleanup(&request).unwrap_err(); - assert!(matches!(error, LifecycleDeleteError::Ambiguous { .. })); + assert!(matches!(error, LifecycleCleanupError::Ambiguous { .. })); assert_eq!(store.delete_calls.load(Ordering::SeqCst), 0); } @@ -1957,11 +2082,11 @@ mod tests { }), delete_calls: AtomicUsize::new(0), }); - let deleter = ArtifactLifecycleDeleter::new(Arc::clone(&store)); + let deleter = ArtifactLifecycleCleaner::new(Arc::clone(&store)); - let error = deleter.delete(&lifecycle_delete_request()).unwrap_err(); + let error = deleter.cleanup(&lifecycle_delete_request()).unwrap_err(); - assert!(matches!(error, LifecycleDeleteError::Ambiguous { .. })); + assert!(matches!(error, LifecycleCleanupError::Ambiguous { .. })); assert_eq!(store.delete_calls.load(Ordering::SeqCst), 1); } @@ -1971,11 +2096,11 @@ mod tests { object_keys: Mutex>, } - impl LifecycleObjectDeleter for FakeDeleter { - fn delete( + impl LifecycleObjectCleaner for FakeDeleter { + fn cleanup( &self, - request: &LifecycleDeleteRequest, - ) -> Result { + request: &LifecycleCleanupRequest, + ) -> Result { self.calls.fetch_add(1, Ordering::SeqCst); self.object_keys .lock() @@ -1983,16 +2108,16 @@ mod tests { .push(request.object_key.clone()); assert_eq!( request.purpose, - LifecycleDeletePurpose::RevisionGarbageCollection + LifecycleCleanupPurpose::RevisionGarbageCollection ); if self.ambiguous { - Err(LifecycleDeleteError::Ambiguous { + Err(LifecycleCleanupError::Ambiguous { evidence: b"provider timed out after delete dispatch".to_vec(), }) } else { - Ok(LifecycleAbsenceProof::from_delete_request( + Ok(LifecycleCleanupProof::from_cleanup_request( request, - LifecycleDeleteDisposition::Deleted, + LifecycleCleanupDisposition::Deleted, )) } } @@ -2043,8 +2168,38 @@ mod tests { store: &meta::MetaShard, request: u8, action: meta::RootFenceAction, - mutations: Vec, + mut mutations: Vec, ) -> meta::MetadataCommand { + // Fixtures seed complete current-format state, including the active + // publication index maintained atomically by production admission. + let active = mutations + .iter() + .filter_map(|mutation| { + let meta::CommandMutation::Put { + family: meta::MetadataFamily::Operation, + key, + value, + } = mutation + else { + return None; + }; + let id = meta::decode_operation_key(root(), OperationKind::Publish, key)?; + let operation = meta::PublishOperationRecord::decode(value).ok()?; + matches!( + operation.phase, + PublishPhase::Uploading + | PublishPhase::Finalizing + | PublishPhase::Aborting + | PublishPhase::Cleaning + ) + .then(|| meta::CommandMutation::Put { + family: meta::MetadataFamily::Operation, + key: meta::operation_key(root(), OperationKind::ActivePublish, id), + value: vec![1], + }) + }) + .collect::>(); + mutations.extend(active); let predicates = mutations .iter() .map(|mutation| match mutation { @@ -2340,29 +2495,29 @@ mod tests { } #[test] - fn absence_proof_constructor_is_deterministic_and_domain_separated() { - let request = LifecycleDeleteRequest { - purpose: LifecycleDeletePurpose::AbortedPublication, + fn cleanup_proof_constructor_is_deterministic_and_domain_separated() { + let request = LifecycleCleanupRequest { + purpose: LifecycleCleanupPurpose::AbortedPublication, object_key: "nokv/artifacts/01010101010101010101010101010101/02020202020202020202020202020202/03030303030303030303030303030303/blocks/0000000000000007".to_owned(), multipart_upload_id: Some(b"multipart-7".to_vec()), }; - let proof = LifecycleAbsenceProof::from_delete_request( + let proof = LifecycleCleanupProof::from_cleanup_request( &request, - LifecycleDeleteDisposition::Deleted, + LifecycleCleanupDisposition::Deleted, ); assert_eq!( proof.digest, [ - 0xc0, 0x04, 0xeb, 0x4f, 0x8c, 0x10, 0x64, 0x40, 0x85, 0xfd, 0xce, 0x1a, 0xc8, 0x95, - 0xd4, 0x94, 0xa2, 0xd0, 0x19, 0x4a, 0xd0, 0xa1, 0xcb, 0x64, 0xba, 0x03, 0x86, 0xa2, - 0x21, 0x88, 0xb9, 0x85, + 0xdd, 0xcf, 0x71, 0xa4, 0x41, 0x4e, 0x6c, 0x96, 0xc9, 0x0a, 0x30, 0xf1, 0xc2, 0x28, + 0x78, 0x54, 0x50, 0x2e, 0x2c, 0x3f, 0x15, 0x56, 0x2c, 0xf8, 0xbd, 0xed, 0xd2, 0x69, + 0xd0, 0x33, 0x7c, 0xe1 ] ); assert_eq!( proof, - LifecycleAbsenceProof::from_delete_request( + LifecycleCleanupProof::from_cleanup_request( &request, - LifecycleDeleteDisposition::Deleted, + LifecycleCleanupDisposition::Deleted, ) ); let unbound_key_digest: [u8; SHA256_BYTES] = @@ -2370,12 +2525,12 @@ mod tests { assert_ne!(proof.digest, unbound_key_digest); let mut changed = request.clone(); - changed.purpose = LifecycleDeletePurpose::RevisionGarbageCollection; + changed.purpose = LifecycleCleanupPurpose::RevisionGarbageCollection; assert_ne!( proof.digest, - LifecycleAbsenceProof::from_delete_request( + LifecycleCleanupProof::from_cleanup_request( &changed, - LifecycleDeleteDisposition::Deleted, + LifecycleCleanupDisposition::Deleted, ) .digest ); @@ -2383,17 +2538,17 @@ mod tests { changed.multipart_upload_id = None; assert_ne!( proof.digest, - LifecycleAbsenceProof::from_delete_request( + LifecycleCleanupProof::from_cleanup_request( &changed, - LifecycleDeleteDisposition::Deleted, + LifecycleCleanupDisposition::Deleted, ) .digest ); assert_ne!( proof.digest, - LifecycleAbsenceProof::from_delete_request( + LifecycleCleanupProof::from_cleanup_request( &request, - LifecycleDeleteDisposition::AlreadyAbsent, + LifecycleCleanupDisposition::AlreadyAbsent, ) .digest ); @@ -2640,14 +2795,14 @@ mod tests { calls: AtomicUsize, } - impl LifecycleObjectDeleter for BlockingDeleter { - fn delete( + impl LifecycleObjectCleaner for BlockingDeleter { + fn cleanup( &self, - request: &LifecycleDeleteRequest, - ) -> Result { + request: &LifecycleCleanupRequest, + ) -> Result { assert_eq!( request.purpose, - LifecycleDeletePurpose::RevisionGarbageCollection, + LifecycleCleanupPurpose::RevisionGarbageCollection, ); if let Some(entered) = self @@ -2669,9 +2824,9 @@ mod tests { self.calls.fetch_add(1, Ordering::SeqCst); - Ok(LifecycleAbsenceProof::from_delete_request( + Ok(LifecycleCleanupProof::from_cleanup_request( request, - LifecycleDeleteDisposition::Deleted, + LifecycleCleanupDisposition::Deleted, )) } } @@ -2849,16 +3004,16 @@ mod tests { calls: AtomicUsize, } - impl LifecycleObjectDeleter for AbortedCleanupRaceDeleter { - fn delete( + impl LifecycleObjectCleaner for AbortedCleanupRaceDeleter { + fn cleanup( &self, - request: &LifecycleDeleteRequest, - ) -> Result { - assert_eq!(request.purpose, LifecycleDeletePurpose::AbortedPublication); + request: &LifecycleCleanupRequest, + ) -> Result { + assert_eq!(request.purpose, LifecycleCleanupPurpose::AbortedPublication); self.calls.fetch_add(1, Ordering::SeqCst); - Ok(LifecycleAbsenceProof::from_delete_request( + Ok(LifecycleCleanupProof::from_cleanup_request( request, - LifecycleDeleteDisposition::Deleted, + LifecycleCleanupDisposition::Deleted, )) } } @@ -2921,6 +3076,7 @@ mod tests { cleanup_state: StagedCleanupState::Owned, }]; let mut operation = meta::PublishOperationRecord { + append_attempt: None, append_intent_digest: None, operation_id: OperationId::from_bytes([0x74; FIXED_ID_BYTES]), identity_digest: [0; SHA256_BYTES], @@ -3231,6 +3387,7 @@ mod tests { let operation_id = OperationId::from_bytes([0x44; FIXED_ID_BYTES]); let mut operation = meta::PublishOperationRecord { + append_attempt: None, append_intent_digest: None, operation_id, identity_digest: [0; SHA256_BYTES], @@ -3517,6 +3674,7 @@ mod tests { }]; let template = |operation_id_byte: u8| { let mut operation = meta::PublishOperationRecord { + append_attempt: None, append_intent_digest: None, operation_id: OperationId::from_bytes([operation_id_byte; FIXED_ID_BYTES]), identity_digest: [0; SHA256_BYTES], @@ -3696,7 +3854,7 @@ mod tests { registry, route(), OwnerLossSignal::default(), - Arc::new(ArtifactLifecycleDeleter::new(Arc::clone(&objects))), + Arc::new(ArtifactLifecycleCleaner::new(Arc::clone(&objects))), test_durability(), LifecycleRunnerOptions { scan_page_size: 8, @@ -3814,6 +3972,7 @@ mod tests { }]; let template = |operation_id_byte: u8| { let mut operation = meta::PublishOperationRecord { + append_attempt: None, append_intent_digest: None, operation_id: OperationId::from_bytes([operation_id_byte; FIXED_ID_BYTES]), identity_digest: [0; SHA256_BYTES], @@ -4019,7 +4178,7 @@ mod tests { registry, route(), OwnerLossSignal::default(), - Arc::new(ArtifactLifecycleDeleter::new(Arc::clone(&objects))), + Arc::new(ArtifactLifecycleCleaner::new(Arc::clone(&objects))), test_durability(), LifecycleRunnerOptions { scan_page_size: 8, diff --git a/crates/nokv-types/src/workspace.rs b/crates/nokv-types/src/workspace.rs index 5e13e7873..71147d89d 100644 --- a/crates/nokv-types/src/workspace.rs +++ b/crates/nokv-types/src/workspace.rs @@ -658,6 +658,8 @@ durable_enum! { CommitRetire = 4, Gc = 5, RegisterGenericIndex = 6, + Append = 7, + ActivePublish = 8, } } @@ -780,6 +782,8 @@ durable_enum! { DeletePending = 2, Deleted = 3, Quarantined = 4, + /// Failed append key is permanently occupied by an immutable seal. + Sealed = 5, } } @@ -1176,6 +1180,8 @@ mod tests { (OperationKind::CommitRetire, 4), (OperationKind::Gc, 5), (OperationKind::RegisterGenericIndex, 6), + (OperationKind::Append, 7), + (OperationKind::ActivePublish, 8), ]); assert_durable_registry(&[ (GenericIndexGenerationState::Building, 1), @@ -1258,6 +1264,7 @@ mod tests { (StagedCleanupState::DeletePending, 2), (StagedCleanupState::Deleted, 3), (StagedCleanupState::Quarantined, 4), + (StagedCleanupState::Sealed, 5), ]); assert_durable_registry(&[ (GcClaimState::Candidate, 1), diff --git a/crates/nokv/src/backend.rs b/crates/nokv/src/backend.rs index 6362b23ff..428c7f34a 100644 --- a/crates/nokv/src/backend.rs +++ b/crates/nokv/src/backend.rs @@ -105,49 +105,17 @@ impl CliWorkbenchBackend { } } - /// Append under a caller-owned identity, observing but never creating the workspace. - #[allow(clippy::too_many_arguments)] + /// Execute an append after metadata-only recovery and input validation. pub fn append_idempotent( &self, - operation_id: wire::OperationIdentity, - target: wire::WorkspacePath, - expected_workspace_incarnation_id: Option, - create_content_type: wire::ContentType, - content_type: Option, - max_logical_size: Option, + options: IdempotentAppendOptions, delta: &[u8], - ) -> Result< - ( - nokv_client::ClientCall, - wire::WorkspaceIdentity, - ), - agent::BackendError, - > { - let incarnation = match expected_workspace_incarnation_id { - Some(incarnation) => incarnation, - None => { - self.client - .get_workspace(wire::GetWorkspaceRequest { - workbench: target.workbench.clone(), - }) - .map_err(map_client_error)? - .value - .workspace_incarnation_id - } - }; - let mut options = - IdempotentAppendOptions::new(operation_id, target, incarnation, create_content_type); - if let Some(content_type) = content_type { - options = options.with_content_type(content_type); - } - if let Some(max_logical_size) = max_logical_size { - options = options.with_max_logical_size(max_logical_size); - } + ) -> Result, agent::BackendError> { + let incarnation = options.expected_workspace_incarnation_id; self.client .append_artifact_idempotent(self.objects.as_ref(), options, delta) - .map(|call| (call, incarnation)) .map_err(|error| { - let mut mapped = map_client_error(error); + let mut mapped = map_append_client_error(error); mapped.details["workspace_incarnation_id"] = json!(super::encode_lowercase_hex(&incarnation.0)); mapped @@ -2806,6 +2774,18 @@ fn map_manifest_read_error(error: ClientError, path: &agent::ScopedPath) -> agen map_client_error(error) } +pub(crate) fn map_append_client_error(error: ClientError) -> agent::BackendError { + if let ClientError::InvalidOptions(message) = error { + return agent::BackendError::new( + agent::BackendErrorKind::Other("InvalidArgument".to_owned()), + message, + false, + json!({}), + ); + } + map_client_error(error) +} + fn map_client_error(error: ClientError) -> agent::BackendError { if let ClientError::AppendUnresolved { operation_id, @@ -3690,6 +3670,7 @@ mod tests { fn running_restore_status(fixture: &RestorePlanFixture) -> wire::OperationStatus { let request = prepare_restore_request(fixture); wire::OperationStatus { + append_preparation: None, token: wire::OperationToken { operation_id: fixture.preparation.operation_id, state_digest: wire::Digest([0x61; 32]), @@ -3855,6 +3836,7 @@ mod tests { lineage_projection: Vec::new(), }; let commit_status = wire::OperationStatus { + append_preparation: None, token: wire::OperationToken { operation_id: identities.operation_id, state_digest: wire::Digest([0x61; 32]), @@ -3885,12 +3867,14 @@ mod tests { failure: None, }; let publish_status = wire::OperationStatus { + append_preparation: None, token: wire::OperationToken { operation_id: identities.manifest_publish_operation_id, state_digest: wire::Digest([0x63; 32]), }, kind: wire::OperationKind::ArtifactPublish, publish_preparation: Some(Box::new(wire::PublishPreparation { + append_attempt: None, append_intent_digest: None, target: manifest_target.clone(), workspace_incarnation_id: binding.workspace_incarnation_id, @@ -7079,31 +7063,57 @@ mod tests { #[test] fn native_append_does_not_admit_a_missing_workspace() { - let (backend, requests, server) = scripted_backend(vec![not_found_failure()]); + let (backend, requests, server) = + scripted_backend(vec![not_found_failure(), not_found_failure()]); let error = backend - .append_idempotent( + .client + .resolve_append_workspace_incarnation( wire::OperationIdentity([0xaa; 16]), - wire::WorkspacePath { + &wire::WorkspacePath { workbench: wire::WorkbenchName::new("absent").unwrap(), path: wire::RelativePath::new("logs/events.jsonl").unwrap(), }, None, - wire::ContentType::new("text/plain").unwrap(), - None, - None, - b"event", ) + .map_err(map_client_error) .unwrap_err(); assert_eq!(error.kind, agent::BackendErrorKind::NotFound); server.join().unwrap(); let requests = requests.lock().unwrap(); - assert_eq!(requests.len(), 1); + assert_eq!(requests.len(), 2); assert!(matches!( &requests[0].operation, + wire::WorkspaceRequest::GetOperation(_) + )); + assert!(matches!( + &requests[1].operation, wire::WorkspaceRequest::GetWorkspace(_) )); } + #[test] + fn append_local_options_use_input_errors_without_reclassifying_uncertain_results() { + let error = + map_append_client_error(ClientError::InvalidOptions("block size is zero".to_owned())); + assert_eq!( + error.kind, + agent::BackendErrorKind::Other("InvalidArgument".to_owned()) + ); + assert!(!error.retryable); + let unresolved = map_append_client_error(ClientError::AppendUnresolved { + operation_id: wire::OperationIdentity([0xaa; 16]), + state: None, + source: Box::new(ClientError::InvalidOptions( + "unknown publication outcome".to_owned(), + )), + }); + assert_eq!( + unresolved.kind, + agent::BackendErrorKind::Other("AppendUnresolved".to_owned()) + ); + assert_eq!(unresolved.details["operation_id"], "aa".repeat(16)); + } + #[test] fn unresolved_append_error_preserves_identity_state_and_underlying_cause() { let error = map_client_error(ClientError::AppendUnresolved { diff --git a/crates/nokv/src/cli.rs b/crates/nokv/src/cli.rs index 9891d1259..a1fccd876 100644 --- a/crates/nokv/src/cli.rs +++ b/crates/nokv/src/cli.rs @@ -16,6 +16,7 @@ pub const DEFAULT_MAX_ARTIFACT_BYTES: usize = 16 * 1024 * 1024; pub const DEFAULT_ETCD_KEY_PREFIX: &str = "/nokv/control"; pub const DEFAULT_ETCD_LEASE_TTL_SECONDS: i64 = 10; pub const DEFAULT_LIFECYCLE_INTERVAL_MILLIS: u64 = 1_000; +pub const DEFAULT_APPEND_ACTIVITY_LEASE_MS: u64 = 1_800_000; pub const DEFAULT_HANDSHAKE_TIMEOUT_MILLIS: u64 = 5_000; pub const DEFAULT_MAX_INFLIGHT_CONNECTIONS: usize = 256; @@ -74,6 +75,7 @@ pub struct ServerConfig { pub node_id: Option, pub metadata_store: Option, pub lifecycle_interval_millis: u64, + pub append_activity_lease_ms: u64, pub recovery_publication: RecoveryPublicationConfig, } @@ -133,6 +135,9 @@ pub enum Command { content_type: Option, }, WorkspacePath(WorkspacePathCommand), + OperationStatus { + operation_id: [u8; 16], + }, Provision { logical_shard_id: String, adopt_legacy_object_namespace: bool, @@ -164,6 +169,7 @@ pub enum WorkspacePathCommand { payload: AppendPayload, content_type: Option, max_logical_size: Option, + block_size: usize, }, Rename { workbench: String, @@ -314,6 +320,7 @@ impl Default for ServerConfig { node_id: None, metadata_store: None, lifecycle_interval_millis: DEFAULT_LIFECYCLE_INTERVAL_MILLIS, + append_activity_lease_ms: DEFAULT_APPEND_ACTIVITY_LEASE_MS, recovery_publication: RecoveryPublicationConfig::LocalOnly, } } @@ -324,6 +331,7 @@ pub fn parse(arguments: impl IntoIterator) -> Result) -> Result { server.bind = parse_address("--bind", next_value(&mut arguments, &argument)?)?; } + "--append-activity-lease-ms" => { + let value = next_value(&mut arguments, &argument)?; + server.append_activity_lease_ms = + parse_number("--append-activity-lease-ms", value.clone())?; + if !(1_000..=nokv_server::MAX_APPEND_ACTIVITY_LEASE_MS) + .contains(&server.append_activity_lease_ms) + { + return Err(CliError::InvalidOption { + option: "--append-activity-lease-ms", + value, + }); + } + append_activity_lease_explicit = true; + } "--handshake-timeout-millis" => { server.handshake_timeout_millis = parse_number( "--handshake-timeout-millis", @@ -516,6 +538,11 @@ pub fn parse(arguments: impl IntoIterator) -> Result) -> Result parse_collect(arguments), "workspace-path" => parse_workspace_path(arguments), + "operation" => parse_operation(arguments), "provision" => parse_provision(arguments), "serve" => Ok(Command::Serve), "schema" => Ok(Command::Schema), @@ -614,6 +643,21 @@ fn parse_command( } } +fn parse_operation(arguments: &mut impl Iterator) -> Result { + let operation = arguments + .next() + .ok_or(CliError::MissingArgument("operation command"))?; + if operation != "status" { + return Err(CliError::UnknownCommand(format!("operation {operation}"))); + } + let identity = arguments + .next() + .ok_or(CliError::MissingArgument("operation identity"))?; + Ok(Command::OperationStatus { + operation_id: parse_append_identity("operation identity", identity)?, + }) +} + fn parse_mcp(arguments: &mut impl Iterator) -> Result { let mut profile = None; while let Some(argument) = arguments.next() { @@ -724,6 +768,7 @@ fn parse_workspace_append( let mut payload = None; let mut content_type = None; let mut max_logical_size = None; + let mut block_size = None; while let Some(argument) = arguments.next() { match argument.as_str() { "--operation-id" if operation_id.is_none() => { @@ -760,13 +805,20 @@ fn parse_workspace_append( next_value(arguments, &argument)?, )?); } + "--block-size" if block_size.is_none() => { + block_size = Some(parse_number( + "--block-size", + next_value(arguments, &argument)?, + )?); + } "--operation-id" | "--expected-workspace-incarnation-id" | "--text" | "--base64" | "--file" | "--content-type" - | "--max-logical-size" => return Err(CliError::UnexpectedArgument(argument)), + | "--max-logical-size" + | "--block-size" => return Err(CliError::UnexpectedArgument(argument)), _ if argument.starts_with("--") => return Err(CliError::UnknownOption(argument)), _ => return Err(CliError::UnexpectedArgument(argument)), } @@ -782,6 +834,7 @@ fn parse_workspace_append( ))?, content_type, max_logical_size, + block_size: block_size.unwrap_or(nokv_object::DEFAULT_ARTIFACT_BLOCK_SIZE), })) } @@ -1047,6 +1100,60 @@ mod tests { ); } + #[test] + fn append_activity_lease_is_an_explicit_bounded_server_option() { + assert_eq!( + parse(args(&["serve"])) + .unwrap() + .server + .append_activity_lease_ms, + 1_800_000 + ); + assert_eq!( + parse(args(&["--append-activity-lease-ms", "1000", "serve"])) + .unwrap() + .server + .append_activity_lease_ms, + 1000 + ); + assert!(parse(args(&["--append-activity-lease-ms", "999", "serve"])).is_err()); + assert!(parse(args(&["--append-activity-lease-ms", "86400001", "serve"])).is_err()); + assert!(parse(args(&["--append-activity-lease-ms", "86400000", "serve"])).is_ok()); + assert!(parse(args(&["--append-activity-lease-ms", "1000", "schema"])).is_err()); + } + + #[test] + fn operation_status_needs_only_identity_and_agent_routing() { + let parsed = parse(args(&[ + "--agent-id", + "44444444444444444444444444444444", + "operation", + "status", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ])) + .unwrap(); + assert_eq!( + parsed.command, + Command::OperationStatus { + operation_id: [0xaa; 16], + } + ); + assert!(parsed.workbench_root.is_none()); + assert!(parsed.client.object.bucket.is_none()); + assert!(parse(args(&["operation", "status", "aa"])).is_err()); + assert!(parse(args(&["operation", "status"])).is_err()); + assert!(parse(args(&["operation", "resume", "aa"])).is_err()); + assert!(parse(args(&[ + "--agent-id", + "44444444444444444444444444444444", + "operation", + "status", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "unexpected", + ])) + .is_err()); + } + #[test] fn workspace_append_requires_one_payload_and_stable_identity() { let prefix = ["append", "run-42", "logs", "events.jsonl"]; @@ -1080,6 +1187,8 @@ mod tests { "text/plain", "--max-logical-size", "1024", + "--block-size", + "1024", ]) .unwrap(); assert_eq!( @@ -1093,6 +1202,7 @@ mod tests { payload: AppendPayload::Text("event".to_owned()), content_type: Some("text/plain".to_owned()), max_logical_size: Some(1024), + block_size: 1024, }) ); for payload in [["--base64", "ZXZlbnQ="], ["--file", "/tmp/delta.bin"]] { diff --git a/crates/nokv/src/main.rs b/crates/nokv/src/main.rs index ae9a9fd79..24dcd03e1 100644 --- a/crates/nokv/src/main.rs +++ b/crates/nokv/src/main.rs @@ -146,6 +146,9 @@ fn run() -> Result<(), String> { ) } Command::WorkspacePath(command) => run_workspace_path(&invocation, command), + Command::OperationStatus { operation_id } => { + run_operation_status(&invocation, nokv_protocol::OperationIdentity(*operation_id)) + } } } @@ -224,7 +227,11 @@ fn build_backend(invocation: &Invocation) -> Result control.as_ref(), admission.root_id, admission.agent_id, - || build_backend_after_agent_admission(invocation), + || { + let client = + connection::connect(&invocation.client).map_err(|error| error.to_string())?; + build_backend_after_agent_admission(invocation, client) + }, ) } #[cfg(not(feature = "etcd"))] @@ -239,23 +246,39 @@ fn build_backend(invocation: &Invocation) -> Result } } -#[cfg(feature = "etcd")] fn build_backend_after_agent_admission( invocation: &Invocation, + client: connection::CliWorkspaceClient, ) -> Result { - let client = connection::connect(&invocation.client).map_err(|error| error.to_string())?; - let preflight = client - .preflight(WORKBENCH_REQUIRED_RPC_CAPABILITIES) - .map_err(|error| format!("workspace preflight failed: {error}"))?; - let expected_namespace = preflight.value.route.object_namespace_id.into(); - let objects = - Arc::new(CliObjectStore::build(&invocation.client.object)?.bind(expected_namespace)?); - objects.validate_agent_capabilities()?; - Ok(CliWorkbenchBackend::new( - client, - objects, - invocation.client.max_artifact_bytes, - )) + #[cfg(feature = "etcd")] + { + let preflight = client + .preflight(WORKBENCH_REQUIRED_RPC_CAPABILITIES) + .map_err(|error| format!("workspace preflight failed: {error}"))?; + let expected_namespace = preflight.value.route.object_namespace_id.into(); + let objects = + Arc::new(CliObjectStore::build(&invocation.client.object)?.bind(expected_namespace)?); + if matches!( + invocation.command, + Command::WorkspacePath(WorkspacePathCommand::Append { .. }) + ) { + objects + .validate_append_capabilities() + .map_err(append_provider_admission_error)?; + } else { + objects.validate_agent_capabilities()?; + } + Ok(CliWorkbenchBackend::new( + client, + objects, + invocation.client.max_artifact_bytes, + )) + } + #[cfg(not(feature = "etcd"))] + { + let _ = (invocation, client); + Err("Agent-facing commands require the nokv etcd feature".to_owned()) + } } fn build_handler(invocation: &Invocation) -> Result { @@ -332,35 +355,62 @@ fn run_workspace_path( payload, content_type, max_logical_size, + block_size, } = command { let operation_id = nokv_protocol::OperationIdentity(*operation_id); let result = (|| { - let target = canonical_workspace_path(workbench, section, path)?; - let (delta, default_content_type) = - append_payload(payload, invocation.client.max_artifact_bytes)?; + let target = + canonical_workspace_path(workbench, section, path).map_err(append_input_error)?; + let (delta, default_content_type) = append_payload( + payload, + invocation + .client + .max_artifact_bytes + .min(nokv_client::MAX_APPEND_DELTA_BYTES), + ) + .map_err(append_input_error)?; let content_type = content_type .as_ref() .map(|value| nokv_protocol::ContentType::new(value.clone())) .transpose() - .map_err(|error| error.to_string())?; + .map_err(|error| append_input_error(error.to_string()))?; let create_content_type = content_type.clone().unwrap_or( nokv_protocol::ContentType::new(default_content_type) - .map_err(|error| error.to_string())?, + .map_err(|error| append_input_error(error.to_string()))?, ); - let backend = build_backend(invocation)?; - let (call, incarnation) = backend - .append_idempotent( + let client = build_workspace_path_client(invocation)?; + let incarnation = client + .resolve_append_workspace_incarnation( operation_id, - target, + &target, expected_workspace_incarnation_id.map(nokv_protocol::WorkspaceIdentity), - create_content_type, - content_type, - *max_logical_size, - &delta, ) + .map_err(|error| agent_error(backend::map_append_client_error(error).into()))?; + let mut options = nokv_client::IdempotentAppendOptions::new( + operation_id, + target, + incarnation, + create_content_type, + ) + .with_block_size(*block_size); + if let Some(content_type) = content_type { + options = options.with_content_type(content_type); + } + if let Some(max_logical_size) = max_logical_size { + options = options.with_max_logical_size(*max_logical_size); + } + if let Some(call) = client + .recover_append_receipt(&mut options, &delta) + .map_err(|error| agent_error(backend::map_append_client_error(error).into()))? + { + return print_json(&append_receipt_json(&call)); + } + let backend = build_backend_after_agent_admission(invocation, client)?; + let call = backend + .append_idempotent(options, &delta) .map_err(|error| agent_error(error.into()))?; - print_json(&append_receipt_json(&call, incarnation)) + print_json(&append_receipt_json(&call)) })(); return result.map_err(|message| append_error_json(operation_id, message)); } @@ -418,13 +468,29 @@ fn append_payload( max_bytes: usize, ) -> Result<(Vec, &'static str), String> { let (bytes, content_type) = match payload { - AppendPayload::Text(text) => (text.as_bytes().to_vec(), "text/plain; charset=utf-8"), - AppendPayload::Base64(encoded) => ( - STANDARD - .decode(encoded) - .map_err(|error| format!("append base64 payload is invalid: {error}"))?, - "application/octet-stream", - ), + AppendPayload::Text(text) => { + if text.len() > max_bytes { + return Err(format!( + "append delta is {} bytes, maximum is {max_bytes}", + text.len() + )); + } + (text.as_bytes().to_vec(), "text/plain; charset=utf-8") + } + AppendPayload::Base64(encoded) => { + let max_encoded_bytes = max_bytes.div_ceil(3).saturating_mul(4); + if encoded.len() > max_encoded_bytes { + return Err(format!( + "append base64 payload exceeds the encoded limit for {max_bytes} decoded bytes" + )); + } + ( + STANDARD + .decode(encoded) + .map_err(|error| format!("append base64 payload is invalid: {error}"))?, + "application/octet-stream", + ) + } AppendPayload::File(path) => ( transfer::read_collect_source(path, max_bytes)?, "application/octet-stream", @@ -439,28 +505,111 @@ fn append_payload( Ok((bytes, content_type)) } -fn append_receipt_json( - call: &nokv_client::ClientCall, - incarnation: nokv_protocol::WorkspaceIdentity, -) -> Value { - let result = &call.value; +fn append_result_json(result: &nokv_protocol::AppendResult) -> Value { json!({ - "status": "success", - "operation": "append", "operation_id": encode_lowercase_hex(&result.operation_id.0), + "publication_operation_id": encode_lowercase_hex(&result.publication_operation_id.0), "artifact_revision_id": encode_lowercase_hex(&result.artifact_revision_id.0), "workbench_id": result.target.workbench.as_str(), "path": result.target.path.as_str(), - "workspace_incarnation_id": encode_lowercase_hex(&incarnation.0), + "workspace_incarnation_id": encode_lowercase_hex(&result.workspace_incarnation_id.0), "workspace_revision": result.workspace_revision, "generation": result.generation, "logical_size": result.logical_size, "body_digest": result.body_digest.as_str(), - "replayed": call.replayed, - "commit_version": call.commit_version, }) } +fn append_receipt_json(call: &nokv_client::ClientCall) -> Value { + let mut receipt = append_result_json(&call.value); + receipt["status"] = json!("success"); + receipt["operation"] = json!("append"); + receipt["state"] = json!("committed"); + receipt["next_action"] = json!("none"); + receipt["replayed"] = json!(call.replayed); + receipt["commit_version"] = json!(call.commit_version); + receipt +} + +fn run_operation_status( + invocation: &Invocation, + operation_id: nokv_protocol::OperationIdentity, +) -> Result<(), String> { + (|| { + let client = build_workspace_path_client(invocation)?; + let call = client + .get_append_operation(operation_id) + .map_err(|error| agent_error(backend::map_append_client_error(error).into()))?; + print_json(&append_status_json(&call)?) + })() + .map_err(|message| append_error_json(operation_id, message)) +} + +fn append_status_json( + call: &nokv_client::ClientCall, +) -> Result { + let recovery = nokv_client::append_operation_recovery(&call.value) + .map_err(|error| agent_error(backend::map_append_client_error(error).into()))?; + let status = &call.value; + let preparation = status + .append_preparation + .as_ref() + .ok_or_else(|| "append status has no preparation".to_owned())?; + let receipt = match &status.result { + Some(nokv_protocol::OperationResult::ArtifactAppend(result)) => { + Some(append_result_json(result)) + } + _ => None, + }; + let attempt_failure = preparation.attempt_failure.as_ref(); + Ok(json!({ + "status": "success", + "operation": "append", + "operation_id": encode_lowercase_hex(&status.token.operation_id.0), + "state": recovery.state.as_str(), + "next_action": recovery.next_action.as_str(), + "observed_state": format!("{:?}", status.state), + "publication_operation_id": encode_lowercase_hex(&preparation.publication_operation_id.0), + "attempt": preparation.attempt, + "attempt_phase": preparation.attempt_phase, + "activity_deadline_ms": preparation.activity_deadline_ms, + "workspace_incarnation_id": encode_lowercase_hex(&preparation.workspace_incarnation_id.0), + "workbench_id": preparation.target.workbench.as_str(), + "path": preparation.target.path.as_str(), + "progress": status.progress, + "cause_code": attempt_failure.map(|failure| format!("{:?}", failure.code)), + "failure_message": attempt_failure.map(|failure| failure.message.as_str()), + "attempt_failure": attempt_failure.map(|failure| json!({ + "code": format!("{:?}", failure.code), + "message": failure.message, + "retryable": failure.retryable, + "conflict": failure.conflict.map(|conflict| format!("{conflict:?}")), + "current_generation": failure.current_generation, + })), + "receipt": receipt, + "commit_version": call.commit_version, + "replayed": call.replayed, + })) +} + +fn append_input_error(message: String) -> String { + agent_error(nokv_agent::AgentError::backend( + "InvalidArgument", + message, + false, + json!({}), + )) +} + +fn append_provider_admission_error(error: nokv_object::ProviderAdmissionError) -> String { + agent_error(nokv_agent::AgentError::backend( + "AppendFailed", + error.to_string(), + false, + json!({"code": error.code()}), + )) +} + fn append_error_json(operation_id: nokv_protocol::OperationIdentity, message: String) -> String { let mut error = serde_json::from_str::(&message) .ok() @@ -469,6 +618,9 @@ fn append_error_json(operation_id: nokv_protocol::OperationIdentity, message: St nokv_agent::AgentError::backend("AppendFailed", message, false, json!({})).as_value() }); error["details"]["operation_id"] = json!(encode_lowercase_hex(&operation_id.0)); + error["details"]["next_action"] = json!("query_same"); + error["details"]["cause_code"] = error["details"]["code"].clone(); + error["details"]["publication_operation_id"] = Value::Null; serde_json::to_string(&error).expect("append error contains only JSON values") } @@ -690,9 +842,10 @@ USAGE: nokv [connection/object options] mcp [--profile workbench|agent] (DEPRECATED) nokv [connection/object options] materialize
nokv [connection/object options] collect
[--replace] [--expected-generation N] [--content-type TYPE] + nokv [route/agent options] operation status nokv [route/agent options] workspace-path rename
--expected-generation N --request-id HEX32 nokv [route/agent options] workspace-path remove
--expected-generation N --request-id HEX32 - nokv [route/agent/object options] workspace-path append
--operation-id HEX32 (--text TEXT | --base64 BASE64 | --file PATH) [--content-type TYPE] [--max-logical-size N] [--expected-workspace-incarnation-id HEX32] + nokv [route/agent/object options] workspace-path append
--operation-id HEX32 (--text TEXT | --base64 BASE64 | --file PATH) [--content-type TYPE] [--max-logical-size N] [--block-size N] [--expected-workspace-incarnation-id HEX32] nokv --root-id HEX32 --agent-id HEX32 --etcd-endpoint URL provision [adoption options] nokv [owner options] serve nokv schema @@ -701,7 +854,7 @@ USAGE: AGENT CONTROL ROUTING: --root-id HEX32 - --agent-id HEX32 is required by provision, workbench, mcp, materialize, collect, and workspace-path + --agent-id HEX32 is required by provision, workbench, mcp, materialize, collect, workspace-path, and operation status AgentId is a durable deployment identity used to prevent root misconfiguration; it is not an authentication credential --metadata-address HOST:PORT --logical-shard-id HEX32 --object-namespace-id HEX32 --placement-generation N --owner-epoch N @@ -720,7 +873,10 @@ AGENT PRESENTATION: mcp defaults to the 18-tool Workbench profile; --profile agent selects the seven path tools workspace-path is a custom CLI surface; it does not add to the fixed 18 Workbench tools workspace-path rename/remove require an explicit lowercase HEX32 request id for exact cross-process replay - workspace-path append requires a stable lowercase HEX32 operation id and an existing workspace + workspace-path append requires a stable lowercase HEX32 logical operation id + operation status observes append recovery without a payload, workspace, or object-store connection + the first append requires an existing workspace; replays resolve its original incarnation + append delta and default total body limits are 16 MiB; --max-logical-size raises the bound body limit append returns the original publication receipt on replay; retain the id and incarnation across restarts append limits the delta with --max-artifact-bytes; --max-logical-size optionally limits the resulting body @@ -730,6 +886,7 @@ OWNER: --adopt-legacy-object-namespace is a one-time explicit migration after verifying bucket/prefix serve validates every active root's Agent binding; it does not require or compare one shard-wide AgentId --root-id HEX32 --etcd-endpoint URL --node-id ID + --append-activity-lease-ms N: serve only, range 1000..=86400000, default 1800000; clock-skew grace is additional --advertise-endpoint HOST:PORT --bind HOST:PORT --etcd-lease-ttl-seconds N (default 10; owner keepalive runs every max(1, N/3) seconds) --handshake-timeout-millis N --max-inflight-connections N @@ -923,7 +1080,7 @@ fn run_server(invocation: &Invocation) -> Result<(), String> { use nokv_control::{NodeId, RecoveryPublication, RootId}; use nokv_server::{ - bootstrap_shard, ArtifactLifecycleDeleter, LeaseMode, LifecycleObjectDeleter, + bootstrap_shard, ArtifactLifecycleCleaner, LeaseMode, LifecycleObjectCleaner, LifecycleRunner, LifecycleRunnerOptions, OpenMode, RecoveryPublicationMode, RootAttach, RootOwnerRegistry, ServerOptions, ShardBoot, WorkspaceServer, }; @@ -1055,13 +1212,16 @@ fn run_server(invocation: &Invocation) -> Result<(), String> { } let objects = Arc::new(CliObjectStore::build(&invocation.client.object)?.bind(expected_namespace)?); - objects.validate_agent_capabilities()?; + objects + .validate_append_capabilities() + .map_err(|error| error.to_string())?; let registry = Arc::new(RootOwnerRegistry::new()); let owner = bootstrap_shard( Arc::clone(&control), Arc::clone(®istry), objects.clone(), ShardBoot { + append_activity_lease_ms: invocation.server.append_activity_lease_ms, shard_id: placement.logical_shard_id, open: metadata, lease: LeaseMode::Acquire { @@ -1112,8 +1272,8 @@ fn run_server(invocation: &Invocation) -> Result<(), String> { ) .map_err(|error| error.to_string())?; let owner_loss = server.owner_loss_signal(); - let lifecycle_objects: Arc = - Arc::new(ArtifactLifecycleDeleter::new(objects)); + let lifecycle_objects: Arc = + Arc::new(ArtifactLifecycleCleaner::new(objects)); let mut lifecycles = Vec::with_capacity(owner_routes.len()); for route in owner_routes { match LifecycleRunner::new( @@ -1215,6 +1375,16 @@ mod tests { assert_eq!(binary.1, "application/octet-stream"); assert!(append_payload(&AppendPayload::Text("event".to_owned()), 4).is_err()); assert!(append_payload(&AppendPayload::Base64("!".to_owned()), 5).is_err()); + assert!(append_payload(&AppendPayload::Base64("!".repeat(12)), 5) + .unwrap_err() + .contains("encoded limit")); + assert!(append_payload(&AppendPayload::Base64("ZXZlbnQ=".to_owned()), 4).is_err()); + assert_eq!( + append_payload(&AppendPayload::Base64("".to_owned()), 0) + .unwrap() + .0, + b"" + ); let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("delta.bin"); std::fs::write(&path, b"event").unwrap(); @@ -1224,13 +1394,67 @@ mod tests { ); } + #[test] + fn append_input_validation_keeps_a_distinct_code_from_provider_failures() { + let operation_id = nokv_protocol::OperationIdentity([0xaa; 16]); + let oversized = append_payload(&AppendPayload::Text("sixsix".to_owned()), 5) + .map_err(append_input_error) + .unwrap_err(); + let invalid: Value = + serde_json::from_str(&append_error_json(operation_id, oversized)).unwrap(); + assert_eq!(invalid["code"], "InvalidArgument"); + assert_eq!(invalid["details"]["operation_id"], "aa".repeat(16)); + assert_eq!(invalid["details"]["next_action"], "query_same"); + let provider: Value = serde_json::from_str(&append_error_json( + operation_id, + "provider unavailable".to_owned(), + )) + .unwrap(); + assert_eq!(provider["code"], "AppendFailed"); + } + + #[test] + fn append_admission_errors_keep_machine_readable_causes_and_identity() { + use nokv_object::{ProviderAdmissionCapability, ProviderAdmissionError}; + let operation_id = nokv_protocol::OperationIdentity([0xaa; 16]); + for (error, cause) in [ + ( + ProviderAdmissionError::Unavailable, + "ProviderAdmissionUnavailable", + ), + ( + ProviderAdmissionError::Inconclusive, + "ProviderAdmissionInconclusive", + ), + ( + ProviderAdmissionError::Rejected { + capability: ProviderAdmissionCapability::ImmutableSealing, + }, + "ProviderAdmissionRejected", + ), + ] { + let result: Value = serde_json::from_str(&append_error_json( + operation_id, + append_provider_admission_error(error), + )) + .unwrap(); + assert_eq!(result["code"], "AppendFailed"); + assert_eq!(result["details"]["cause_code"], cause); + assert_eq!(result["details"]["operation_id"], "aa".repeat(16)); + assert_eq!(result["details"]["next_action"], "query_same"); + assert_eq!(result["retryable"], false); + } + } + #[test] fn append_receipt_keeps_historical_publication_and_error_identity() { let call = nokv_client::ClientCall { - value: nokv_protocol::PublishResult { + value: nokv_protocol::AppendResult { operation_id: nokv_protocol::OperationIdentity([0xaa; 16]), + publication_operation_id: nokv_protocol::OperationIdentity([0xee; 16]), artifact_revision_id: nokv_protocol::ArtifactRevisionIdentity([0xbb; 16]), target: canonical_workspace_path("run-42", "logs", "events.jsonl").unwrap(), + workspace_incarnation_id: nokv_protocol::WorkspaceIdentity([0xdd; 16]), workspace_revision: 7, generation: 3, logical_size: 12, @@ -1239,13 +1463,16 @@ mod tests { commit_version: None, replayed: true, }; - let receipt = append_receipt_json(&call, nokv_protocol::WorkspaceIdentity([0xdd; 16])); + let receipt = append_receipt_json(&call); assert_eq!(receipt["operation_id"], "aa".repeat(16)); + assert_eq!(receipt["publication_operation_id"], "ee".repeat(16)); assert_eq!(receipt["artifact_revision_id"], "bb".repeat(16)); assert_eq!(receipt["generation"], 3); assert_eq!(receipt["logical_size"], 12); assert_eq!(receipt["workspace_incarnation_id"], "dd".repeat(16)); assert_eq!(receipt["replayed"], true); + assert_eq!(receipt["state"], "committed"); + assert_eq!(receipt["next_action"], "none"); assert!(receipt["commit_version"].is_null()); let error = nokv_agent::AgentError::backend( "AppendUnresolved", @@ -1263,6 +1490,82 @@ mod tests { assert_eq!(result["code"], "AppendUnresolved"); } + #[test] + fn append_status_does_not_report_pending_or_cleaned_attempts_as_committed() { + let mut call = nokv_client::ClientCall { + value: nokv_protocol::OperationStatus { + token: nokv_protocol::OperationToken { + operation_id: nokv_protocol::OperationIdentity([0xaa; 16]), + state_digest: nokv_protocol::Digest([0x11; 32]), + }, + kind: nokv_protocol::OperationKind::ArtifactAppend, + append_preparation: Some(Box::new(nokv_protocol::AppendPreparation { + intent_digest: nokv_protocol::Digest([0x22; 32]), + target: canonical_workspace_path("run-42", "logs", "events.jsonl").unwrap(), + workspace_incarnation_id: nokv_protocol::WorkspaceIdentity([0xbb; 16]), + attempt: 2, + publication_operation_id: nokv_protocol::OperationIdentity([0xcc; 16]), + artifact_revision_id: nokv_protocol::ArtifactRevisionIdentity([0xdd; 16]), + attempt_phase: nokv_protocol::AppendAttemptPhase::Finalizing, + activity_deadline_ms: 123_000, + attempt_failure: None, + })), + publish_preparation: None, + commit_preparation: None, + restore_preparation: None, + state: nokv_protocol::OperationState::Running, + progress: nokv_protocol::OperationProgress { + completed_rows: 1, + total_rows: Some(1), + completed_bytes: 4, + total_bytes: Some(4), + }, + result: None, + failure: None, + }, + commit_version: None, + replayed: false, + }; + let pending = append_status_json(&call).unwrap(); + assert_eq!(pending["state"], "pending"); + assert_eq!(pending["next_action"], "poll"); + assert_eq!(pending["attempt_phase"], "finalizing"); + assert_eq!(pending["attempt"], 2); + assert_eq!(pending["activity_deadline_ms"], 123_000); + assert_eq!(pending["operation_id"], "aa".repeat(16)); + assert_eq!(pending["publication_operation_id"], "cc".repeat(16)); + assert!(pending["receipt"].is_null()); + let preparation = call.value.append_preparation.as_mut().unwrap(); + preparation.attempt_phase = nokv_protocol::AppendAttemptPhase::Cleaned; + preparation.attempt_failure = Some(nokv_protocol::RpcFailure { + code: nokv_protocol::ErrorCode::OperationFailed, + message: "append attempt lost its frozen path claim".to_owned(), + retryable: false, + conflict: None, + current_generation: None, + route_hint: None, + }); + let cleaned = append_status_json(&call).unwrap(); + assert_eq!(cleaned["state"], "ready_to_retry"); + assert_eq!(cleaned["next_action"], "resubmit_same"); + assert_eq!(cleaned["cause_code"], "OperationFailed"); + assert_eq!( + cleaned["failure_message"], + "append attempt lost its frozen path claim" + ); + assert_eq!(cleaned["attempt_failure"]["code"], "OperationFailed"); + assert!(cleaned["receipt"].is_null()); + call.value.state = nokv_protocol::OperationState::Quarantined; + call.value + .append_preparation + .as_mut() + .unwrap() + .attempt_phase = nokv_protocol::AppendAttemptPhase::Quarantined; + let quarantined = append_status_json(&call).unwrap(); + assert_eq!(quarantined["next_action"], "operator_reconcile"); + assert!(quarantined["receipt"].is_null()); + } + #[test] fn workspace_path_maps_sections_without_accepting_reserved_or_duplicated_paths() { let path = canonical_workspace_path("run-42", "outputs", "result.json").unwrap(); diff --git a/crates/nokv/src/object_store.rs b/crates/nokv/src/object_store.rs index bb541198d..5c4abde73 100644 --- a/crates/nokv/src/object_store.rs +++ b/crates/nokv/src/object_store.rs @@ -5,15 +5,16 @@ //! Object-provider composition for the custom Agent CLI. -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, Mutex, OnceLock}; use nokv_object::{ admit_artifact_provider, ensure_object_namespace, load_object_namespace, verify_object_namespace, ArtifactObjectStore, ArtifactStoreCapabilities, ImmutableCreateOutcome, LocalHotTier, LocalHotTierOptions, ObjectDeleteOutcome, ObjectError, - ObjectInfo, ObjectKey, ObjectRange, ProviderAdmissionError, ProviderAdmissionProfile, - ProviderAdmissionReceipt, ProviderHandleIdentity, S3ArtifactStore, S3ArtifactStoreOptions, - TieredArtifactStore, TieredArtifactStoreOptions, DEFAULT_ARTIFACT_BLOCK_SIZE, + ObjectInfo, ObjectKey, ObjectRange, ObjectSealOutcome, ProviderAdmissionError, + ProviderAdmissionProfile, ProviderAdmissionReceipt, ProviderHandleIdentity, S3ArtifactStore, + S3ArtifactStoreOptions, TieredArtifactStore, TieredArtifactStoreOptions, + DEFAULT_ARTIFACT_BLOCK_SIZE, }; use nokv_types::ObjectNamespaceId; @@ -31,7 +32,9 @@ enum CliObjectStoreInner { pub struct CliObjectStore { inner: CliObjectStoreInner, namespace_id: Option, - admission: Arc>>, + admission: Arc>, + append_admission: Arc>, + admission_probe: Arc>, } impl CliObjectStore { @@ -82,6 +85,8 @@ impl CliObjectStore { inner: CliObjectStoreInner::S3(durable), namespace_id: None, admission: Arc::new(OnceLock::new()), + append_admission: Arc::new(OnceLock::new()), + admission_probe: Arc::new(Mutex::new(())), }); }; let hot = LocalHotTier::new(LocalHotTierOptions::new(cache_root, config.hot_cache_bytes)) @@ -94,26 +99,49 @@ impl CliObjectStore { )), namespace_id: None, admission: Arc::new(OnceLock::new()), + append_admission: Arc::new(OnceLock::new()), + admission_probe: Arc::new(Mutex::new(())), }) } /// Verify the immutable object semantics required by every Agent tool /// before the CLI advertises its MCP surface. pub fn validate_agent_capabilities(&self) -> Result<(), String> { - self.cache_admission(|store| { + self.cache_admission(&self.admission, |store| { let profile = ProviderAdmissionProfile::single_put(DEFAULT_ARTIFACT_BLOCK_SIZE)?; admit_artifact_provider(store, profile) }) + .map_err(|error| error.to_string()) + } + + /// Prove monotonic sealing before admitting stable append uploads. + pub fn validate_append_capabilities(&self) -> Result<(), ProviderAdmissionError> { + self.cache_admission(&self.append_admission, |store| { + let profile = ProviderAdmissionProfile::single_put(DEFAULT_ARTIFACT_BLOCK_SIZE)? + .with_append_sealing(); + admit_artifact_provider(store, profile) + }) } fn cache_admission( &self, + admission: &OnceLock, probe: impl FnOnce(&S3ArtifactStore) -> Result, - ) -> Result<(), String> { - match self.admission.get_or_init(|| probe(self.durable())) { - Ok(_) => Ok(()), - Err(error) => Err(error.to_string()), + ) -> Result<(), ProviderAdmissionError> { + if admission.get().is_some() { + return Ok(()); + } + let _guard = self + .admission_probe + .lock() + .map_err(|_| ProviderAdmissionError::Inconclusive)?; + if admission.get().is_none() { + // Cache evidence of success, never a transient outage. A caller + // using this same handle may safely try the probe again later. + let receipt = probe(self.durable())?; + admission.get_or_init(|| receipt); } + Ok(()) } pub fn bind(mut self, expected: ObjectNamespaceId) -> Result { @@ -157,7 +185,7 @@ impl ArtifactObjectStore for CliObjectStore { } fn provider_admission_receipt(&self) -> Option<&ProviderAdmissionReceipt> { - self.admission.get().and_then(|result| result.as_ref().ok()) + self.append_admission.get().or_else(|| self.admission.get()) } fn create_immutable( @@ -171,6 +199,13 @@ impl ArtifactObjectStore for CliObjectStore { } } + fn seal_immutable(&self, key: &ObjectKey) -> Result { + match &self.inner { + CliObjectStoreInner::S3(store) => store.seal_immutable(key), + CliObjectStoreInner::CachedS3(store) => store.seal_immutable(key), + } + } + fn read(&self, key: &ObjectKey, range: Option) -> Result, ObjectError> { match &self.inner { CliObjectStoreInner::S3(store) => store.read(key, range), @@ -240,7 +275,7 @@ mod tests { } #[test] - fn one_provider_handle_runs_admission_at_most_once_even_after_failure() { + fn one_provider_handle_retries_admission_after_a_failed_probe() { let config = ObjectConfig { bucket: Some("artifacts".to_owned()), ..ObjectConfig::default() @@ -250,14 +285,14 @@ mod tests { for _ in 0..2 { let error = store - .cache_admission(|_| { + .cache_admission(&store.admission, |_| { calls.fetch_add(1, Ordering::SeqCst); Err(ProviderAdmissionError::Inconclusive) }) .unwrap_err(); - assert!(error.contains("inconclusive")); + assert_eq!(error, ProviderAdmissionError::Inconclusive); } - assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(calls.load(Ordering::SeqCst), 2); assert!(store.provider_admission_receipt().is_none()); } } diff --git a/crates/nokv/src/transfer.rs b/crates/nokv/src/transfer.rs index 44dbc0584..fad2d7ff6 100644 --- a/crates/nokv/src/transfer.rs +++ b/crates/nokv/src/transfer.rs @@ -5,8 +5,8 @@ //! Explicit single-artifact transfer helpers for local executables. -use std::fs::{self, OpenOptions}; -use std::io::Write; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; pub fn read_collect_source(source: &Path, max_bytes: usize) -> Result, String> { @@ -35,7 +35,33 @@ pub fn read_collect_source(source: &Path, max_bytes: usize) -> Result, S metadata.len() )); } - let bytes = fs::read(source) + let input = File::open(source) + .map_err(|error| format!("cannot open collect source {}: {error}", source.display()))?; + if !input + .metadata() + .map_err(|error| format!("cannot inspect opened source {}: {error}", source.display()))? + .is_file() + { + return Err(format!( + "collect source {} is not a regular file", + source.display() + )); + } + read_collect_bytes(source, input, max_bytes) +} + +fn read_collect_bytes( + source: &Path, + input: impl Read, + max_bytes: usize, +) -> Result, String> { + let limit = u64::try_from(max_bytes) + .unwrap_or(u64::MAX) + .saturating_add(1); + let mut bytes = Vec::new(); + input + .take(limit) + .read_to_end(&mut bytes) .map_err(|error| format!("cannot read collect source {}: {error}", source.display()))?; if bytes.len() > max_bytes { return Err(format!( @@ -116,6 +142,25 @@ mod tests { .contains("regular file")); } + #[test] + fn collect_bounds_reads_when_the_source_grows_after_inspection() { + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("growing.bin"); + fs::write(&source, b"first").unwrap(); + let mut input = File::open(&source).unwrap(); + assert_eq!(input.metadata().unwrap().len(), 5); + OpenOptions::new() + .append(true) + .open(&source) + .unwrap() + .write_all(&[b'x'; 1024]) + .unwrap(); + let error = read_collect_bytes(&source, &mut input, 5).unwrap_err(); + assert!(error.contains("grew to 6 bytes, maximum is 5")); + use std::io::Seek; + assert_eq!(input.stream_position().unwrap(), 6); + } + #[test] fn materialize_never_overwrites() { let root = tempfile::tempdir().unwrap(); diff --git a/docs/architecture.md b/docs/architecture.md index c6da54925..f60bec1b2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -159,8 +159,9 @@ continuations retain one exact root read version. Live continuations may move to a newer root read version only while the target workspace incarnation and revision remain unchanged; target drift fails closed, and an initial bounded collection may restart in full but never merges workspace revisions. This -contract is gated by operation schema `nokv.workspace.rpc.v10`; v10 adds exact -append-intent binding and publication preparation in operation status. v7 added the +contract is gated by operation schema `nokv.workspace.rpc.v11`; v11 adds durable logical append parents, +predecessor-fenced publication attempts, and metadata-only append status. The +initial exact append-intent binding was introduced by v10. v7 added the exact read-version fence used by path point reads, v8 added the required typed `Prefix`/`Exact` catalog path match so an artifact cannot inherit fields from same-name descendants, and v9 added an explicit query profile plus tagged query @@ -169,13 +170,13 @@ added the provider-neutral object namespace identity to every root route. Each TCP connection starts with the fixed-width, schema-neutral transport handshake v1. The client offers one exact operation schema, and the server -accepts only `nokv.workspace.rpc.v10`; the handshake version remains stable when +accepts only `nokv.workspace.rpc.v11`; the handshake version remains stable when the operation schema changes. For upgrade diagnostics, the server recognizes only operation-first envelopes from the public v2 client and the post-tag v3 client. It parses a bounded route/request header, ignores the operation, emits one schema-readable failure, and closes without dispatch. The v2 client gets -its exact v2 failure envelope; the v3 client gets a v10 failure envelope so its -decoder reports the v10/v3 schema mismatch. Unknown legacy schemas, malformed +its exact v2 failure envelope; the v3 client gets a v11 failure envelope so its +decoder reports the v11/v3 schema mismatch. Unknown legacy schemas, malformed envelopes, and malformed handshakes fail closed. This is a read-only rejection path, not a legacy response decoder or operation fallback. diff --git a/docs/development/append-idempotency.md b/docs/development/append-idempotency.md index 7407bf8b3..acbd01233 100644 --- a/docs/development/append-idempotency.md +++ b/docs/development/append-idempotency.md @@ -5,142 +5,24 @@ SPDX-License-Identifier: Apache-2.0 # Append identity and response-loss recovery -A harness can persist a caller operation ID before sending an append. Retrying -that ID means one publication attempt, even after the harness or shard owner -restarts. A different operation ID represents a different action, including -when its bytes are identical. The native CLI is the primary integration -surface; the Python SDK uses the same client implementation. - -## Contract and scope - -The stable append request binds the complete caller intent with a full SHA-256 -commitment in the existing publish operation record. The intent includes the -normalized target and explicit append inputs and policies. It is distinct from -the generation-dependent immutable publication plan. The client computes this -commitment from its typed inputs; the server exact-binds it as opaque input and -does not claim to verify delta bytes that the metadata RPC does not carry. - -One operation identity admits at most one publication attempt. It never derives -a second identity after a timeout, read conflict, or path-generation conflict. -Before consulting the live path, a retry authenticates the operation's intent, -target, workspace incarnation, and artifact revision. Identity reuse with a -changed request fails closed. - -Callers must allocate operation IDs uniquely across the public lifecycle kinds -within one root: append/publication, commit build, and restore. Domain-separated -IDs persisted by the harness are one way to meet this requirement. The server -also enforces this rule atomically: each initial admission checks that both -other kinds' operation keys are absent in the same metadata command that -creates its own row. A preliminary GetOperation lookup alone cannot close this -race. No separate registry or new key family is needed. GetOperation remains -fail-closed if corrupt or externally supplied state contains multiple kinds for -one identity. Internal commit and restore manifest publications use distinct, -domain-separated identities and keep their existing lifecycle authority. - -A published operation returns its original compact receipt: operation ID, -target, workspace revision, generation, artifact revision, logical size, and -body digest. This is a historical result. A later append, removal, or replacement -of the live path does not revoke that success and does not trigger another -append. The receipt does not claim that its revision is still the live head or -that its bytes remain retained forever. - -The response envelope's `commit_version` and `replayed` are metadata about the -current call, not fields of the stable historical result. The first publication -may carry a metadata commit version; recovery through GetOperation carries no -new commit version and marks the client result as replayed. Consumers should -compare the receipt fields listed above, rather than requiring byte-identical -JSON envelopes across the first call and retries. - -An admitted operation without a terminal success returns a queryable pending or -terminal failure result. A new Begin RPC cannot replan, take over, or clean up -that operation. Byte-identical retries of the original RPC continue through the -existing request ledger. The existing fenced publish lifecycle owns recovery, -including the publication-absence proof required before Finalizing cleanup. -The caller must not interpret timeout, NotFound, or Finalizing as proof that an -append did not happen, nor automatically retry with a fresh operation ID. - -For example, a queue-driven harness persists an action ID for task 42, step 7 -before appending its completion event. If its worker dies after publication, -the replacement worker sends the same ID and event and recovers that receipt. -Two independently scheduled steps may emit identical events and still need two -different IDs. A branch of a run also needs its own action identity: changing -the target workspace under the parent's ID is an intent mismatch. - -A worker that dies during upload has a different recovery boundary. Its new -process can discover the admitted operation, but this API does not resume that -upload or create another attempt. The existing owner lifecycle can finish its -fenced cleanup. Workloads requiring automatic completion despite admitted CAS -conflicts or abandoned uploads need the durable attempt mapping described below. - -The workspace incarnation fence is mandatory for stable append. It prevents a -request for an old workspace instance from writing into a later instance with -the same name. Root routing and owner fencing continue to apply to lookups, -request deduplication, and mutation. - -Terminal operations currently have no automatic GC. Their retained records -provide the identity tombstone and historical receipt. Any future retention -policy must preserve an equivalent replay contract before removing them. This -change does not add a hidden artifact reference or pin every old revision just -to recover metadata. PublishPreparation projects existing target, incarnation, -revision, and intent fields; it does not store a full descriptor, whose legal -index fields can exceed the 16 KiB operation-record budget. - -## Architecture alternatives - -| Alternative | Reason for the decision | -| --- | --- | -| Stable caller ID alone | Re-reading the live head builds a different append plan after success; an ID flag alone cannot authenticate or recover the original intent. | -| Selected: one attempt through the existing publish lifecycle | Keeps publication, cleanup, fencing, and durability in one state machine while making response-loss recovery exact and bounded. | -| Logical append with durable attempt mapping | Appropriate if automatic rebase after concurrent-writer conflicts becomes a required product contract. It needs a durable parent or predecessor fence proving every earlier attempt cannot publish. It is outside this single-attempt change. | -| RPC request-ID reuse | Protects an exact encoded command, including route-dependent fields. It cannot alone represent a logical append across process and owner changes. | -| Content-hash deduplication | Incorrectly suppresses two intentional actions carrying identical bytes. | -| Immutable event files keyed by action ID | A useful harness design for event-oriented workloads, but changes the byte-append contract and needs an ordering/projection policy. It does not repair append. | - -Deterministic attempt numbering without durable predecessor records is unsafe: -one caller may skip an unadmitted attempt after a read conflict while another -caller later publishes that same attempt. Both it and the first caller's next -attempt can then succeed. This change deliberately has no such attempt chain. - -## Storage and protocol versions - -The strict workspace system format advances from 10 to 11; the publication -record family advances from value format 4 to 5; the RPC schema advances from -`nokv.workspace.rpc.v9` to `nokv.workspace.rpc.v10`. Older stores and protocol -schemas are rejected. No migration, compatibility decoder, or marker-only -upgrade is introduced. Red and green acceptance therefore use independently -initialized stores. Green recovery tests reopen the same green-version Holt -directory after owner termination. - -## Acceptance requirements - -Tests must retain actual owner/object-provider execution evidence for response -loss followed by a new CLI process and owner restart; one logical append must -produce one event. Additional cases cover a newer live head, changed intent, -concurrent callers with the same ID, distinct IDs with identical bytes, -incarnation changes, and admitted pending operations. Unit tests protect exact -binding, codec versions, server request deduplication, historical receipts, -owner fencing, and refusal to replan or clean up an admitted operation. - -The executable gate requires Docker, AWS CLI, etcd, and etcdctl. Build the CLI -with `cargo build -p nokv --features etcd`, freeze a copy of that binary, and -install a wheel built from the same checkout into an isolated Python environment. -Run the full qualification from the repository root: - -```shell -python3 scripts/workbench/append_identity_recovery_gate.py \ - --mode qualified --scenario all \ - --source-dir "$PWD" --nokv-bin /absolute/path/to/frozen-nokv \ - --python-executable /absolute/path/to/venv/bin/python \ - --evidence-dir /absolute/path/to/new-evidence-directory -``` - -Use `--mode baseline` with a frozen pre-fix binary and a separate evidence -directory to reproduce the duplicate append. The gate retains the dropped -successful RPC response, process/owner restart evidence, materialized bytes, -binary and Python-extension identities, current source hashes, and cleanup -results. `GREEN_PARTIAL` identifies a selected subset or a run without Python; -pending-operation cases qualify safety separately from completion. - -This contract covers NoKV publication. It does not make email, browser actions, -payments, or other external effects execute once. A downstream harness must -assign and persist identities for those effects at their own authority. +The supported append contract is specified in +[Durable append product contract](append-product-spec.md). A caller persists one +logical operation ID and its complete intent. NoKV can use successive immutable +publication attempts only after durably proving the previous attempt is cleaned; +parent advancement and child admission are atomic. Successful publication and +the logical receipt are also atomic. + +A completed operation returns its original receipt after process restart, +response loss, or later changes to the live file. An unfinished operation exposes +`pending`, `ready_to_retry`, or `quarantined`, with a prescribed recovery action. +Status queries need neither the original payload nor an available object store. +The native CLI is the primary interface, followed by the Python and Rust SDKs. + +The earlier single-attempt implementation prevented duplicate appends after a +lost success response but could not finish an action after an abandoned upload +was cleaned. Its executable +[identity recovery gate](../../scripts/workbench/append_identity_recovery_gate.py) +is retained for reproducible historical evidence. Its safety-only pending cases +do not establish completion qualification for the current product contract. +Use the [product acceptance gate](../../scripts/workbench/append_product_acceptance_gate.py) +and the normative contract's acceptance matrix for the current implementation. diff --git a/docs/development/append-product-spec.md b/docs/development/append-product-spec.md new file mode 100644 index 000000000..206563fbc --- /dev/null +++ b/docs/development/append-product-spec.md @@ -0,0 +1,344 @@ + + +# Durable append product contract + +A queue worker saves an action ID and its event before appending the event to a +workspace file. If its process dies, its replacement submits the same action +and receives one durable result. Concurrent writers may change the file while +that action is recovering. NoKV must either finish the action exactly once or +report a durable, actionable recovery state without allowing two effects. + +This document specifies that feature. Execution receipts establish which +requirements passed on a particular build and deployment. The broader release +gates in [workspace acceptance](workspace-acceptance.md) remain independent. + +## Consumer responsibilities and supported interfaces + +The native `nokv workspace-path append` command is the primary interface, followed by Python +`Client.append_bytes` and Rust `WorkspaceClient::append_artifact_idempotent`. +They share the Rust client state machine. Owner preflight advertises +`artifact_append_v1` so downstream clients can require the logical append +contract explicitly. `nokv operation status` and Python +`Client.operation_status` query the logical operation using metadata only. +The frozen Workbench tool schemas remain unchanged. + +The caller must durably retain the following before its first submission: + +- A root-scoped 128-bit logical operation ID, unique across append, publication, + commit build, and restore operations. +- The normalized workspace/path and intended workspace incarnation, once known. +- The exact delta bytes, create content type, optional replacement content type, + block size, and maximum resulting size. + +The caller must reuse those inputs when redelivering an action. Independent +actions need independent IDs even when their bytes match. A fork of a run uses +its own identity; retargeting an admitted ID to the fork is an intent mismatch. +Callers must not allocate a new ID merely because an RPC timed out. A digest is +not a substitute for retaining the delta: NoKV does not reconstruct payloads +from a hash or execute the harness's external actions. + +When a caller omits the incarnation, the shared client first queries the +logical operation. An admitted operation resolves to its original incarnation +even if its workspace name has since been removed or rebound. Only an absent +operation resolves through the live workspace name. An explicit incarnation +must match and is never silently replaced. The server checks the fence during +admission and publication. + +## Identity, intent, and immutable attempts + +The logical operation owns a full SHA-256 commitment to the canonical intent. +It includes the root, logical ID, incarnation, length-delimited target and +content-type fields, explicit option tags, block size, effective result-size +limit, delta length, and full delta SHA-256. Omitted result-size limits normalize +to the documented default before hashing. Distinct optional content-type +policies remain distinct even when their strings happen to match the current +file. Scheduling retry budgets and transport configuration are not intent. + +The metadata service binds this commitment exactly. Metadata RPCs do not carry +the original delta, so the service does not claim to independently recompute +the caller's full intent hash. Object upload verification and manifest/body +validation retain their existing responsibilities. + +Each logical operation has one current numbered publication attempt, starting +at zero. Child publication and artifact revision identities are SHA-256-derived +from the root, logical ID, attempt number, and separate versioned domains. The +server verifies this derivation. The logical ID is distinct from the child ID; +a consumer must persist and retry the logical ID. + +The parent contains the canonical intent, original target/incarnation, current +attempt identity, and eventual compact receipt. Each child publication carries +its parent/attempt binding and its immutable generation-dependent plan. An +active child cannot be replaced by a newly planned child. + +## Atomicity and recovery invariants + +1. Creating or advancing the parent and admitting its child happen in one + metadata command. There is no externally admitted parent-to-missing-child + interval and no separate advance RPC. +2. The first child requires an absent parent. A successor requires the exact + current predecessor to be durably `Cleaned`. `Failed`, timeout, `NotFound`, + stale routing, `Finalizing`, and `Quarantined` do not prove cleanup. +3. A child's final publication verifies that it is still the parent's current + attempt. Finalizing the child, committing the file/manifest/references/indexes + and change event, and persisting the parent's successful receipt happen in + the same metadata transaction. +4. A successful parent never advances. Replaying success returns the original + receipt without consulting the live path or reading object bodies. +5. Operation-kind collision checks execute atomically during admission. A + preliminary lookup alone does not establish uniqueness under concurrency. +6. Owner epochs, workspace incarnations, row digests, and publication-absence + proofs continue to fence recovery. A failed read or uncertain commit cannot + authorize cleanup, a successor, or an inferred success. +7. Cleanup permanently seals a failed append child's staged object keys before + retiring their staging rows. Its revision claim remains reserved so that + neither delayed uploads nor later reuse of that revision can revive its + payload. Parent receipts do not pin historical artifact bytes. + +A definitive generation conflict during completion can put the exact losing +child into fenced abort/cleanup after proving it did not publish. The client +can then re-read the current head and admit a successor under the same logical +ID. Concurrent successful appends have a generation order; NoKV does not promise +that this order matches caller start times or external queue order. + +## Observable outcomes and caller actions + +| State | Meaning | Required next action | +| --- | --- | --- | +| `committed` | Parent and child have an atomic successful receipt. | `none`; acknowledge the queued action. | +| `pending` | The current child is uploading, finalizing, aborting, or cleaning. | `poll`; keep the same action and payload. | +| `ready_to_retry` | The predecessor is durably cleaned and cannot publish. | `resubmit_same`; send the original logical ID, delta, and options. | +| `quarantined` | Automatic recovery cannot prove a safe transition. | `operator_reconcile`; preserve evidence and the original ID. | +| Query error or unknown transport outcome | No authoritative recovery observation was obtained. | Query the same ID again; do not infer absence or allocate a replacement action. | +| Intent mismatch | The ID belongs to a different intent or lifecycle. | Correct the caller's persisted action mapping; do not mutate that ID's intent. | + +Errors carry the logical operation ID where available, the structured cause +code, and observed wire state where known. An error envelope's conservative +`query_same` action does not claim knowledge of the current child. The metadata +status result supplies authoritative attempt, child ID, phase, activity +deadline, original incarnation/target, current attempt failure when present, +and receipt. An attempt failure describes its child, not a terminal failure of +the logical action. Applications must use +structured fields, not parse human-readable messages. + +A missing logical operation is a point-in-time observation. Another caller may +admit it immediately afterward. Resubmitting the same ID is safe because atomic +admission handles that race; changing to a fresh ID is not justified. + +The synchronous client performs bounded local submissions and cleanup polling. +It may return `pending` while owner recovery continues. Eventual completion +requires redelivery with the same payload, a reachable owner/object provider, +and progress by the fenced cleanup lifecycle. Perpetual contention, exhausted +capacity, unavailable dependencies, or quarantine cannot be converted into a +success promise by retrying more aggressively. + +## Failed-object sealing and delayed uploads + +A successful DELETE proves absence at one instant. It cannot prevent an old +PUT from arriving afterward. A cleanup implementation that then forgets its +staged keys can leak an unreachable payload even while logical append +idempotency remains correct. Product acceptance reproduces this window by +holding a real conditional PUT before it reaches the provider, killing its +caller, allowing cleanup, and then releasing that same request. + +Stable append therefore closes failed object keys monotonically. If a key is +absent, cleanup creates a zero-byte seal with `If-None-Match: *`. If payload +already exists, it replaces it with the same seal using `If-Match` against the +observed provider ETag. Aborted append cleanup never DELETEs these keys. This +also prevents an old cleanup request from deleting a newer seal and reopening +the late-upload window. A delayed nonempty upload cannot replace a seal. An +exact empty-body replay still cannot authorize publication by a terminal child. + +`Sealed` is distinct from physical deletion or absence. The failed child's +revision claim remains permanently associated with its terminal operation, +including after a logical successor succeeds. The ordinary collector for +published revisions cannot claim that unpublished reserved revision. The +operator reconciliation API requires the append-specific `provider_objects_sealed` +verdict; its older absence verdict cannot bypass this contract. An ambiguous +provider result cannot be converted into an unproved successful cleanup. + +For a quarantined append, the operator first captures the logical status and +exact quarantined child token, then inspects that child's persisted staged +records and reserved revision in the metadata snapshot. Every registered key +must be addressed in its bound object namespace; guessing keys from a live +file listing is insufficient. For each key, record HEAD evidence. If absent, +conditionally PUT an empty body with `If-None-Match: *`; otherwise use its ETag +with `If-Match`. Reobserve on a conditional conflict, and require a final HEAD +showing an existing zero-byte object. Do not DELETE during this procedure. +Preserve the key-by-key transcript and its SHA-256 digest. Submit +`ReconcileQuarantinedArtifactPublish` with that exact child token, +`provider_objects_sealed`, a reason, and the transcript's `evidence_digest`. +The RPC checks the metadata half of the verdict; it does not execute the +provider repair or independently certify the operator's transcript. A changed +token or any unproved key requires a fresh inspection. Finally query the +logical ID and follow its returned action using the original intent. + +Object-provider admission for stable append includes sealing conformance in +addition to immutable creation. Provider receipt verification is bound to the +actual handle and block-size profile. Generic publication admission retains +its existing requirements; the native owner that advertises append support +must qualify sealing. A local evictable cache cannot supply the permanent +proof; a tiered store seals its durable provider and invalidates its hot copy. + +These guarantees depend on the admitted provider honoring the conditional +writes. The real acceptance receipt identifies the tested RustFS build and +configuration. It does not certify every product describing itself as +S3-compatible. The artifact prefix must have no lifecycle expiration rule or +external delete/overwrite process that can remove live seals. A provider delete +marker is not a live empty seal: [AWS conditional-write semantics](https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html) +allow `If-None-Match` creation when the current version is a delete marker. With versioned buckets, replacement clears the +current payload only; old object versions may retain bytes and require a +separately qualified retention policy. The acceptance configuration and this +feature do not certify deletion of historical provider versions. Ordinary +publication cleanup remains a separate lifecycle contract; this feature's seal +semantics apply to stable append attempts. + +## Example recovery timelines + +| Failure | Durable observation | Recovery | +| --- | --- | --- | +| Worker dies before admission. | No parent. | Replacement submits the same ID; attempt zero is admitted once. | +| Begin succeeds but its response is lost; worker or owner dies. | Parent and child already exist. | Wait for fenced cleanup, then submit the same ID to create the next child atomically. | +| An object PUT succeeds but the client loses its response. | The child may be pending; publication is not inferred. | Query and wait for a safe child outcome. Unknown upload results never authorize an unfenced successor. | +| A different action wins a generation CAS. | Losing child is safely aborted and cleaned. | The same logical append replans against the new head and completes once. | +| Complete commits but its response is lost. | Parent contains the committed receipt. | Replacement returns that receipt; it uploads and appends nothing. | +| File is later appended, removed, or replaced. | Parent retains its original success. | Status and replay still return the historical result without changing the new live head. | +| Workspace name is deleted and recreated. | Parent retains the original incarnation. | Old success remains queryable; an unfinished old intent cannot write into the new instance. | + +A zero-byte action uses the same contract. It creates an empty file when absent. +A new ID may advance an existing file's generation even though its bytes remain +unchanged. Replaying that ID cannot advance the generation again. + +## Size, resource, and retention contract + +The maximum delta is 16 MiB. The default maximum resulting artifact is also +16 MiB, including the existing body and the new delta. A caller may explicitly +increase the result-size limit; that limit becomes part of the immutable +intent. Raising it does not raise the delta limit. Compaction/rematerialization +must enforce the effective bound before allocating or reading the full result. + +CLI file reads are capped even if a file grows after its initial metadata check. +Text and encoded input bounds are checked before unnecessary allocation; Python +checks byte input length before cloning it into a Rust-owned buffer. Rust +callers own their input allocation but are checked before publication I/O. +Block sizes must remain within the admitted object provider's capabilities. +CLI `--block-size` preserves complete intent parity with Python and Rust. + +Append activity leases are an operator setting, exposed by +`nokv serve --append-activity-lease-ms`; the allowed range is 1 second to 24 hours and the default is 30 minutes. Recovery also +obeys the existing clock-skew safety allowance and lifecycle scheduling. The +lease must cover the deployment's maximum interval between durable upload +progress updates. There is no independent client upload heartbeat. A short +lease can be used for controlled acceptance; its measured recovery time is not +a promise for a deployment using the default lease. + +Published and cleaned child records and logical receipts are currently retained +without automatic expiration. This is an explicit identity-retention cost. Failed attempts also retain their +revision reservations and zero-byte object seals; seals retain no former payload +bytes but still incur provider key/metadata overhead. +Removing these records without an equivalent tombstone/replay protocol would +allow forgotten IDs to execute again. Receipts preserve metadata, not permanent +access to the old revision's object bytes. Ordinary snapshot/hold/GC policies +control body retention. + +The lifecycle runner first performs a read-only consistency check over retained +publications, parents, and activity markers. It rejects corruption without +automatic repair. This initial validation cost grows with retained history. +Subsequent recovery passes scan an active-publication index, updated atomically +with child lifecycle transitions, so every routine pass does not rescan all +past receipts before finding abandoned active work. Retained row count and +storage growth still require deployment capacity planning; this change does +not introduce an unbounded-throughput or unlimited-retention SLA. + +## Downstream demand behind the contract + +The following are documented downstream requirements, not claims of existing +NoKV integrations or partnerships. Sources were checked on 2026-09-23. + +| Downstream evidence | Consequence for append | +| --- | --- | +| [Temporal AI engineering patterns](https://go.temporal.io/platform-hub/ai-engineering/ai-patterns) describes retryable tool Activities and caller-supplied idempotency keys. | An action ID must survive worker replacement and distinguish deliberate repeated actions from redelivery. | +| [AWS Durable Execution idempotency guidance](https://docs.aws.amazon.com/durable-execution/patterns/best-practices/idempotency/) requires repeat-safe external work. | A recovered append must return its original outcome; merely refusing all further attempts leaves durable workflows unable to finish. | +| [LangGraph persistence](https://docs.langchain.com/oss/python/langgraph/persistence) separates thread checkpoints from shared durable state. | A harness checkpoint alone cannot prove whether an independently committed file append happened; the state layer needs its own receipt and queryable recovery state. | + +The last column states the design inference for NoKV. These sources establish +functional demand, not a benchmark target or a promised commercial SLA. + +## Architecture decisions + +| Candidate | Decision and reason | +| --- | --- | +| One caller ID bound to one publication forever | Protects duplicate effects but strands an admitted action after safe cleanup or a lost CAS. Insufficient for redelivery that must finish. | +| Reuse exact request IDs | Useful RPC deduplication, but cannot represent a logical append across changed routing, process state, or generation-dependent plans. | +| Client-only deterministic attempt counters | Unsafe: an absent predecessor can be admitted by a delayed caller after another caller skips it. Durable predecessor fencing is required. | +| Persist the entire plan and resume that frozen upload | Requires additional descriptor/payload retention and holds; the frozen generation can still lose its CAS permanently. It does not remove the need for safe logical successors. | +| Delete failed objects and forget their keys | Rejected for stable append: a delayed PUT can recreate an untracked payload after deletion. | +| Periodically rescan all failed keys forever | Can eventually reclaim finite delayed writes, but retains an ongoing history-sized scan and leaves a reopening window after every DELETE. | +| Monotonic zero-byte seals plus revision reservations | Selected for failed append objects. Requires qualified conditional replacement and permanent small reservations; avoids delayed DELETE/PUT reopening races. | +| Separate parent advance and child admission | Creates a missing-child interval and requires another recovery protocol. Atomic Begin keeps the predecessor proof and successor admission together. | +| Logical parent plus fenced immutable attempts | Selected. Reuses publication durability, cleanup, reference lifetime, and owner fencing while adding durable redelivery completion. | +| Content hash as the identity | Suppresses two intentional actions with the same bytes. Hashes authenticate intent; they do not identify business actions. | +| One immutable event file per action | A useful downstream event model, but requires its own ordering/projection policy and changes the byte-append interface. | + +## Version and deployment boundary + +The workspace RPC schema is `nokv.workspace.rpc.v11`, the system format is 12, +and the publication value format is 6. A parent row is a new operation kind. +Older stores and protocol clients are rejected explicitly. Existing Holt stores +are inspected through the locked backend's read-only open path before writable +recovery, so a format rejection does not rewrite their files. There is no silent +migration, compatibility decoder, marker-only upgrade, or mixed-version write +mode. Operators must preserve existing data and use the repository's qualified +upgrade/export path when one exists; this feature does not manufacture an +upgrade procedure for an incompatible store. + +Red and green fault tests initialize separate version-appropriate stores. A +green restart test must reopen the exact same green Holt directory. NoKV's +locked Holt dependency, operating system, object provider, owner binary, client +binary, Python extension, source commit, and configuration belong in the +acceptance receipt. A locally checked-out Holt head is not evidence for the +backend linked into the tested NoKV binary. + +## Functional acceptance requirements + +The product gate is +[`append_product_acceptance_gate.py`](../../scripts/workbench/append_product_acceptance_gate.py). +Run its `--help` for the exact invocation. Preserve the previous frozen binary +for red tests and use a separately frozen candidate binary and matching Python +wheel for green tests. Do not replace a red failure oracle with a weaker green +one. The earlier identity gate is retained as historical evidence of the +narrower single-attempt contract. + +The functional matrix must include: + +- Same-ID concurrent fresh callers synchronized before admission; distinct IDs + contending on one file; identical bytes under independent IDs; root isolation. +- Successful-response loss at admission and publication, owner and caller + SIGKILL, same-store reopen, and caller death while the owner stays healthy. +- Staged-object and manifest boundaries, object-provider faults, actual delayed + PUT arrival after cleanup, monotonic zero-byte seals, and recovery from proven + failed attempts without an additional user action ID. A seal with an + unprovable outcome must quarantine, reject the old absence verdict, and + complete only after exact-token reconciliation with sealed-object evidence. +- Metadata-only status with the delta file gone; historical append replay + with its original bytes while the object provider is unavailable, after a + newer live generation, removal, and workspace name reuse. +- Exact typed intent and incarnation rejection, zero bytes, block boundaries, + size-limit rejection before admission, and explicit larger result bounds. +- CLI/Python/Rust semantic parity, incompatible-client/store rejection without + mutation, and deterministic metadata tests for predecessor holes, parent + receipt atomicity, cross-kind collisions, and maximum legal command shape. + +Each scenario checks both safety and completion. Compare the entire ordered +content, expected generation increments, stable historical receipt fields, +unchanged live metadata on replay, and relevant object inventory. A safe failure +cannot make a required completion scenario green. Tests must not use Python +`assert` for acceptance conditions that disappear under optimized execution. + +Record latency distributions and failures for the measured workload separately +from correctness. Report unexecuted applicable release gates as `NOT QUALIFIED`. +A command-boundary SIGKILL proves that boundary; it does not prove every internal +WAL/fsync interruption point. Internal commit barriers and machine-crash tests +must identify their exact scope and use independent normal-binary recovery. diff --git a/docs/metadata-schema.md b/docs/metadata-schema.md index b14672397..967208962 100644 --- a/docs/metadata-schema.md +++ b/docs/metadata-schema.md @@ -19,15 +19,16 @@ Every logical-shard store has one authoritative marker: System("schema") -> value_format_version = 1 schema_id = "nokv_workspace" - format_version = 11 + format_version = 12 ``` Startup is fail-closed: - an empty store is initialized with the exact supported marker and logical keyspace catalog; -- format-10 and older stores are rejected without writes; there is no marker-only - upgrade because format 11 changes the durable publication operation codec; +- format-11 and older stores are rejected without writes; there is no marker-only + upgrade because format 12 adds logical append records and changes the durable + publication operation codec; - a nonempty current store opens only when its marker, value format, and configured adapter catalog match this contract; - a missing, malformed, unknown-version, or inconsistent store is rejected. @@ -84,7 +85,7 @@ exact key is a strict prefix of another valid path key. A child/subtree prefix appends NUL, so `a` cannot match `ab`. The empty path has no `PathCurrent` record; the workspace root is synthesized from `WorkspaceCurrent`. This path key layout was introduced by system format version 8 and is retained by -version 11. +version 12. The one shared normalizer enforces: @@ -105,22 +106,24 @@ float, timestamp, bytes, and string values. ## Durable Format Registry -`System.format_version` is `11`. Version 11 adds the optional full SHA-256 -append-intent commitment to the existing publication operation record. It -retains the format-10 Generic index families and the format-9 RecoveryOutbox -fixed-width decimal LSN keys. Existing logical recovery and object formats are -unchanged. +`System.format_version` is `12`. Version 12 adds logical append parents, +publication-attempt bindings, and the active-publication recovery index in the +existing Operation family. It retains the format-11 full SHA-256 append intent, +format-10 Generic index families, and format-9 RecoveryOutbox fixed-width decimal +LSN keys. Existing logical recovery and object formats are unchanged. -Ordinary open does not migrate a format-10 or older marker. A marker-only -upgrade would reinterpret old publication records without their required codec -version. Migration remains not qualified; every older or unknown marker is -fail-closed and unchanged. +Ordinary open does not migrate a format-11 or older marker. A marker-only +upgrade would reinterpret publication records without their required codec and +omit the active index. Migration remains not qualified; every older or unknown +marker is fail-closed and unchanged. Durable codecs are independently versioned: publish operation, staged-object, and manifest-row records use value version -`5` and reject version `4`; the operation binds an optional 256-bit append -intent before any object publication. Other publication rows retain their -existing layouts. +`6` and reject version `5`; an append publication binds both the full 256-bit +intent and logical parent/attempt. `AppendOperationRecord` uses value version +`1` and retains the current attempt and optional compact receipt. An +`Operation(ActivePublish)` marker has the exact one-byte value `[1]`. Other +publication rows retain their existing layouts. publication-owned workspace/path/revision records use value version `2`; `CommitRecord` uses version `3` and dual-decodes version `2`, while its member, consumer, head, and tag records remain version `2`; `ChangeEvent` and the @@ -463,14 +466,28 @@ History val: previous versioned value or tombstone ``` -Public operation IDs are root-scoped across Publish, BuildCommit, and Restore. -Although their keys retain the lifecycle kind, each initial admission includes -absent predicates for the other two kinds in the same command as its own +Public operation IDs are root-scoped across Append, Publish, BuildCommit, and +Restore. Although their keys retain the lifecycle kind, each initial admission +includes absent predicates for the other three kinds in the same command as its own operation insertion. A competing lifecycle cannot invalidate an acknowledged operation's later lookup by admitting the same identity. Internal manifest publications retain distinct operation IDs. Terminal operation retention keeps this exclusion after success or cleanup. +A logical append parent and its child publication are admitted together. Only a +current child proven `Cleaned` permits atomically advancing the parent and +admitting a successor. Final publication writes the parent receipt together +with the child `Published` result and visible path/revision/index/reference +changes. Parent and child records are retained for replay; they do not create +an extra body-retention reference. See the +[append product contract](development/append-product-spec.md). + +`Operation(ActivePublish)` is an internal recovery index keyed by the physical +publication ID. It contains only Uploading, Finalizing, Aborting, and Cleaning +children. Publication transitions add or remove it in the same command that +changes the child. The lifecycle scans this index instead of retained terminal +history. A malformed marker or missing referenced child fails closed. + `ReadChanges` treats `(commit_version, event_sequence)` as an append-only log position. Its opaque cursor is bound to the root, query scope, and optional `after_commit_version`; unlike frozen search and catalog cursors, it may resume @@ -703,6 +720,17 @@ path/dedupe publication exists. Publication and takeover both change the same operation row, so one wins. Cleanup may mutate the ledger or issue external DELETE only while it owns `Aborting`/`Cleaning`. +For a stable append child, successful cleanup is `Aborted/Sealed`, not +`Aborted/Deleted`. Its exact staged keys become permanent zero-byte objects +through conditional creation or ETag-conditional replacement, without DELETE. +Only after sealing may cleanup remove their staging rows and reach `Cleaned`. +The child's `ArtifactRevisionClaim` is retained permanently, including when a +successor becomes current, to reserve and explain those provider keys. A late +immutable PUT cannot replace the seal. Reconciliation of a quarantined append +requires the distinct `ProviderObjectsSealed` verdict and preserves that claim; +`ProviderObjectsAbsent` is insufficient. Generic publications and published +revision GC retain their existing deletion semantics. + A late upload completion must observe the operation state; after abort it joins cleanup instead of publishing. Ambiguous multipart completion, late PUT, or DELETE remains ledger-owned and `Quarantined` until reconciled. Object listing diff --git a/scripts/workbench/append_product_acceptance_gate.py b/scripts/workbench/append_product_acceptance_gate.py new file mode 100644 index 000000000..190a67948 --- /dev/null +++ b/scripts/workbench/append_product_acceptance_gate.py @@ -0,0 +1,3261 @@ +#!/usr/bin/env python3 +# Copyright 2024-2026 The NoKV Authors. +# SPDX-License-Identifier: Apache-2.0 +"""Bounded append product acceptance over real CLI, etcd, RustFS and Holt. + +Safety, completion and performance are independent qualifications. Running every +scenario in this file does not qualify the complete NoKV workspace or cross-host +recovery. Binary provenance is independent from the current source checkout. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import contextlib +import dataclasses +import hashlib +import http.client +import http.server +import json +import os +from pathlib import Path +import platform +import shutil +import socket +import subprocess +import sys +import threading +import time +import traceback + +import append_identity_recovery_gate as base + + +class ContractViolation(RuntimeError): + """A live, observed product outcome violated an explicit oracle.""" + + +def require(condition, message, evidence=None): + # These checks must remain active under python -O. + if not condition: + raise ContractViolation(f"{message}: {evidence!r}") + + +class PublicationProxy(base.EvidenceDropProxy): + """Metadata proxy with an exact pre-Begin rendezvous and observed ACK loss.""" + + def __init__(self, port, timeout, stack): + super().__init__(port, timeout, stack) + self.rendezvous = None + self.rendezvous_entries = [] + + def begin_rendezvous(self, count): + require(self.rendezvous is None, "only one admission rendezvous") + self.rendezvous = threading.Barrier(count) + self.rendezvous_entries = [] + + def _forward(self, client): + operation = None + try: + client.settimeout(self.timeout) + with self._lock: + target = self._target + with socket.create_connection(target, timeout=self.timeout) as owner: + owner.settimeout(self.timeout) + while True: + first = client.recv(4) + if not first: + return + header = first + base._recv_exact(client, 4 - len(first)) + request = base._recv_exact(client, int.from_bytes(header, "big")) + req = None + if not base.is_handshake_frame(request): + req = base.unpack(request) + operation = req["payload"]["operation"]["operation"] + barrier = None + with self._lock: + if ( + operation == "begin_artifact_publish" + and self.rendezvous is not None + ): + if ( + len(self.rendezvous_entries) + < self.rendezvous.parties + ): + barrier = self.rendezvous + self.rendezvous_entries.append(req) + if barrier is not None: + self.stack.record( + "admission-rendezvous.jsonl", + { + "at": base.live.now(), + "request": req, + "forwarded_before_rendezvous": False, + }, + ) + barrier.wait(timeout=self.timeout) + owner.sendall(header + request) + rh = base._recv_exact(owner, 4) + response = base._recv_exact(owner, int.from_bytes(rh, "big")) + callback = None + if req is None: + self.handshake_request = request + else: + rsp = base.unpack(response) + with self._lock: + self.wire_schema = req["schema"] + self.latest_route = req["payload"]["route"] + self.stack.record( + "publication-rpc-trace.jsonl", + { + "at": base.live.now(), + "operation": operation, + "request": req, + "response": rsp, + }, + ) + if rsp["payload"]["outcome"]["status"] == "success": + with self._lock: + if self._drop_operation == operation: + callback, self._on_drop = self._on_drop, None + self._drop_operation = None + if callback is not None: + self.last_drop = { + "at": base.live.now(), + "operation": operation, + "request": req, + "response": rsp, + "request_hex": request.hex(), + "response_hex": response.hex(), + "delivered_to_client": False, + "owner_pid": self.stack.owner.pid, + } + self.stack.record("dropped-rpc.jsonl", self.last_drop) + if callback is not None: + callback() + self._dropped.set() + return + client.sendall(rh + response) + except (ConnectionError, OSError, TimeoutError) as error: + # Closing a killed caller/owner connection is the selected fault. + self.stack.record( + "proxy-connections.jsonl", + {"at": base.live.now(), "operation": operation, "closure": repr(error)}, + ) + except Exception as error: + self._errors.append(repr(error)) + self._dropped.set() + finally: + client.close() + + +class ObjectProxy: + """HTTP transport faults after real provider persistence, without changing SigV4.""" + + def __init__(self, stack, port, upstream_port): + self.stack, self.port, self.upstream_port = stack, port, upstream_port + self.lock = threading.Lock() + self.rule = None + self.hold_put = False + self.held = threading.Event() + self.release = threading.Event() + self.released_response = threading.Event() + self.held_request = None + self.reject_matching = None + self.rejected_requests = [] + self.fired = [] + self.errors = [] + outer = self + + class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): + pass + + def forward(self): + try: + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + chunks = bytearray() + while True: + line = self.rfile.readline() + chunks.extend(line) + length = int(line.split(b";", 1)[0], 16) + if length: + chunks.extend(self.rfile.read(length + 2)) + else: + while True: + trailer = self.rfile.readline() + chunks.extend(trailer) + if trailer == b"\r\n": + break + break + payload = bytes(chunks) + else: + payload = self.rfile.read( + int(self.headers.get("Content-Length", "0")) + ) + with outer.lock: + rejected = ( + outer.reject_matching is not None + and self.command == outer.reject_matching["method"] + and outer.reject_matching["path_contains"] in self.path + ) + if rejected: + event = { + "at": base.live.now(), + "method": self.command, + "path": self.path, + "injected_status": 503, + "forwarded_to_provider": False, + } + with outer.lock: + outer.rejected_requests.append(event) + outer.stack.record("object-http-transcript.jsonl", event) + body = b"ServiceUnavailableInjected temporary admission outage" + self.send_response(503) + self.send_header("Content-Type", "application/xml") + self.send_header("Content-Length", str(len(body))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(body) + self.close_connection = True + return + held = False + with outer.lock: + if ( + outer.hold_put + and self.command == "PUT" + and "/nokv/artifacts/" in self.path + ): + held, outer.hold_put = True, False + outer.held_request = { + "at": base.live.now(), + "method": self.command, + "path": self.path, + "request_bytes": len(payload), + "request_sha256": hashlib.sha256(payload).hexdigest(), + "if_none_match": self.headers.get("If-None-Match"), + "if_match": self.headers.get("If-Match"), + "upstream_sent": False, + } + if held: + outer.stack.record( + "held-object-put.jsonl", dict(outer.held_request) + ) + outer.held.set() + require( + outer.release.wait(2 * outer.stack.timeout), + "held provider PUT was released within the gate deadline", + ) + with contextlib.closing( + http.client.HTTPConnection( + "127.0.0.1", + outer.upstream_port, + timeout=outer.stack.timeout, + ) + ) as upstream: + upstream.putrequest( + self.command, + self.path, + skip_host=True, + skip_accept_encoding=True, + ) + for key, value in self.headers.items(): + if key.lower() != "connection": + upstream.putheader(key, value) + upstream.putheader("Connection", "close") + upstream.endheaders(payload) + response = upstream.getresponse() + body = response.read() + rule = None + with outer.lock: + if ( + outer.rule + and self.command == outer.rule["method"] + and "/nokv/artifacts/" in self.path + and 200 <= response.status < 300 + and ( + outer.rule["action"] != "seal_ack_loss_head_failure" + or ( + self.headers.get("Content-Length") == "0" + and self.headers.get("If-Match") is not None + ) + ) + ): + rule, outer.rule = outer.rule, None + if rule["action"] == "seal_ack_loss_head_failure": + outer.reject_matching = { + "method": "HEAD", + "path_contains": self.path, + } + event = { + "at": base.live.now(), + "method": self.command, + "path": self.path, + "upstream_status": response.status, + "request_bytes": len(payload), + "request_sha256": hashlib.sha256(payload).hexdigest(), + "response_bytes": len(body), + "if_none_match": self.headers.get("If-None-Match"), + "if_match": self.headers.get("If-Match"), + "fault": rule["action"] if rule else None, + } + outer.stack.record("object-http-transcript.jsonl", event) + if held: + outer.held_request["released_upstream_response"] = event + outer.stack.record( + "held-object-put.jsonl", + {"at": base.live.now(), "upstream_sent": True, **event}, + ) + outer.released_response.set() + if rule: + with outer.lock: + outer.fired.append(event) + if rule["action"] in ( + "drop_success", + "seal_ack_loss_head_failure", + ): + self.close_connection = True + self.connection.shutdown(socket.SHUT_RDWR) + return + if rule["action"] == "corrupt_body": + require( + bool(body), + "corruption needs a nonempty provider read", + ) + body = bytes([body[0] ^ 1]) + body[1:] + elif rule["action"] == "read_503": + body = b"ServiceUnavailable" + status = ( + 503 + if rule and rule["action"] == "read_503" + else response.status + ) + self.send_response(status) + for key, value in response.getheaders(): + if key.lower() not in ( + "transfer-encoding", + "content-length", + "connection", + ): + self.send_header(key, value) + self.send_header( + "Content-Length", + response.getheader("Content-Length", "0") + if self.command == "HEAD" + else str(len(body)), + ) + self.send_header("Connection", "close") + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + self.close_connection = True + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + except (ConnectionRefusedError, TimeoutError) as error: + outer.stack.record( + "object-http-transcript.jsonl", + { + "at": base.live.now(), + "method": self.command, + "path": self.path, + "upstream_transport_error": type(error).__name__, + }, + ) + self.close_connection = True + except Exception as error: + outer.errors.append(repr(error)) + self.close_connection = True + + do_GET = do_PUT = do_HEAD = do_DELETE = do_POST = forward + + self.server = http.server.ThreadingHTTPServer(("127.0.0.1", port), Handler) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + def start(self): + self.thread.start() + + def arm(self, method, action): + with self.lock: + require(self.rule is None, "object fault already armed") + self.rule = {"method": method, "action": action} + return len(self.fired) + + def verify_fault(self, before): + require( + len(self.fired) == before + 1 and self.rule is None, + "object fault actually reached successful provider response", + self.fired, + ) + require(not self.errors, "object proxy errors", self.errors) + return self.fired[-1] + + def hold_next_put(self): + with self.lock: + require(not self.hold_put, "provider PUT already held") + self.hold_put = True + self.held.clear() + self.release.clear() + self.released_response.clear() + self.held_request = None + + def close(self): + self.release.set() + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=2) + + +class ProductStack(base.IsolatedStack): + def __init__(self, *args, publication_lease_ms=None, **kwargs): + super().__init__(*args, **kwargs) + self.proxy = PublicationProxy(self.proxy_port, self.timeout, self) + self.callers = {} + self.append_timings = [] + self.publication_lease_ms = publication_lease_ms + self.object_proxy = ObjectProxy(self, base.infra.free_port(), self.s3_port) + self.object_endpoint = f"http://127.0.0.1:{self.object_proxy.port}" + self.config = dataclasses.replace( + self.config, object_endpoint=self.object_endpoint + ) + + def __enter__(self): + self.object_proxy.start() + return super().__enter__() + + def start(self, label, command): + command = list(command) + if label.startswith("owner-") and self.publication_lease_ms is not None: + command[-1:-1] = [ + "--append-activity-lease-ms", + str(self.publication_lease_ms), + ] + return super().start(label, command) + + def run( + self, + command, + *, + label="command", + check=True, + env=None, + timeout=None, + input_text=None, + ): + command = list(map(str, command)) + started = base.live.now() + process = subprocess.Popen( + command, + cwd=self.source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE if input_text is not None else None, + env=env, + ) + self.processes.append(process) + if "workspace-path" in command and "append" in command: + with self.lock: + self.callers[label] = process + timed_out = False + try: + stdout, stderr = process.communicate( + input_text, timeout=timeout or self.timeout + ) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + result = subprocess.CompletedProcess( + command, process.returncode, stdout, stderr + ) + redacted = [ + value.split("=")[0] + "=" + if value.startswith(("RUSTFS_ACCESS_KEY=", "RUSTFS_SECRET_KEY=")) + else value + for value in base.live.redact_argv(command) + ] + self.record( + "processes.jsonl", + { + "label": label, + "pid": process.pid, + "argv": redacted, + "started_at": started, + "ended_at": base.live.now(), + "returncode": result.returncode, + "timeout": timed_out, + "stdout": stdout, + "stderr": stderr, + }, + ) + if timed_out: + raise TimeoutError(f"{label} timed out with pid {process.pid}") + if check and result.returncode: + raise RuntimeError(f"{label}: {stderr or stdout}") + return result + + def kill_caller(self, label): + with self.lock: + process = self.callers.get(label) + require( + process is not None and process.poll() is None, + "fault targets a live caller", + label, + ) + self.record( + "caller-faults.jsonl", + { + "at": base.live.now(), + "caller_pid": process.pid, + "label": label, + "signal": "SIGKILL", + "owner_pid": self.owner.pid, + "owner_alive": self.owner.poll() is None, + }, + ) + process.kill() + + def close(self): + if self.closed: + return + try: + super().close() + finally: + self.object_proxy.close() + + +def append( + stack, + workbench, + operation_id, + payload, + *, + label, + path="effects.log", + content_type="text/plain", + max_size=16 * 1024 * 1024, + incarnation=None, + config=None, + block_size=None, +): + payload = payload.encode() if isinstance(payload, str) else payload + fixture = stack.evidence_dir / (label + ".input") + fixture.write_bytes(payload) + command = [ + *base.live.client_args(config or stack.config), + "workspace-path", + "append", + workbench, + "logs", + path, + "--operation-id", + operation_id, + "--file", + str(fixture), + ] + if content_type is not None: + command += ["--content-type", content_type] + if max_size is not None: + command += ["--max-logical-size", str(max_size)] + if incarnation is not None: + command += ["--expected-workspace-incarnation-id", incarnation] + if block_size is not None: + command += ["--block-size", str(block_size)] + started = time.monotonic() + process = stack.run(command, label=label, check=False) + elapsed = time.monotonic() - started + if process.returncode == -9: + result = {"status": "caller_killed", "operation_id": operation_id} + else: + raw = ( + process.stdout + if process.returncode == 0 + else process.stderr.strip().removeprefix("nokv: ") + ) + try: + result = json.loads(raw) + except json.JSONDecodeError as error: + raise ContractViolation( + f"append returned no structured result: {raw}" + ) from error + require( + (process.returncode == 0) == (result.get("status") == "success"), + "append exit/result agree", + result, + ) + timing = { + "label": label, + "operation_id": operation_id, + "wall_seconds": elapsed, + "payload_bytes": len(payload), + "returncode": process.returncode, + "status": result.get("status"), + "code": result.get("code"), + "cause_code": result.get("details", {}).get("cause_code"), + } + with stack.lock: + stack.append_timings.append(timing) + stack.record( + "product-append-transcript.jsonl", + { + "at": base.live.now(), + **timing, + "workbench": workbench, + "path": path, + "payload_sha256": hashlib.sha256(payload).hexdigest(), + "result": result, + }, + ) + return result + + +def case_result(*, safety="NOT QUALIFIED", completion="NOT QUALIFIED", **evidence): + return { + "safety": safety, + "completion": completion, + "performance": "NOT QUALIFIED", + **evidence, + } + + +def record_machine_profile(stack): + profile = { + "os": platform.platform(), + "machine": platform.machine(), + "logical_cpu_count": os.cpu_count(), + "python_version": sys.version, + "python_optimization": sys.flags.optimize, + "append_activity_lease_ms": stack.publication_lease_ms, + "rustfs_image_digest": base.PINNED_RUSTFS_IMAGE, + "provider_endpoint_class": "loopback HTTP to an isolated digest-pinned RustFS container through an evidence proxy", + "bucket_policy_scope": "Fresh isolated bucket created by this runner; no external provider qualification is inferred.", + } + commands = { + "observed_rust_toolchain_not_build_proof": ["rustc", "--version", "--verbose"], + "etcd_version": [stack.etcd_binary, "--version"], + "aws_cli_version": [stack.aws, "--version"], + "filesystem_capacity": ["df", "-k", str(stack.evidence_dir)], + } + if sys.platform == "darwin": + commands["hardware_model_cpu_memory"] = [ + "sysctl", + "-n", + "hw.model", + "hw.logicalcpu", + "hw.memsize", + ] + for label, command in commands.items(): + if shutil.which(str(command[0])) is None: + profile[label] = { + "status": "NOT QUALIFIED", + "reason": "executable unavailable", + } + continue + result = stack.run(command, label="environment-" + label, check=False) + profile[label] = { + "returncode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + } + stack.json("machine-profile.json", profile) + return "machine-profile.json" + + +def canonical_receipt(value): + fields = ( + "operation_id", + "publication_operation_id", + "artifact_revision_id", + "workbench_id", + "path", + "workspace_incarnation_id", + "workspace_revision", + "generation", + "logical_size", + "body_digest", + ) + require( + all(field in value for field in fields), + "complete logical/publication receipt", + value, + ) + return {field: value[field] for field in fields} + + +def validate_parent_child_mapping(stack, operation_id, status, receipt): + require( + status.get("operation_id") == operation_id + and status.get("state") == "committed", + "logical parent is committed", + status, + ) + attempt = status.get("attempt") + require( + isinstance(attempt, int) and attempt >= 0, + "status declares a nonnegative durable attempt", + status, + ) + suffix = ( + bytes.fromhex(stack.root_id) + + bytes.fromhex(operation_id) + + attempt.to_bytes(8, "big") + ) + child = hashlib.sha256(b"nokv.append.publication.v2\0" + suffix).hexdigest()[:32] + revision = hashlib.sha256(b"nokv.append.revision.v2\0" + suffix).hexdigest()[:32] + require( + status.get("publication_operation_id") + == receipt["publication_operation_id"] + == child, + "root/logical-ID/attempt derive the committed child", + status, + ) + require( + receipt["artifact_revision_id"] == revision + and receipt["operation_id"] == operation_id, + "receipt binds logical identity to derived revision", + receipt, + ) + require( + canonical_receipt(status["receipt"]) == canonical_receipt(receipt), + "parent stores complete original receipt", + status, + ) + + +def operation_status(stack, operation_id, label, *, config=None): + cfg = config or stack.config + command = [ + str(stack.binary), + *base.live.control_args(cfg), + "--agent-id", + cfg.agent_id, + "operation", + "status", + operation_id, + ] + response = stack.run(command, label=label, check=False) + raw = ( + response.stdout + if response.returncode == 0 + else response.stderr.strip().removeprefix("nokv: ") + ) + try: + result = json.loads(raw) + except json.JSONDecodeError as error: + raise ContractViolation( + f"operation status returned no structured JSON: {raw}" + ) from error + require( + (response.returncode == 0) == (result.get("status") == "success"), + "status exit/result agree", + result, + ) + stack.record( + "product-status-transcript.jsonl", + { + "at": base.live.now(), + "operation_id": operation_id, + "label": label, + "result": result, + "object_configuration_supplied": False, + }, + ) + return result + + +def live_path(stack, workbench, label, path="effects.log"): + # The raw read is supplementary evidence through the same public owner RPC; + # the mutation itself and final byte read always use the native product CLI. + result = base.raw_rpc( + stack, + { + "operation": "get_path", + "request": { + "target": {"workbench": workbench, "path": "logs/" + path}, + "view": "live", + "expected_read_version": None, + "range": None, + "plan_page": None, + "if_none_match": None, + }, + }, + label, + ) + require(result.get("status") == "success", "independent live path query", result) + body = result["body"] + # Remove read-time metadata; canonical path/revision/generation remain. + return body + + +def inventory(stack, label): + from fork_restore_recovery_gate import object_inventory + + result = object_inventory( + stack.aws, + stack.object_endpoint, + stack.bucket, + stack.object_root, + stack.aws_env, + stack.source, + stack.timeout, + stack.evidence, + label, + ) + return {key: value for key, value in result.items() if "/nokv/artifacts/" in key} + + +def replay_unchanged( + stack, + workbench, + operation_id, + payload, + original, + label, + *, + repetitions=2, + **options, +): + validate_parent_child_mapping( + stack, + operation_id, + operation_status(stack, operation_id, label + "-parent-status"), + original, + ) + before = live_path(stack, workbench, label + "-live-before") + objects_before = inventory(stack, label + "-objects-before") + outcomes = [] + for index in range(repetitions): + result = append( + stack, + workbench, + operation_id, + payload, + label=f"{label}-{index:04d}", + **options, + ) + require( + result.get("status") == "success" and result.get("replayed") is True, + "same-ID replay succeeds", + result, + ) + require( + canonical_receipt(result) == canonical_receipt(original), + "complete historical receipt unchanged", + (result, original), + ) + outcomes.append(result) + after = live_path(stack, workbench, label + "-live-after") + require( + after == before, + "replay does not change live revision, generation or path metadata", + (before, after), + ) + require( + inventory(stack, label + "-objects-after") == objects_before, + "replay creates no payload object", + ) + return { + "repetitions": repetitions, + "live_path_unchanged": True, + "payload_inventory_unchanged": True, + "receipt": canonical_receipt(outcomes[-1]), + } + + +def caller_boundary_completion(stack, deadline, operation): + suffix = ( + operation.removesuffix("_artifact_publish") + .removeprefix("stage_artifact_") + .removeprefix("mark_artifact_") + ) + workbench = "product-caller-" + suffix.replace("_", "-") + operation_id = hashlib.sha256(workbench.encode()).hexdigest()[:32] + text = "caller-recovery-event\n" + label = "caller-kill-" + suffix + stack.cli("workbench_create", {"id": workbench}) + owner_pid = stack.owner.pid + stack.proxy.arm(operation, lambda: stack.kill_caller(label)) + first = append( + stack, workbench, operation_id, text, label=label, max_size=1024 * 1024 + ) + stack.proxy.wait_dropped() + require(first.get("status") == "caller_killed", "caller was actually killed", first) + require( + stack.owner.pid == owner_pid and stack.owner.poll() is None, + "same healthy owner survived caller death", + ) + admitted = operation_status(stack, operation_id, label + "-status-after-kill") + require( + admitted.get("status") == "success", + "logical identity admitted before caller death", + admitted, + ) + recovered, outcomes = recover_same_identity( + stack, + workbench, + operation_id, + text, + label=label + "-recovery", + deadline=deadline, + observed_status=admitted, + ) + if recovered is None: + return case_result( + safety="NOT QUALIFIED", + completion="FAIL", + failure="Caller died after a durable publication boundary and the same logical request did not complete within the declared deadline.", + boundary=operation, + owner_survived=True, + status_after_kill=admitted, + retry_outcomes=outcomes, + ) + exact_body(stack, workbench, text.encode(), label + "-bytes") + replay = replay_unchanged( + stack, + workbench, + operation_id, + text, + recovered, + label + "-replay", + max_size=1024 * 1024, + ) + status = operation_status(stack, operation_id, label + "-terminal-status") + require( + status.get("state") == "committed" and status.get("next_action") == "none", + "terminal status is actionable", + status, + ) + require( + canonical_receipt(status["receipt"]) == canonical_receipt(recovered), + "status original receipt", + status, + ) + return case_result( + safety="PASS", + completion="PASS", + boundary=operation, + caller_sigkill=True, + same_owner_pid=owner_pid, + status_after_kill=admitted, + terminal_status=status, + replay=replay, + retry_outcomes=outcomes, + ) + + +def deterministic_same_identity(stack, deadline): + workbench, operation_id, text = ( + "product-admission-race", + "c2" * 16, + "one-concurrent-event\n", + ) + stack.cli("workbench_create", {"id": workbench}) + stack.proxy.begin_rendezvous(2) + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + futures = [ + pool.submit( + append, + stack, + workbench, + operation_id, + text, + label=f"admission-racer-{index}", + max_size=1024 * 1024, + ) + for index in range(2) + ] + first = [future.result() for future in futures] + entries = stack.proxy.rendezvous_entries + require( + len(entries) == 2, + "two Begin requests were held before either was forwarded", + entries, + ) + finally: + stack.proxy.rendezvous = None + recovered, outcomes = recover_same_identity( + stack, + workbench, + operation_id, + text, + label="admission-reconcile", + deadline=deadline, + ) + require(recovered is not None, "concurrent same identity completes", outcomes) + for result in first: + if result.get("status") == "success": + require( + canonical_receipt(result) == canonical_receipt(recovered), + "all acknowledged results conserved", + first, + ) + else: + require( + result.get("details", {}).get("operation_id") == operation_id + and result.get("code") == "AppendUnresolved", + "admission race error is typed and keeps identity", + result, + ) + require( + recovered["generation"] == 1, "exactly one publication generation", recovered + ) + exact_body(stack, workbench, text.encode(), "admission-exact-body") + replay = replay_unchanged( + stack, + workbench, + operation_id, + text, + recovered, + "admission-replay", + max_size=1024 * 1024, + ) + intent_race = different_intent_admission(stack) + return case_result( + safety="PASS", + completion="PASS", + requests_held_before_forward=2, + initial_results=first, + retry_outcomes=outcomes, + replay=replay, + different_intent_race=intent_race, + ) + + +def different_intent_admission(stack): + workbench, operation_id = "product-admission-intent-race", "cf" * 16 + payloads = (b"intent-A\n", b"intent-B\n") + stack.cli("workbench_create", {"id": workbench}) + stack.proxy.begin_rendezvous(2) + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + futures = [ + pool.submit( + append, + stack, + workbench, + operation_id, + payload, + label=f"intent-racer-{index}", + ) + for index, payload in enumerate(payloads) + ] + results = [future.result() for future in futures] + require( + len(stack.proxy.rendezvous_entries) == 2, + "two distinct intents reached Begin before either was forwarded", + ) + finally: + stack.proxy.rendezvous = None + successful = [ + index + for index, result in enumerate(results) + if result.get("status") == "success" + ] + require( + len(successful) == 1, + "healthy concurrent different intents have exactly one successful logical winner", + results, + ) + winner_index = successful[0] + winner, loser = results[winner_index], results[1 - winner_index] + require( + loser.get("code") == "AppendUnresolved" + and loser.get("details", {}).get("code") == "RequestReplayMismatch" + and loser.get("details", {}).get("operation_id") == operation_id, + "losing intent has a precise replay-mismatch error", + loser, + ) + exact_body(stack, workbench, payloads[winner_index], "intent-race-exact-winner") + replay = replay_unchanged( + stack, + workbench, + operation_id, + payloads[winner_index], + winner, + "intent-race-winner-replay", + ) + return { + "initial_results": results, + "winner_index": winner_index, + "exact_expected_bytes_hex": payloads[winner_index].hex(), + "replay": replay, + } + + +def object_put_response_loss(stack, deadline): + workbench, operation_id, payload = ( + "product-object-put-loss", + "c3" * 16, + "provider-persisted-event\n", + ) + stack.cli("workbench_create", {"id": workbench}) + before = stack.object_proxy.arm("PUT", "drop_success") + first = append( + stack, + workbench, + operation_id, + payload, + label="put-loss-first", + max_size=1024 * 1024, + ) + fault = stack.object_proxy.verify_fault(before) + recovered, outcomes = recover_same_identity( + stack, + workbench, + operation_id, + payload, + label="put-loss-recover", + deadline=deadline, + ) + require( + recovered is not None, + "same identity completes after actual successful PUT response loss", + outcomes, + ) + if first.get("status") == "success": + require( + canonical_receipt(first) == canonical_receipt(recovered), + "PUT retry preserves acknowledged receipt", + first, + ) + exact_body(stack, workbench, payload.encode(), "put-loss-exact-body") + replay = replay_unchanged( + stack, + workbench, + operation_id, + payload, + recovered, + "put-loss-replay", + max_size=1024 * 1024, + ) + return case_result( + safety="PASS", + completion="PASS", + fault=fault, + first_call=first, + retry_outcomes=outcomes, + replay=replay, + ) + + +def object_read_integrity(stack, deadline): + workbench, seed_id, append_id = "product-object-read-fault", "c4" * 16, "c5" * 16 + stack.cli("workbench_create", {"id": workbench}) + seed = append(stack, workbench, seed_id, "seed\n", label="read-fault-seed") + require(seed.get("status") == "success", "read-fault seed", seed) + before_live = live_path(stack, workbench, "read-fault-live-before") + before = stack.object_proxy.arm("GET", "corrupt_body") + result = append( + stack, + workbench, + append_id, + "tail\n", + label="read-fault-first", + max_size=1024 * 1024, + ) + fault = stack.object_proxy.verify_fault(before) + require( + result.get("status") == "error", + "corrupt provider bytes cannot be published", + result, + ) + require( + live_path(stack, workbench, "read-fault-live-rejected") == before_live, + "corruption leaves live path unchanged", + ) + recovered, outcomes = recover_same_identity( + stack, + workbench, + append_id, + "tail\n", + label="read-fault-recover", + deadline=deadline, + ) + require( + recovered is not None, + "same identity works after a pre-admission read fault clears", + outcomes, + ) + exact_body(stack, workbench, b"seed\ntail\n", "read-fault-final") + return case_result( + safety="PASS", + completion="PASS", + fault=fault, + first_call=result, + retry_outcomes=outcomes, + ) + + +def distinct_identity_exact_bytes(stack, deadline): + workbench, count = "product-distinct-race", 4 + stack.cli("workbench_create", {"id": workbench}) + ids = [ + hashlib.sha256(f"distinct-{index}".encode()).hexdigest()[:32] + for index in range(count) + ] + payloads = [f"event-{index:02d}\n" for index in range(count)] + barrier = threading.Barrier(count) + workload_started = time.monotonic() + + def call(index): + barrier.wait(timeout=stack.timeout) + return append( + stack, + workbench, + ids[index], + payloads[index], + label=f"distinct-initial-{index}", + max_size=1024 * 1024, + ) + + with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool: + initial = list(pool.map(call, range(count))) + receipts = [] + attempt_indexes = {} + for index in range(count): + result, outcomes = recover_same_identity( + stack, + workbench, + ids[index], + payloads[index], + label=f"distinct-reconcile-{index}", + deadline=deadline, + ) + require( + result is not None, "each distinct request eventually completes", outcomes + ) + if initial[index].get("status") == "success": + require( + canonical_receipt(initial[index]) == canonical_receipt(result), + "initial acknowledgement conserved", + (initial[index], result), + ) + receipts.append(result) + status = operation_status( + stack, ids[index], f"distinct-terminal-attempt-{index}" + ) + validate_parent_child_mapping(stack, ids[index], status, result) + attempt_indexes[ids[index]] = status["attempt"] + workload_seconds = time.monotonic() - workload_started + generations = [receipt["generation"] for receipt in receipts] + require( + sorted(generations) == list(range(1, count + 1)), + "one contiguous generation per logical append", + receipts, + ) + ordered = sorted(zip(receipts, payloads), key=lambda pair: pair[0]["generation"]) + expected = "".join(payload for _, payload in ordered).encode() + exact_body(stack, workbench, expected, "distinct-complete-byte-oracle") + live_before = live_path(stack, workbench, "distinct-live-before-replay") + for index in range(count): + result = append( + stack, + workbench, + ids[index], + payloads[index], + label=f"distinct-history-{index}", + max_size=1024 * 1024, + ) + require( + canonical_receipt(result) == canonical_receipt(receipts[index]), + "distinct historical receipts preserved", + result, + ) + require( + live_path(stack, workbench, "distinct-live-after-replay") == live_before, + "historical replays preserve final live generation", + ) + stack.restart() + exact_body(stack, workbench, expected, "distinct-reopened-byte-oracle") + timings = [ + value + for value in stack.append_timings + if value["label"].startswith(("distinct-initial-", "distinct-reconcile-")) + ] + ordered_latencies = sorted(value["wall_seconds"] for value in timings) + + def percentile(fraction): + position = fraction * (len(ordered_latencies) - 1) + low = int(position) + high = min(low + 1, len(ordered_latencies) - 1) + return ordered_latencies[low] + ( + ordered_latencies[high] - ordered_latencies[low] + ) * (position - low) + + measurement = { + "qualification": "Measured diagnostic sample, no performance SLA is asserted.", + "process_model": "Fresh native CLI per call; running shared services; cache state not controlled.", + "initial_concurrency": count, + "logical_operations": count, + "application_calls": len(timings), + "error_calls": sum(value["status"] != "success" for value in timings), + "calls_per_identity": { + op: sum(value["operation_id"] == op for value in timings) for op in ids + }, + "last_durable_attempt_index": attempt_indexes, + "payload_bytes": [len(payload.encode()) for payload in payloads], + "workload_seconds_including_terminal_status_queries": workload_seconds, + "logical_operations_per_second": count / workload_seconds, + "latency_seconds": { + "p50": percentile(0.50), + "p95": percentile(0.95), + "p99": percentile(0.99), + "max": max(ordered_latencies), + }, + "raw_calls": timings, + } + return case_result( + safety="PASS", + completion="PASS", + initial_results=initial, + receipts=receipts, + exact_expected_utf8=expected.decode(), + owner_reopen_verified=True, + performance_measurement=measurement, + ) + + +def cross_root_identity(stack): + workbench, operation_id = "product-cross-root", "d1" * 16 + root_b = hashlib.sha256((stack.root_id + "second-root").encode()).hexdigest()[:32] + config_b = dataclasses.replace(stack.config, root_id=root_b) + stack.run(base.live.provision_command(config_b), label="provision-second-root") + stack.restart() + stack.cli("workbench_create", {"id": workbench}) + stack.cli("workbench_create", {"id": workbench}, config=config_b) + first = append(stack, workbench, operation_id, b"root-A\n", label="root-a-append") + second = append( + stack, + workbench, + operation_id, + b"root-B\n", + label="root-b-append", + config=config_b, + ) + require( + first.get("status") == second.get("status") == "success", + "same caller ID independently admitted in two roots", + (first, second), + ) + require( + first["publication_operation_id"] != second["publication_operation_id"] + and first["artifact_revision_id"] != second["artifact_revision_id"], + "root binds child and revision identities", + (first, second), + ) + status_a = operation_status(stack, operation_id, "root-a-status") + status_b = operation_status(stack, operation_id, "root-b-status", config=config_b) + require( + canonical_receipt(status_a["receipt"]) == canonical_receipt(first) + and canonical_receipt(status_b["receipt"]) == canonical_receipt(second), + "status cannot expose another root's receipt", + ) + exact_body(stack, workbench, b"root-A\n", "root-a-final") + target = stack.evidence_dir / "root-b-final.data" + stack.run( + [ + *base.live.client_args(config_b), + "materialize", + workbench, + "logs", + "effects.log", + str(target), + ], + label="root-b-final", + ) + require(target.read_bytes() == b"root-B\n", "root B exact independent content") + return case_result( + safety="PASS", + completion="PASS", + root_a=stack.root_id, + root_b=root_b, + receipt_a=first, + receipt_b=second, + ) + + +def boundary_and_capacity(stack): + block = 4 * 1024 * 1024 + outcomes = [] + for index, size in enumerate((0, 1, block - 1, block, block + 1)): + workbench = f"product-size-{size}" + operation_id = hashlib.sha256(workbench.encode()).hexdigest()[:32] + stack.cli("workbench_create", {"id": workbench}) + payload = (b"\0\xffboundary\n" * (size // 11 + 1))[:size] + require(len(payload) == size, "boundary fixture exact size") + result = append( + stack, + workbench, + operation_id, + payload, + label=f"size-{size}", + content_type="application/octet-stream", + max_size=None, + ) + require( + result.get("status") == "success" + and result.get("logical_size") == size + and result.get("generation") == 1, + "zero/block-boundary append", + result, + ) + exact_body(stack, workbench, payload, f"size-{size}-body") + replay = replay_unchanged( + stack, + workbench, + operation_id, + payload, + result, + f"size-{size}-replay", + repetitions=1, + content_type="application/octet-stream", + max_size=16 * 1024 * 1024, + ) + outcomes.append( + {"size": size, "receipt": result, "default_explicit_limit_replay": replay} + ) + if size == 1: + empty_id = "d0" * 16 + empty = append( + stack, + workbench, + empty_id, + b"", + label="existing-zero-delta", + content_type="application/octet-stream", + max_size=None, + ) + require( + empty.get("status") == "success" + and empty["generation"] == 2 + and empty["logical_size"] == 1, + "new identity with empty delta publishes one new generation", + empty, + ) + replay_unchanged( + stack, + workbench, + empty_id, + b"", + empty, + "existing-zero-replay", + repetitions=1, + content_type="application/octet-stream", + max_size=None, + ) + exact_body(stack, workbench, payload, "existing-zero-unchanged-bytes") + workbench = "product-body-limit" + stack.cli("workbench_create", {"id": workbench}) + limit = 16 * 1024 * 1024 + full = b"x" * limit + first = append( + stack, workbench, "d2" * 16, full, label="body-limit-exact", max_size=None + ) + require( + first.get("status") == "success" and first["logical_size"] == limit, + "default exact body limit admitted", + first, + ) + before = live_path(stack, workbench, "body-limit-live-before") + objects_before = inventory(stack, "body-limit-inventory-before") + over = append( + stack, workbench, "d3" * 16, b"+", label="body-limit-plus-one", max_size=None + ) + require( + over.get("status") == "error" + and over.get("code") in ("InvalidArgument", "AppendUnresolved"), + "body limit rejected with typed error", + over, + ) + oversized = append( + stack, + workbench, + "d4" * 16, + b"y" * (limit + 1), + label="delta-limit-plus-one", + max_size=64 * 1024 * 1024, + ) + require( + oversized.get("status") == "error" + and oversized.get("code") == "InvalidArgument", + "oversized delta rejected before admission", + oversized, + ) + for op in ("d3" * 16, "d4" * 16): + status = operation_status(stack, op, "capacity-unadmitted-" + op[:2]) + require( + status.get("status") == "error" and status.get("code") == "NotFound", + "capacity rejects have no durable operation", + status, + ) + require( + live_path(stack, workbench, "body-limit-live-after-rejects") == before, + "capacity rejection preserves live metadata", + ) + require( + inventory(stack, "body-limit-inventory-after") == objects_before, + "capacity rejection writes no payload objects", + ) + raised = append( + stack, + workbench, + "d5" * 16, + b"+", + label="body-limit-explicit-increase", + max_size=32 * 1024 * 1024, + ) + require( + raised.get("status") == "success" and raised["logical_size"] == limit + 1, + "explicit larger body bound allows qualified delta", + raised, + ) + exact_body(stack, workbench, full + b"+", "body-limit-raised-content") + changed_bound = append( + stack, + workbench, + "d5" * 16, + b"+", + label="body-limit-intent-mismatch", + max_size=64 * 1024 * 1024, + ) + require( + changed_bound.get("details", {}).get("code") == "RequestReplayMismatch", + "admitted explicit body bound is immutable intent", + changed_bound, + ) + return case_result( + safety="PASS", + completion="PASS", + boundary_results=outcomes, + delta_limit_bytes=limit, + body_limit_default_bytes=limit, + default_overflow=over, + oversized_delta=oversized, + explicit_increase=raised, + changed_bound=changed_bound, + ) + + +def publication_batch_boundaries(stack): + block_size = 65536 + outcomes = [] + for blocks in (191, 192, 193): + workbench = f"product-batch-{blocks}" + operation_id = hashlib.sha256(workbench.encode()).hexdigest()[:32] + stack.cli("workbench_create", {"id": workbench}) + payload = b"".join( + index.to_bytes(4, "big") * (block_size // 4) for index in range(blocks) + ) + result = append( + stack, + workbench, + operation_id, + payload, + label=f"batch-{blocks}-original", + content_type="application/octet-stream", + block_size=block_size, + ) + require( + result.get("status") == "success" + and result.get("logical_size") == blocks * block_size, + "bounded batch-spanning append succeeds", + result, + ) + exact_body(stack, workbench, payload, f"batch-{blocks}-readback") + replay = replay_unchanged( + stack, + workbench, + operation_id, + payload, + result, + f"batch-{blocks}-replay", + repetitions=1, + content_type="application/octet-stream", + block_size=block_size, + ) + outcomes.append( + { + "blocks": blocks, + "block_size": block_size, + "receipt": result, + "replay": replay, + } + ) + return case_result( + safety="PASS", completion="PASS", max_publication_batch_rows=192, rows=outcomes + ) + + +def completed_replay_provider_unavailable(stack): + workbench, operation_id, payload = ( + "product-replay-provider-unavailable", + "e8" * 16, + b"durable-before-outage\n", + ) + stack.cli("workbench_create", {"id": workbench}) + original = append( + stack, workbench, operation_id, payload, label="provider-outage-original" + ) + require(original.get("status") == "success", "provider outage original", original) + before = live_path(stack, workbench, "provider-outage-live-before") + unavailable = dataclasses.replace( + stack.config, object_endpoint=f"http://127.0.0.1:{base.infra.free_port()}" + ) + replay = append( + stack, + workbench, + operation_id, + payload, + label="provider-unavailable-replay", + config=unavailable, + ) + require( + replay.get("status") == "success" + and replay.get("replayed") is True + and canonical_receipt(replay) == canonical_receipt(original), + "completed append replay requires only metadata despite unreachable S3 endpoint", + replay, + ) + (stack.evidence_dir / "provider-outage-original.input").unlink() + status = operation_status( + stack, operation_id, "provider-unavailable-status-without-payload" + ) + require( + canonical_receipt(status["receipt"]) == canonical_receipt(original), + "ID-only status works after payload file deletion", + ) + require( + live_path(stack, workbench, "provider-outage-live-after") == before, + "offline-provider replay does not mutate live metadata", + ) + return case_result( + safety="PASS", + completion="PASS", + original_receipt=original, + replay_receipt=replay, + status_without_payload=status, + object_endpoint_unreachable=True, + ) + + +def late_put_after_cleanup(stack, deadline): + workbench, operation_id, payload = ( + "product-late-put", + "fc" * 16, + b"one-late-provider-event\n", + ) + label = "late-put-caller" + stack.cli("workbench_create", {"id": workbench}) + owner_pid = stack.owner.pid + stack.object_proxy.hold_next_put() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit( + append, + stack, + workbench, + operation_id, + payload, + label=label, + max_size=1024 * 1024, + ) + require( + stack.object_proxy.held.wait(stack.timeout), + "actual complete artifact PUT received before forwarding", + ) + held = stack.object_proxy.held_request + stack.kill_caller(label) + require( + future.result().get("status") == "caller_killed", + "caller SIGKILL while actual PUT is held", + ) + require( + stack.owner.pid == owner_pid and stack.owner.poll() is None, + "owner survives held-PUT caller death", + ) + until = time.monotonic() + deadline + observations = [] + while True: + status = operation_status( + stack, operation_id, f"late-put-cleanup-status-{len(observations):03d}" + ) + observations.append(status) + if status.get("next_action") == "resubmit_same": + break + if time.monotonic() >= until: + return case_result( + completion="FAIL", + failure="Held old PUT did not reach a clean recoverable predecessor before the declared deadline.", + status_observations=observations, + held_request=held, + ) + time.sleep(0.5) + require( + str(status.get("attempt_phase")).lower() == "cleaned", + "previous attempt durably cleaned before old PUT is sent", + status, + ) + previous_child = status["publication_operation_id"] + object_key = held["path"].removeprefix("/" + stack.bucket + "/") + before_release = inventory(stack, "late-put-before-release-inventory") + guard_before = ( + object_key in before_release and before_release[object_key][0] == 0 + ) + require( + held.get("if_none_match") == "*", + "actual late immutable PUT carries If-None-Match *", + held, + ) + stack.object_proxy.release.set() + require( + stack.object_proxy.released_response.wait(stack.timeout), + "old PUT reached the real provider after cleanup", + ) + late_status = held["released_upstream_response"]["upstream_status"] + require( + 200 <= late_status < 300 or late_status == 412, + "held PUT reached real conditional provider decision", + held, + ) + recovered, outcomes = recover_same_identity( + stack, + workbench, + operation_id, + payload, + label="late-put-logical-recovery", + deadline=deadline, + observed_status=status, + ) + require( + recovered is not None, + "same logical append completes after old caller was killed", + outcomes, + ) + require( + recovered["publication_operation_id"] != previous_child, + "completed append uses a new child after previous cleanup", + recovered, + ) + exact_body(stack, workbench, payload, "late-put-exact-one-effect") + # Observe multiple real lifecycle ticks after the new child committed. + time.sleep(2) + final_objects = inventory(stack, "late-put-final-inventory") + late_object_present = object_key in final_objects + guard_after = late_object_present and final_objects[object_key][0] == 0 + safe = guard_before and guard_after and late_status == 412 + result = case_result( + safety="PASS" if safe else "FAIL", + completion="PASS", + logical_effect_safety="PASS", + object_lifetime_safety="PASS" if safe else "FAIL", + same_owner_pid=owner_pid, + status_before_release=status, + previous_publication_operation_id=previous_child, + held_request=held, + late_object_key=object_key, + zero_guard_before_release=guard_before, + zero_guard_after_release=guard_after, + late_provider_status=late_status, + late_object_present_after_cleanup=late_object_present, + old_object_size_after_cleanup=final_objects.get(object_key, [None])[0], + terminal_receipt=recovered, + status_observations=observations, + provider_request_lifetime_bound="No finite request-completion lifetime is assumed; this gate deliberately delays a fully received PUT until after metadata cleanup. Cleanup must keep an empty immutable-key guard and the late If-None-Match PUT must receive 412.", + ) + if not safe: + result["failure"] = ( + "Cleanup did not retain a zero-byte immutable-key guard that rejects the late old PUT; one logical effect does not qualify the provider object lifecycle." + ) + return result + finally: + stack.object_proxy.release.set() + + +def seal_quarantine_operator_recovery(stack, deadline): + """Resolve one real provider ambiguity using an inspected child token.""" + workbench, operation_id = "product-seal-quarantine", "fd" * 16 + payload, label = b"operator-recovered-event\n", "seal-quarantine-caller" + stack.cli("workbench_create", {"id": workbench}) + owner_pid = stack.owner.pid + stack.object_proxy.rule = {"method": "PUT", "action": "seal_ack_loss_head_failure"} + stack.proxy.arm("mark_artifact_objects_uploaded", lambda: stack.kill_caller(label)) + try: + first = append( + stack, + workbench, + operation_id, + payload, + label=label, + max_size=1024 * 1024, + block_size=65536, + ) + stack.proxy.wait_dropped() + require(first.get("status") == "caller_killed", "uploaded caller killed", first) + require( + stack.owner.pid == owner_pid and stack.owner.poll() is None, + "owner survives the uploaded caller", + ) + until, observations = time.monotonic() + deadline, [] + while True: + status = operation_status( + stack, operation_id, f"seal-quarantine-poll-{len(observations):03d}" + ) + observations.append(status) + if status.get("state") == "quarantined": + break + require( + time.monotonic() < until, + "seal ambiguity durably quarantines", + observations, + ) + time.sleep(0.5) + require( + status.get("next_action") == "operator_reconcile" + and status.get("receipt") is None + and status.get("cause_code"), + "quarantine is actionable and has no success receipt", + status, + ) + child = status["publication_operation_id"] + faults = [ + event + for event in stack.object_proxy.fired + if event.get("fault") == "seal_ack_loss_head_failure" + ] + require( + len(faults) == 1 + and faults[0]["request_bytes"] == 0 + and faults[0]["if_match"] + and 200 <= faults[0]["upstream_status"] < 300, + "a conditional empty seal really persisted before its ACK was lost", + faults, + ) + require( + any( + event["method"] == "HEAD" and event["path"] == faults[0]["path"] + for event in stack.object_proxy.rejected_requests + ), + "post-seal HEAD really failed", + ) + query = { + "operation": "get_operation", + "request": {"operation_id": list(bytes.fromhex(child))}, + } + inspected = base.raw_rpc(stack, query, "quarantine-inspected-child") + require( + inspected.get("status") == "success" + and inspected["body"]["value"]["state"] == "quarantined", + "exact quarantined publication child is inspectable", + inspected, + ) + token = inspected["body"]["value"]["token"] + retry = append( + stack, + workbench, + operation_id, + payload, + label="quarantined-same-id-cannot-advance", + max_size=1024 * 1024, + block_size=65536, + ) + require( + retry.get("status") == "error" and retry.get("code") == "AppendUnresolved", + "quarantined append cannot silently create a successor", + retry, + ) + after_retry = operation_status( + stack, operation_id, "quarantine-parent-after-retry" + ) + require( + after_retry.get("state") == "quarantined" + and after_retry.get("publication_operation_id") == child + and after_retry.get("attempt") == status["attempt"], + "same logical identity remains on the quarantined child", + after_retry, + ) + + # This controlled fault uses the acknowledged public staging transcript + # as its complete ledger. Status does not expose a staged-key listing API. + rows, counts = {}, set() + for line in ( + (stack.evidence_dir / "publication-rpc-trace.jsonl") + .read_text() + .splitlines() + ): + event = json.loads(line) + if event["response"]["payload"]["outcome"]["status"] != "success": + continue + request = event["request"]["payload"]["operation"]["request"] + if ( + event["operation"] == "begin_artifact_publish" + and bytes(request["operation_id"]).hex() == child + ): + counts.add(request["staged_object_count"]) + if ( + event["operation"] == "stage_artifact_objects" + and bytes(request["token"]["operation_id"]).hex() == child + ): + for row in request["objects"]: + require( + row["sequence"] not in rows or rows[row["sequence"]] == row, + "acknowledged staged sequence has one immutable identity", + row, + ) + rows[row["sequence"]] = row + require( + counts == {1} and set(rows) == {0}, + "acknowledged Begin and staging prove the complete single-key ledger", + (counts, rows), + ) + stack.json( + "operator-acknowledged-staged-ledger.json", + { + "child": child, + "rows": list(rows.values()), + "source": "Successful Begin and StageArtifactObjects public RPC transcript; controlled one-key fixture.", + }, + ) + with stack.object_proxy.lock: + stack.object_proxy.reject_matching = None + verified = [] + for sequence, row in sorted(rows.items()): + key = stack.object_root.rstrip("/") + "/" + row["object_identity"] + output = stack.evidence_dir / f"operator-sealed-key-{sequence}.data" + read = stack.run( + base.infra.aws_command( + stack.aws, + stack.object_endpoint, + "s3api", + "get-object", + "--bucket", + stack.bucket, + "--key", + key, + str(output), + ), + label=f"operator-provider-read-{sequence}", + env=stack.aws_env, + ) + metadata = json.loads(read.stdout) + require( + output.read_bytes() == b"" and metadata["ContentLength"] == 0, + "every acknowledged staged key is permanently sealed at the real provider", + metadata, + ) + verified.append( + { + "sequence": sequence, + "key": key, + "size": 0, + "etag": metadata["ETag"], + "sha256": hashlib.sha256(b"").hexdigest(), + } + ) + evidence = json.dumps( + {"child": child, "token": token, "verified_keys": verified}, sort_keys=True + ).encode() + (stack.evidence_dir / "operator-provider-verification.json").write_bytes( + evidence + ) + verdict = { + "operation": "reconcile_quarantined_artifact_publish", + "request": { + "token": token, + "resolution": "provider_objects_absent", + "reason": "Controlled acceptance verifies the complete acknowledged key ledger as permanently sealed.", + "evidence_digest": list(hashlib.sha256(evidence).digest()), + }, + } + wrong = base.raw_rpc(stack, verdict, "operator-obsolete-absence-verdict") + require( + wrong.get("status") == "failure" + and wrong.get("body", {}).get("code") == "precondition_failed" + and wrong.get("body", {}).get("conflict") == "operation_state" + and wrong.get("body", {}).get("retryable") is False, + "obsolete absence verdict is precisely rejected for append", + wrong, + ) + unchanged = base.raw_rpc(stack, query, "operator-child-after-wrong-verdict") + require( + unchanged == inspected, + "rejected verdict does not change inspected child/token", + unchanged, + ) + verdict["request"]["resolution"] = "provider_objects_sealed" + resolved = base.raw_rpc(stack, verdict, "operator-exact-sealed-verdict") + require( + resolved.get("status") == "success", + "exact inspected child token accepts sealed proof", + resolved, + ) + ready = operation_status(stack, operation_id, "operator-parent-ready") + require( + ready.get("state") == "ready_to_retry" + and ready.get("next_action") == "resubmit_same", + "operator reconciliation makes the original logical ID resumable", + ready, + ) + # Keep identical block geometry and bound in the immutable append intent. + recovered = append( + stack, + workbench, + operation_id, + payload, + label="operator-same-id-completion", + max_size=1024 * 1024, + block_size=65536, + ) + require( + recovered.get("status") == "success" + and recovered["publication_operation_id"] != child, + "same logical ID completes through its next child", + recovered, + ) + exact_body(stack, workbench, payload, "operator-exact-one-effect") + final_inventory = inventory(stack, "operator-final-inventory") + require( + all(final_inventory.get(row["key"], [None])[0] == 0 for row in verified), + "operator recovery retains all old zero guards", + final_inventory, + ) + replay = replay_unchanged( + stack, + workbench, + operation_id, + payload, + recovered, + "operator-completed-replay", + max_size=1024 * 1024, + block_size=65536, + ) + return case_result( + safety="PASS", + completion="PASS", + caller_sigkill=True, + same_owner_pid=owner_pid, + quarantine_status=status, + fault=faults[0], + status_observations=observations, + inspected_child=inspected, + verified_provider_keys=verified, + obsolete_verdict=wrong, + sealed_verdict=resolved, + terminal_receipt=recovered, + replay=replay, + ) + finally: + with stack.object_proxy.lock: + stack.object_proxy.reject_matching = None + stack.object_proxy.rule = None + + +def identity_errors_and_status(stack): + workbench, operation_id, payload = ( + "product-intent-status", + "e1" * 16, + b"status-seed\n", + ) + stack.cli("workbench_create", {"id": workbench}) + original = append( + stack, + workbench, + operation_id, + payload, + label="intent-original", + content_type="application/x-events", + ) + require(original.get("status") == "success", "intent seed", original) + before = live_path(stack, workbench, "intent-live-before") + rejects = [] + changes = ( + ("payload", {"payload": b"changed\n"}), + ("path", {"path": "other.log"}), + ("type", {"content_type": "text/plain"}), + ("limit", {"max_size": 32 * 1024 * 1024}), + ("block_size", {"block_size": 65536}), + ("incarnation", {"incarnation": "ff" * 16}), + ) + for name, changed in changes: + options = { + "payload": payload, + "content_type": "application/x-events", + **changed, + } + result = append( + stack, workbench, operation_id, label="intent-mismatch-" + name, **options + ) + details = result.get("details", {}) + expected_cause = ( + "Conflict" if name == "incarnation" else "RequestReplayMismatch" + ) + require( + result.get("status") == "error" + and result.get("code") == "AppendUnresolved" + and details.get("code") == expected_cause + and details.get("operation_id") == operation_id, + "altered intent has exact mismatch cause", + result, + ) + if name == "incarnation": + require( + details.get("conflict") == "WorkspaceIncarnation", + "explicit incarnation is a typed fence conflict", + result, + ) + rejects.append({"changed": name, "result": result}) + require( + live_path(stack, workbench, "intent-live-after-rejects") == before, + "rejections do not mutate live metadata", + ) + unknown = operation_status(stack, "ef" * 16, "status-unknown") + require( + unknown.get("status") == "error" and unknown.get("code") == "NotFound", + "unknown identity has typed status", + unknown, + ) + current = operation_status(stack, operation_id, "status-existing") + require( + current.get("state") == "committed" and current.get("next_action") == "none", + "committed status has actionable state", + current, + ) + require( + canonical_receipt(current["receipt"]) == canonical_receipt(original), + "status returns complete original receipt", + ) + query = { + "operation": "get_operation", + "request": {"operation_id": list(bytes.fromhex(operation_id))}, + } + before_collision = base.raw_rpc(stack, query, "append-family-before-collision") + require( + before_collision.get("status") == "success" + and before_collision["body"]["value"]["kind"] == "artifact_append", + "logical ID resolves to append family", + before_collision, + ) + collision = base.raw_rpc( + stack, + { + "operation": "commit", + "request": { + "operation_id": list(bytes.fromhex(operation_id)), + "workbench": workbench, + "workspace_incarnation_id": list( + bytes.fromhex(original["workspace_incarnation_id"]) + ), + "commit_id": [0xA1] * 32, + "content_digest": "sha256:" + "a2" * 32, + "manifest_digest": "sha256:" + "a3" * 32, + "projection_input_digest": [0xA4] * 32, + "tree_manifest_revision_id": [0xA5] * 16, + "replace": False, + "run_manifest_condition": "create_only", + "expected_head_generation": None, + "parents": [], + "producer": None, + "lineage_projection": [], + }, + }, + "commit-reuses-logical-append-id", + ) + require( + collision.get("status") == "failure" + and collision.get("body", {}).get("code") + in ("conflict", "request_replay_mismatch"), + "another operation family cannot admit the logical append ID", + collision, + ) + after_collision = base.raw_rpc(stack, query, "append-family-after-collision") + require( + after_collision.get("status") == "success" + and after_collision["body"]["value"]["kind"] == "artifact_append" + and after_collision["body"]["value"]["result"] + == before_collision["body"]["value"]["result"], + "cross-family rejection leaves original append queryable with the same durable result", + after_collision, + ) + require( + live_path(stack, workbench, "append-family-live-after-collision") == before, + "cross-family reuse causes no path metadata mutation", + ) + tail = append( + stack, + workbench, + "e2" * 16, + b"tail\n", + label="status-later-writer", + content_type=None, + ) + require(tail.get("status") == "success", "later independent append", tail) + replay = replay_unchanged( + stack, + workbench, + operation_id, + payload, + original, + "status-historical-replay", + content_type="application/x-events", + ) + live = live_path(stack, workbench, "status-current-type") + require( + live["value"]["metadata"]["descriptor"]["content_type"] + == "application/x-events", + "omitted type inherits live type", + live, + ) + # Deleting the live path must not delete the historical operation receipt. + stack.run( + [ + *stack.client_args, + "workspace-path", + "remove", + workbench, + "logs", + "effects.log", + "--expected-generation", + str(tail["generation"]), + "--request-id", + "e3" * 16, + ], + label="status-remove-current", + ) + after_remove = operation_status(stack, operation_id, "status-after-path-remove") + require( + canonical_receipt(after_remove["receipt"]) == canonical_receipt(original), + "status survives current path removal", + ) + removed_replay = append( + stack, + workbench, + operation_id, + payload, + label="status-removed-replay", + content_type="application/x-events", + ) + require( + canonical_receipt(removed_replay) == canonical_receipt(original), + "append replay after deletion returns old receipt", + ) + exact_body(stack, workbench, None, "status-path-stays-absent") + return case_result( + safety="PASS", + completion="PASS", + mismatches=rejects, + unknown_status=unknown, + original_status=current, + cross_family_rejection=collision, + later_writer=tail, + replay=replay, + status_after_remove=after_remove, + ) + + +def repeated_identity_retention(stack, repetitions): + workbench, operation_id, payload = ( + "product-replay-retention", + "e4" * 16, + b"retained-event\n", + ) + stack.cli("workbench_create", {"id": workbench}) + original = append( + stack, workbench, operation_id, payload, label="retention-original" + ) + require(original.get("status") == "success", "retention original", original) + replay = replay_unchanged( + stack, + workbench, + operation_id, + payload, + original, + "retention-replay", + repetitions=repetitions, + ) + stack.restart() + reopened = replay_unchanged( + stack, + workbench, + operation_id, + payload, + original, + "retention-after-reopen", + repetitions=2, + ) + exact_body(stack, workbench, payload, "retention-body") + return case_result( + safety="PASS", + completion="PASS", + observed_replays=repetitions + 2, + owner_reopen=True, + replay=replay, + reopened=reopened, + retention_horizon="Only this run's elapsed time is observed; advertised long-term retention and high-cardinality storage growth remain NOT QUALIFIED.", + ) + + +PYTHON_STATUS_PROGRAM = r""" +import hashlib, json, sys +from pathlib import Path +import nokv +from nokv import _native +config = json.load(sys.stdin) +client = nokv.Client(config["root_id"], nokv.RoutingConfig.etcd([config["etcd_endpoint"]], key_prefix=config["etcd_prefix"]), object_store=None, workbench_root=config["workbench_root"]) +try: + result = client.operation_status(config["operation_id"]) + print(json.dumps({"result": result, "native_sha256": hashlib.sha256(Path(_native.__file__).read_bytes()).hexdigest(), "python_version": sys.version, "object_store": None})) +except Exception as error: + print(json.dumps({"error": type(error).__name__, "message": str(error)})) +""" + +PYTHON_APPEND_PROGRAM = r""" +import hashlib, json, sys +from pathlib import Path +import nokv +from nokv import _native +config = json.load(sys.stdin) +objects = None if config["metadata_only"] else nokv.ObjectStoreConfig.s3(config["bucket"], root=config["object_root"], endpoint=config["object_endpoint"], access_key_id=config["access_key"], secret_access_key=config["secret_key"]) +client = nokv.Client(config["root_id"], nokv.RoutingConfig.etcd([config["etcd_endpoint"]], key_prefix=config["etcd_prefix"]), object_store=objects, workbench_root=config["workbench_root"]) +try: + result = client.append_bytes(config["workbench"], "logs/effects.log", bytes.fromhex(config["payload_hex"]), config["operation_id"], content_type=config["content_type"], block_size=config["block_size"], max_logical_size=config["max_logical_size"]) + print(json.dumps({"status":"success", "result":result, "object_store_configured":objects is not None, "native_sha256":hashlib.sha256(Path(_native.__file__).read_bytes()).hexdigest()})) +except Exception as error: + print(json.dumps({"status":"error", "type":type(error).__name__, "message":str(error), **{field:getattr(error, field, None) for field in ("operation_id", "code", "cause_code", "next_action", "publication_operation_id", "state", "retryable")}})) +""" + +PYTHON_ADMISSION_RECOVERY_PROGRAM = r""" +import json, os, sys, time +from pathlib import Path +import nokv +config = json.load(sys.stdin) +control = Path(config["control_directory"]) +client = nokv.Client(config["root_id"], nokv.RoutingConfig.etcd([config["etcd_endpoint"]], key_prefix=config["etcd_prefix"]), nokv.ObjectStoreConfig.s3(config["bucket"], root=config["object_root"], endpoint=config["object_endpoint"], access_key_id=config["access_key"], secret_access_key=config["secret_key"]), workbench_root=config["workbench_root"]) + +def emit(name, value): + value.update(client_id=id(client), pid=os.getpid()) + (control / (name + ".json")).write_text(json.dumps(value)) + return value + +def wait(name): + deadline = time.monotonic() + 100 + while not (control / name).exists(): + if time.monotonic() >= deadline: + raise RuntimeError("test coordinator did not release " + name) + time.sleep(.05) + +def append_once(): + try: + return {"status":"success", "result":client.append_bytes(config["workbench"], "logs/effects.log", b"admission-recovered\n", config["operation_id"], content_type="text/plain", max_logical_size=1048576)} + except Exception as error: + return {"status":"error", "type":type(error).__name__, "message":str(error), **{field:getattr(error,field,None) for field in ("operation_id","code","cause_code","next_action","state")}} + +# Generic publication binds and caches this exact Client's object handle before +# the append-specific admission fault is armed by the parent coordinator. +seed = client.publish_bytes(config["workbench"], "logs/effects.log", b"python-bound-seed\n", content_type="text/plain") +ready = emit("ready", {"status":"success", "seed":seed}) +wait("first-go") +first = emit("first", append_once()) +wait("second-go") +second = emit("second", append_once()) +print(json.dumps({"ready":ready,"first":first,"second":second})) +""" + + +def wait_json_file(path, timeout): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + return json.loads(path.read_text()) + except (FileNotFoundError, json.JSONDecodeError): + time.sleep(0.05) + raise TimeoutError(f"Python coordinator did not receive {path}") + + +def python_same_client_admission_recovery(stack): + workbench, operation_id = "product-python-admission-recovery", "eb" * 16 + stack.cli("workbench_create", {"id": workbench}) + control = stack.evidence_dir / "python-same-client-control" + control.mkdir() + config = { + "root_id": stack.root_id, + "etcd_endpoint": stack.etcd_endpoint, + "etcd_prefix": stack.etcd_prefix, + "workbench_root": stack.config.workbench_root, + "bucket": stack.bucket, + "object_root": stack.object_root, + "object_endpoint": stack.object_endpoint, + "access_key": stack.access_key, + "secret_key": stack.secret_key, + "workbench": workbench, + "operation_id": operation_id, + "control_directory": str(control), + } + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit( + stack.run, + [stack.python_executable, "-c", PYTHON_ADMISSION_RECOVERY_PROGRAM], + label="python-one-client-admission-recovery", + input_text=json.dumps(config), + timeout=2 * stack.timeout, + ) + try: + ready = wait_json_file(control / "ready.json", stack.timeout) + exact_body( + stack, + workbench, + b"python-bound-seed\n", + "python-admission-seed-before-fault", + ) + with stack.object_proxy.lock: + prior_faults = len(stack.object_proxy.rejected_requests) + stack.object_proxy.reject_matching = { + "method": "PUT", + "path_contains": "/seal-", + } + (control / "first-go").touch() + first = wait_json_file(control / "first.json", stack.timeout) + faults = stack.object_proxy.rejected_requests[prior_faults:] + if not faults: + raise base.infra.NotQualified( + "Append-specific seal admission was not reached by this installed Python Client; no outage was injected." + ) + require( + first.get("status") == "error" + and first.get("operation_id") == operation_id + and first.get("code") == "AppendFailed" + and first.get("cause_code") == "ProviderAdmissionInconclusive" + and first.get("next_action") == "query_same" + and first.get("state") is None, + "temporary seal admission failure is typed and keeps logical identity", + first, + ) + absent = operation_status( + stack, operation_id, "python-admission-failure-unadmitted" + ) + require( + absent.get("code") == "NotFound", + "provider admission failure creates no logical operation", + absent, + ) + exact_body( + stack, + workbench, + b"python-bound-seed\n", + "python-admission-failure-no-effect", + ) + with stack.object_proxy.lock: + stack.object_proxy.reject_matching = None + (control / "second-go").touch() + second = wait_json_file(control / "second.json", stack.timeout) + require( + second.get("status") == "success", + "same live Python Client recovers after temporary seal-admission failure", + second, + ) + require( + ready["client_id"] == first["client_id"] == second["client_id"] + and ready["pid"] == first["pid"] == second["pid"], + "both attempts use exactly one Client instance in one Python process", + ) + process = future.result(timeout=stack.timeout) + finally: + with stack.object_proxy.lock: + stack.object_proxy.reject_matching = None + (control / "first-go").touch() + (control / "second-go").touch() + exact_body( + stack, + workbench, + b"python-bound-seed\nadmission-recovered\n", + "python-admission-recovered-body", + ) + replay = replay_unchanged( + stack, + workbench, + operation_id, + b"admission-recovered\n", + second["result"], + "python-admission-cli-replay", + max_size=1024 * 1024, + ) + return { + "same_client_process": json.loads(process.stdout), + "injected_503_requests": faults, + "unadmitted_status_after_failure": absent, + "replay": replay, + } + + +def python_append( + stack, + workbench, + operation_id, + payload, + *, + label, + block_size=4 * 1024 * 1024, + metadata_only=False, +): + config = { + "root_id": stack.root_id, + "etcd_endpoint": stack.etcd_endpoint, + "etcd_prefix": stack.etcd_prefix, + "workbench_root": stack.config.workbench_root, + "bucket": stack.bucket, + "object_root": stack.object_root, + "object_endpoint": stack.object_endpoint, + "access_key": stack.access_key, + "secret_key": stack.secret_key, + "metadata_only": metadata_only, + "workbench": workbench, + "operation_id": operation_id, + "payload_hex": payload.hex(), + "content_type": "text/plain", + "block_size": block_size, + "max_logical_size": 1024 * 1024, + } + process = stack.run( + [stack.python_executable, "-c", PYTHON_APPEND_PROGRAM], + label=label, + input_text=json.dumps(config), + ) + result = json.loads(process.stdout) + stack.record( + "product-python-append.jsonl", + { + "at": base.live.now(), + "label": label, + "operation_id": operation_id, + "block_size": block_size, + "metadata_only": metadata_only, + "payload_sha256": hashlib.sha256(payload).hexdigest(), + "result": result, + }, + ) + return result + + +def python_metadata_status(stack): + if stack.python_executable is None: + return case_result( + reason="No isolated installed Python SDK executable supplied." + ) + workbench, operation_id = "product-python-status", "e5" * 16 + stack.cli("workbench_create", {"id": workbench}) + original = append( + stack, + workbench, + operation_id, + "python-status-event\n", + label="python-status-cli-original", + max_size=1024 * 1024, + ) + require(original.get("status") == "success", "Python status original", original) + replay = python_append( + stack, + workbench, + operation_id, + b"python-status-event\n", + label="python-append-cli-replay", + ) + require( + replay.get("status") == "success" + and canonical_receipt(replay["result"]) == canonical_receipt(original), + "CLI to installed Python original receipt", + replay, + ) + reverse_id, reverse_payload = "e9" * 16, b"python-64k-event\n" + python_original = python_append( + stack, + workbench, + reverse_id, + reverse_payload, + label="python-fresh-nondefault-block", + block_size=65536, + ) + require( + python_original.get("status") == "success" + and python_original["result"].get("replayed") is False, + "Python fresh publication with nondefault block size", + python_original, + ) + reverse_replay = replay_unchanged( + stack, + workbench, + reverse_id, + reverse_payload, + python_original["result"], + "cli-python-nondefault-replay", + repetitions=2, + block_size=65536, + max_size=1024 * 1024, + ) + exact_body( + stack, + workbench, + b"python-status-event\npython-64k-event\n", + "python-interoperability-complete-bytes", + ) + live_before = live_path(stack, workbench, "python-metadata-replay-live-before") + metadata_only_replay = python_append( + stack, + workbench, + reverse_id, + reverse_payload, + label="python-no-object-store-replay", + block_size=65536, + metadata_only=True, + ) + require( + metadata_only_replay.get("status") == "success" + and metadata_only_replay.get("object_store_configured") is False + and canonical_receipt(metadata_only_replay["result"]) + == canonical_receipt(python_original["result"]), + "Python append replay works without configured object store", + metadata_only_replay, + ) + require( + live_path(stack, workbench, "python-metadata-replay-live-after") == live_before, + "metadata-only Python append replay does not change live head", + ) + (stack.evidence_dir / "python-status-cli-original.input").unlink() + config = { + "root_id": stack.root_id, + "etcd_endpoint": stack.etcd_endpoint, + "etcd_prefix": stack.etcd_prefix, + "workbench_root": stack.config.workbench_root, + "operation_id": operation_id, + } + before = (stack.evidence_dir / "object-http-transcript.jsonl").read_text() + process = stack.run( + [stack.python_executable, "-c", PYTHON_STATUS_PROGRAM], + label="python-metadata-only-status", + input_text=json.dumps(config), + ) + result = json.loads(process.stdout) + require( + "error" not in result + and canonical_receipt(result["result"]["receipt"]) + == canonical_receipt(original), + "Python status requires no object store or payload", + result, + ) + after = (stack.evidence_dir / "object-http-transcript.jsonl").read_text() + require(after == before, "status performs no S3 I/O") + mismatch = python_append( + stack, + workbench, + operation_id, + b"different\n", + label="python-intent-mismatch", + metadata_only=True, + ) + require( + mismatch.get("type") == "AppendError" + and mismatch.get("cause_code") == "RequestReplayMismatch" + and mismatch.get("operation_id") == operation_id, + "Python mismatch exposes stable cause and identity", + mismatch, + ) + admission_recovery = python_same_client_admission_recovery(stack) + return case_result( + safety="PASS", + completion="PASS", + installed_python={ + "executable": str(stack.python_executable), + "native_sha256": replay["native_sha256"], + }, + cli_to_python=replay, + python_original=python_original, + python_to_cli=reverse_replay, + metadata_only_append_replay=metadata_only_replay, + metadata_only_status=result, + mismatch=mismatch, + same_client_admission_recovery=admission_recovery, + ) + + +def legacy_wire_rejected(stack, legacy_binary): + if legacy_binary is None: + return case_result( + reason="No frozen incompatible legacy binary supplied for real wire rejection." + ) + workbench, operation_id = "product-legacy-wire", "e6" * 16 + stack.cli("workbench_create", {"id": workbench}) + seed = append( + stack, workbench, "e7" * 16, "current-format\n", label="legacy-wire-seed" + ) + require(seed.get("status") == "success", "current server fixture", seed) + before = live_path(stack, workbench, "legacy-wire-live-before") + objects = inventory(stack, "legacy-wire-inventory-before") + config = dataclasses.replace(stack.config, binary=legacy_binary) + command = [ + *base.live.client_args(config), + "workspace-path", + "append", + workbench, + "logs", + "effects.log", + "--operation-id", + operation_id, + "--text", + "must-not-write\n", + ] + result = stack.run(command, label="real-legacy-append-rejection", check=False) + require( + result.returncode != 0, "incompatible legacy client is rejected", result.stdout + ) + error = result.stderr.lower() + require( + any( + term in error + for term in ("schema", "handshake", "capability", "wire version") + ), + "legacy rejection identifies incompatible protocol instead of unrelated CLI error", + result.stderr, + ) + require( + "unknown command" not in error and "unrecognized" not in error, + "not a parser-only legacy rejection", + result.stderr, + ) + # The proxy intentionally captured the rejected old client's handshake. + # Refresh the supplementary probe profile using the current native surface. + stack.cli( + "workbench_stat", + {"id": workbench, "section": "logs", "path": "effects.log"}, + label="current-client-after-legacy-rejection", + ) + require( + live_path(stack, workbench, "legacy-wire-live-after") == before, + "old client causes no path/revision/generation mutation", + ) + require( + inventory(stack, "legacy-wire-inventory-after") == objects, + "old client creates no payload object", + ) + status = operation_status(stack, operation_id, "legacy-wire-operation-absent") + require( + status.get("code") == "NotFound", + "old client admitted no logical operation", + status, + ) + return case_result( + safety="PASS", + completion="PASS", + legacy_binary=str(legacy_binary), + legacy_binary_sha256=base.live.digest_file(legacy_binary), + rejection=result.stderr, + operation_status=status, + qualification="Old client against current owner only; actual old Holt-format reopen is a separate required scope.", + ) + + +def directory_manifest(directory): + return { + str(path.relative_to(directory)): base.live.digest_file(path) + for path in sorted(directory.rglob("*")) + if path.is_file() + } + + +def control_records(stack, label): + result = stack.run( + [ + stack.etcdctl, + "--endpoints=" + stack.etcd_endpoint, + "get", + stack.etcd_prefix, + "--prefix", + "--write-out=json", + ], + label=label, + ) + return json.loads(result.stdout).get("kvs", []) + + +def legacy_format_rejected(stack, legacy_binary): + if legacy_binary is None: + return case_result( + reason="No actual incompatible old binary supplied to create the durable fixture." + ) + evidence_dir = stack.evidence_dir / "legacy-format" + with base.IsolatedStack( + evidence_dir, + binary=legacy_binary, + source=stack.source, + label="old-format", + timeout=stack.timeout, + ) as old: + old.proxy.capture_rpc = True + workbench = "old-format-fixture" + old.cli("workbench_create", {"id": workbench}) + first = base.native_append( + old, workbench, "f1" * 16, "old-format-data\n", label="old-format-original" + ) + require( + first.get("status") == "success", "actual old binary wrote fixture", first + ) + old.owner.kill() + old.owner.wait(timeout=10) + base.infra.wait_session_absent( + old.etcdctl, + old.etcd_endpoint, + f"{old.etcd_prefix}/sessions/{old.shard_id}", + old.source, + old.timeout, + ) + cloned = evidence_dir / "old-format-reopen-copy" + shutil.copytree(old.config.metadata, cloned) + before = directory_manifest(cloned) + control_before = control_records(old, "old-format-control-before") + objects_before = inventory(old, "old-format-objects-before") + config = dataclasses.replace( + old.config, binary=stack.binary, metadata=cloned, metadata_mode="reopen" + ) + process = old.start("new-reader-old-format", base.live.server_command(config)) + try: + code = process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + raise ContractViolation( + "Current owner did not fail closed when reopening actual old format" + ) + log = (evidence_dir / "new-reader-old-format.log").read_text() + require( + code != 0 + and any( + term in log.lower() + for term in ("schema", "registry", "incompatible", "version") + ), + "actual old store rejected for format incompatibility", + {"exit_code": code, "log": log}, + ) + after = directory_manifest(cloned) + manifest = { + "before": before, + "after": after, + "exact_files_unchanged": before == after, + } + old.json("old-format-files.json", manifest) + control_unchanged = ( + control_records(old, "old-format-control-after") == control_before + ) + payload_unchanged = inventory(old, "old-format-objects-after") == objects_before + old.restart() + require( + base.materialize( + old, workbench, "logs", "effects.log", "old-format-original-reopened" + ) + == b"old-format-data\n", + "original old store remains independently readable", + ) + safe = before == after and control_unchanged and payload_unchanged + return case_result( + safety="PASS" if safe else "FAIL", + completion="PASS", + legacy_binary=str(legacy_binary), + legacy_sha256=base.live.digest_file(legacy_binary), + original_receipt=first, + rejected_exit_code=code, + rejected_log=log, + cloned_store_all_file_hashes_unchanged=before == after, + control_unchanged=control_unchanged, + payload_unchanged=payload_unchanged, + original_old_store_reopened=True, + **( + { + "failure": "Actual incompatible-format rejection modified durable files or control/object state; the independent original store readback does not satisfy the no-write requirement." + } + if not safe + else {} + ), + ) + + +def exact_body(stack, workbench, expected, label): + stat = stack.cli( + "workbench_stat", + {"id": workbench, "section": "logs", "path": "effects.log"}, + label=label + "-stat", + check=False, + ) + if stat.get("status") == "success": + body = base.materialize(stack, workbench, "logs", "effects.log", label) + else: + require(stat.get("code") == "NotFound", "unexpected stat failure", stat) + body = None + require( + body == expected, "exact visible bytes", {"actual": body, "expected": expected} + ) + return stat + + +def recover_same_identity( + stack, workbench, operation_id, text, *, label, deadline, observed_status=None +): + outcomes = [] + statuses = [] + query_first = observed_status is not None + until = time.monotonic() + deadline + while True: + if query_first: + status = observed_status or operation_status( + stack, operation_id, f"{label}-status-{len(statuses):03d}" + ) + observed_status = None + statuses.append(status) + stack.json(label + "-status-observations.json", statuses) + if status.get("state") == "committed": + require( + status.get("receipt") is not None + and status.get("next_action") == "none" + and status["receipt"].get("operation_id") == operation_id, + "committed status has the caller's durable receipt and no recovery action", + status, + ) + return { + **status["receipt"], + "status": "success", + "replayed": True, + "commit_version": status.get("commit_version"), + }, outcomes + if status.get("state") == "quarantined": + return None, outcomes + if status.get("next_action") == "poll" or ( + status.get("status") == "error" and status.get("code") != "NotFound" + ): + if time.monotonic() >= until: + return None, outcomes + time.sleep(min(0.5, max(0, until - time.monotonic()))) + continue + require( + status.get("next_action") == "resubmit_same" + or status.get("code") == "NotFound", + "status gives explicit same-identity recovery instruction", + status, + ) + result = append( + stack, + workbench, + operation_id, + text, + label=f"{label}-{len(outcomes):03d}", + max_size=1024 * 1024, + ) + outcomes.append(result) + stack.json(label + "-outcomes.json", outcomes) + if result.get("status") == "success": + return result, outcomes + require( + result.get("status") == "error" + and result.get("details", {}).get("operation_id") == operation_id, + "failure retains the caller identity", + result, + ) + query_first = result.get("details", {}).get("next_action") == "query_same" + if time.monotonic() >= until: + return None, outcomes + time.sleep(min(0.5, max(0, until - time.monotonic()))) + + +def owner_begin_completion(stack, deadline): + """Committed Begin, lost response, real owner SIGKILL, same-ID completion.""" + workbench = "product-owner-begin-completion" + operation_id, text = "b1" * 16, "one-recoverable-event\n" + stack.cli("workbench_create", {"id": workbench}) + stack.proxy.arm("begin_artifact_publish", lambda: stack.owner.kill()) + first = base.native_append( + stack, + workbench, + operation_id, + text, + label="owner-begin-first", + check=False, + ) + stack.proxy.wait_dropped() + dropped = stack.proxy.last_drop + require(dropped is not None, "successful Begin fault must actually fire") + require(first.get("status") == "error", "first caller observed no success", first) + stack.owner.wait(timeout=10) + require( + stack.owner.returncode == -9, + "owner exited from SIGKILL", + stack.owner.returncode, + ) + stack.restart() + exact_body(stack, workbench, None, "owner-begin-before-retry") + recovered, outcomes = recover_same_identity( + stack, + workbench, + operation_id, + text, + label="owner-begin-same-identity", + deadline=deadline, + ) + expected = text.encode() if recovered is not None else None + visible = exact_body(stack, workbench, expected, "owner-begin-final") + result = case_result( + safety="PASS", + completion="PASS" if recovered is not None else "FAIL", + operation_id=operation_id, + first_call=first, + retry_outcomes=outcomes, + recovered_receipt=recovered, + final_stat=visible, + fault={ + "operation": dropped["operation"], + "owner_pid": dropped["owner_pid"], + "successful_owner_response_observed": True, + "delivered_to_caller": False, + }, + oracle="An admitted logical append completes after owner recovery when the same ID and payload are supplied; no second logical effect.", + ) + if recovered is None: + result["failure"] = ( + "Same identity remains terminally Failed after admitted Begin and owner recovery; zero effects cannot satisfy completion." + ) + return result + + +def main(): + native_cases = { + "owner_begin_completion": lambda stack, args: owner_begin_completion( + stack, args.completion_timeout_seconds + ), + **{ + "caller_" + operation: ( + lambda stack, args, operation=operation: caller_boundary_completion( + stack, args.completion_timeout_seconds, operation + ) + ) + for operation in ( + "begin_artifact_publish", + "stage_artifact_objects", + "mark_artifact_objects_uploaded", + "stage_artifact_manifest", + "complete_artifact_publish", + ) + }, + "deterministic_same_identity": lambda stack, args: deterministic_same_identity( + stack, args.completion_timeout_seconds + ), + "distinct_identity_exact_bytes": lambda stack, + args: distinct_identity_exact_bytes(stack, args.completion_timeout_seconds), + "object_put_response_loss": lambda stack, args: object_put_response_loss( + stack, args.completion_timeout_seconds + ), + "object_read_integrity": lambda stack, args: object_read_integrity( + stack, args.completion_timeout_seconds + ), + "cross_root_identity": lambda stack, args: cross_root_identity(stack), + "boundary_and_capacity": lambda stack, args: boundary_and_capacity(stack), + "publication_batch_boundaries": lambda stack, + args: publication_batch_boundaries(stack), + "identity_errors_and_status": lambda stack, args: identity_errors_and_status( + stack + ), + "repeated_identity_retention": lambda stack, args: repeated_identity_retention( + stack, args.replay_count + ), + "completed_replay_provider_unavailable": lambda stack, + args: completed_replay_provider_unavailable(stack), + "python_metadata_status": lambda stack, args: python_metadata_status(stack), + "legacy_wire_rejected": lambda stack, args: legacy_wire_rejected( + stack, args.legacy_nokv_bin + ), + "legacy_format_rejected": lambda stack, args: legacy_format_rejected( + stack, args.legacy_nokv_bin + ), + "late_put_after_cleanup": lambda stack, args: late_put_after_cleanup( + stack, args.completion_timeout_seconds + ), + "seal_quarantine_operator_recovery": lambda stack, + args: seal_quarantine_operator_recovery(stack, args.completion_timeout_seconds), + } + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--nokv-bin", type=Path, required=True) + parser.add_argument("--source-dir", type=Path, required=True) + parser.add_argument("--evidence-dir", type=Path, required=True) + parser.add_argument("--completion-timeout-seconds", type=float, default=60) + parser.add_argument("--timeout-seconds", type=int, default=60) + parser.add_argument( + "--append-activity-lease-ms", + type=int, + help="The production server's supported activity lease; omit for its default.", + ) + parser.add_argument("--python-executable", type=Path) + parser.add_argument("--legacy-nokv-bin", type=Path) + parser.add_argument("--replay-count", type=int, default=100) + parser.add_argument("--scenario", choices=("all", *native_cases), default="all") + parser.add_argument( + "--scenarios", + nargs="+", + choices=tuple(native_cases), + help="Run an explicit ordered subset; it never qualifies the full matrix.", + ) + args = parser.parse_args() + results = { + "schema": "nokv.append_product_acceptance.v1", + "status": "NOT QUALIFIED", + "started_at": base.live.now(), + "command": sys.argv, + "scenarios": {}, + "qualification_scope": "Bounded append on one local Holt owner and real RustFS; this is not whole-workspace or cross-host commercial qualification.", + "safety": "NOT QUALIFIED", + "completion": "NOT QUALIFIED", + "performance": "NOT QUALIFIED", + "workspace_release": "NOT QUALIFIED", + "unqualified_release_scopes": { + "whole_workspace_gates_0_through_8": "NOT QUALIFIED", + "owner_internal_durable_commit_boundaries": "NOT QUALIFIED", + "long_term_retention_and_high_cardinality_gc": "NOT QUALIFIED", + "real_demo_harness_queue_redelivery": "NOT QUALIFIED", + "workload_matched_performance_slo": "NOT QUALIFIED", + "cross_host_shared_recovery": "NOT QUALIFIED", + }, + "harness_path": str(Path(__file__).resolve()), + "harness_sha256": base.live.digest_file(Path(__file__)), + "python_runtime": {"version": sys.version, "optimize": sys.flags.optimize}, + "os": platform.platform(), + } + stack = None + exit_code = 2 + try: + # Base stack contains assertions too; refusing optimization prevents a + # disabled helper oracle from contaminating qualification. + require( + sys.flags.optimize == 0, "qualification requires Python assertions enabled" + ) + stack = ProductStack( + args.evidence_dir, + binary=args.nokv_bin, + source=args.source_dir, + label="append-product", + timeout=args.timeout_seconds, + publication_lease_ms=args.append_activity_lease_ms, + ) + stack.python_executable = args.python_executable + shutil.copyfile(Path(__file__), args.evidence_dir / "executed-harness.py") + with stack: + stack.proxy.capture_rpc = True + results["machine_profile"] = record_machine_profile(stack) + require( + not args.scenarios or args.scenario == "all", + "choose --scenario or --scenarios, not both", + ) + cases = ( + [(name, native_cases[name]) for name in args.scenarios] + if args.scenarios + else native_cases.items() + if args.scenario == "all" + else [(args.scenario, native_cases[args.scenario])] + ) + for name, case in cases: + print("START " + name, flush=True) + started = time.monotonic() + try: + result = case(stack, args) + except Exception as error: + result = case_result( + safety="FAIL" + if isinstance(error, ContractViolation) + else "NOT QUALIFIED", + completion="NOT QUALIFIED", + failure=repr(error), + traceback=traceback.format_exc(), + ) + result["elapsed_seconds"] = time.monotonic() - started + results["scenarios"][name] = result + stack.json("product-results.json", results) + print( + json.dumps( + { + "scenario": name, + "safety": result["safety"], + "completion": result["completion"], + } + ), + flush=True, + ) + if "FAIL" in (result["safety"], result["completion"]) or result.get( + "failure" + ): + break + for dimension in ("safety", "completion"): + statuses = [value[dimension] for value in results["scenarios"].values()] + results[dimension] = ( + "FAIL" + if "FAIL" in statuses + else "PASS" + if statuses and all(value == "PASS" for value in statuses) + else "NOT QUALIFIED" + ) + results["all_listed_scenarios_executed"] = set(results["scenarios"]) == set( + native_cases + ) + results["status"] = ( + "FAIL" + if "FAIL" in (results["safety"], results["completion"]) + else "PASS" + if results["safety"] == results["completion"] == "PASS" + else "NOT QUALIFIED" + ) + results["qualified_scopes"] = [ + name + for name, value in results["scenarios"].items() + if value["safety"] == value["completion"] == "PASS" + ] + if "legacy_format_rejected" not in results["qualified_scopes"]: + results["unqualified_release_scopes"][ + "old_holt_format_reopen_no_write" + ] = "NOT QUALIFIED" + exit_code = {"FAIL": 1, "NOT QUALIFIED": 2, "PASS": 0}[results["status"]] + except base.infra.NotQualified as error: + results["not_qualified_reason"] = str(error) + except Exception as error: + results["status"] = ( + "FAIL" if isinstance(error, ContractViolation) else "NOT QUALIFIED" + ) + results["error"] = repr(error) + results["traceback"] = traceback.format_exc() + exit_code = 1 if isinstance(error, ContractViolation) else 2 + finally: + results["completed_at"] = base.live.now() + args.evidence_dir.mkdir(parents=True, exist_ok=True) + if stack is not None: + cleanup_path = args.evidence_dir / "cleanup.json" + if cleanup_path.is_file(): + cleanup = json.loads(cleanup_path.read_text()) + if any( + not isinstance(value, dict) or value.get("exit_code") != 0 + for value in cleanup["results"] + ): + results["cleanup_failure"] = cleanup + if results["status"] == "PASS": + results["status"], exit_code = "NOT QUALIFIED", 2 + stack.json("product-results.json", results) + else: + (args.evidence_dir / "product-results.json").write_text( + json.dumps(results, indent=2) + "\n" + ) + print( + json.dumps( + { + "status": results["status"], + "safety": results["safety"], + "completion": results["completion"], + "evidence": str(args.evidence_dir), + } + ), + flush=True, + ) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) From e5f81daf68075b3ae71bbf2ad0995fd036039dcd Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 23 Sep 2026 02:22:33 +1000 Subject: [PATCH 3/5] test: bound append acceptance reads by expected body size Signed-off-by: wchwawa --- .../append_product_acceptance_gate.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/workbench/append_product_acceptance_gate.py b/scripts/workbench/append_product_acceptance_gate.py index 190a67948..a6cff1931 100644 --- a/scripts/workbench/append_product_acceptance_gate.py +++ b/scripts/workbench/append_product_acceptance_gate.py @@ -2885,7 +2885,24 @@ def exact_body(stack, workbench, expected, label): check=False, ) if stat.get("status") == "success": - body = base.materialize(stack, workbench, "logs", "effects.log", label) + require(expected is not None, "unexpected visible artifact", stat) + target = stack.evidence_dir / (label + ".data") + # The read contract has its own bound; an append case that explicitly + # admits a larger body must read exactly that expected size as well. + stack.run( + [ + *stack.client_args, + "--max-artifact-bytes", + str(max(1, len(expected))), + "materialize", + workbench, + "logs", + "effects.log", + str(target), + ], + label=label, + ) + body = target.read_bytes() else: require(stat.get("code") == "NotFound", "unexpected stat failure", stat) body = None From 920c58f86537ce5d9692a9e0e90081c5f985d51b Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 23 Sep 2026 03:27:21 +1000 Subject: [PATCH 4/5] feat: expose append inspection and fenced owner recovery Signed-off-by: wchwawa --- README.md | 42 +- crates/nokv-client/src/append_operations.rs | 309 ++++ .../src/append_operations_tests.rs | 367 +++++ crates/nokv-client/src/artifact.rs | 3 +- crates/nokv-client/src/error.rs | 25 +- crates/nokv-client/src/idempotent_append.rs | 7 +- crates/nokv-client/src/lib.rs | 5 + crates/nokv-meta/src/workspace/codec.rs | 6 +- crates/nokv-meta/src/workspace/mod.rs | 24 +- crates/nokv-meta/src/workspace/publication.rs | 756 ++++++++- .../workspace/publish_operation_records.rs | 106 +- crates/nokv-meta/src/workspace/restore.rs | 1 + crates/nokv-protocol/src/append.rs | 195 ++- crates/nokv-protocol/src/artifact_publish.rs | 2 +- crates/nokv-protocol/src/codec.rs | 90 +- crates/nokv-protocol/src/lib.rs | 5 +- crates/nokv-protocol/src/request.rs | 6 +- crates/nokv-protocol/src/response.rs | 6 +- crates/nokv-protocol/src/types.rs | 4 +- crates/nokv-python/README.md | 27 +- crates/nokv-python/python/nokv/__init__.py | 12 +- crates/nokv-python/src/client.rs | 287 +++- crates/nokv-python/src/python_value.rs | 37 +- crates/nokv-python/tests/test_api_surface.py | 32 + crates/nokv-server/src/executor.rs | 492 +++++- crates/nokv-server/src/lifecycle.rs | 41 +- crates/nokv/src/backend.rs | 101 +- crates/nokv/src/cli.rs | 211 ++- crates/nokv/src/main.rs | 221 ++- docs/architecture.md | 12 +- docs/development/append-product-spec.md | 137 +- docs/metadata-schema.md | 37 +- .../append_operations_acceptance_gate.py | 1374 +++++++++++++++++ .../append_product_acceptance_gate.py | 310 +--- 34 files changed, 4729 insertions(+), 561 deletions(-) create mode 100644 crates/nokv-client/src/append_operations.rs create mode 100644 crates/nokv-client/src/append_operations_tests.rs create mode 100644 scripts/workbench/append_operations_acceptance_gate.py diff --git a/README.md b/README.md index 45d172774..6d881e902 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,8 @@ real-service acceptance gate is still **not qualified**. | `nokv workspace-path rename` / `nokv workspace-path remove` | Apply an explicit generation- and request-id-fenced path mutation. | Custom CLI surface; not one of the 18 Workbench tools. | | `nokv workspace-path append` | Append bytes under a durable logical identity and recover the original receipt across process restarts. | The first admission requires an existing workspace. Retries use fenced publication attempts under the same identity; the 18-tool schema is unchanged. | | `nokv operation status` | Query an append by its logical identity and report its receipt or next recovery action. | Metadata only: no payload, current workspace, or object-store connection is required. | +| `nokv operation inspect` | Inspect a bounded page of retained staged objects and save the exact operation token. | Metadata only; the cursor binds one observation and fails if state changes. | +| `nokv operation recover` | Ask the owner to retry quarantined append cleanup. | No payload or object credentials; an explicit saved state digest replays one durable recovery request. | | `nokv provision` | Bind `RootId` to `AgentId`, object namespace, logical shard, and persisted placement through etcd. | `AgentId` prevents accidental root reuse; it is not authentication. | | `nokv serve` | Start one explicit metadata owner from create, same-namespace reopen, or recovery-log state. | Shared recovery publication is opt-in and not currently qualified. | | `nokv mcp` | Deprecated stdio transport retained only because qualification runners still use it. | **Unsupported for integration.** | @@ -289,12 +291,42 @@ The response supplies a machine-readable recovery action: | `committed` | `none` | Consume the original `receipt`. | | `pending` | `poll` | Query the same logical identity again after waiting. | | `ready_to_retry` | `resubmit_same` | Resubmit the original append inputs under the same logical identity. | -| `quarantined` | `operator_reconcile` | Keep the identity and escalate the recorded inconsistency for operator reconciliation. | +| `quarantined` | `retry_cleanup` | Inspect the failure, restore provider availability, then ask the owner to retry cleanup. | Status also retains the current physical attempt's `cause_code`, `failure_message`, and structured `attempt_failure`, including when the logical operation is `ready_to_retry`. +Inspect and recover without object credentials: + +```shell +nokv [route/agent options] operation inspect aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa --limit 32 +nokv [route/agent options] operation recover aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + --expected-state-digest +``` + +Inspection returns `operation_token` and `publication_token`, each with +`operation_id` and `state_digest`, plus the object namespace, `registered_count`, +`cleanup_cursor`, `remaining_count`, and `entries`. Entries describe retained +staged objects, not a historical inventory of retired keys. Each entry includes +its sequence, identity, expected length, digest, and optional multipart token. +The page limit defaults to 32 and may range from 1 to 192. Continue with the +opaque base64 `next_cursor` using `--cursor`; a stale cursor returns +`Conflict/OperationState`, requiring a fresh inspection instead of mixing pages. + +Save the logical token before requesting recovery. Reuse its state digest after +a lost reply or process restart. The owner performs sealing; this command never +resends the append delta. `requested=true` means a cleanup request was durably +accepted, including a replay. Its historical `recovery_receipt` contains the +logical id, publication id, cleanup retry count, and expected state digest. +The top-level status is a fresh observation and may be newer than that receipt; +acceptance is not cleanup completion. Omitting the digest targets the current +state once and is a no-op for pending, cleaned, or committed operations. A later +quarantine needs an intentional new request with a newly inspected token. +`AppendCleanupUnresolved` preserves the saved digest, any known recovery receipt, +and `next_action=retry_same_cleanup`; generic retry handlers must not choose a +new token. After `ready_to_retry`, resubmit the original append inputs. + An interrupted publication must be fenced and cleaned before a successor attempt can start. Status queries only observe this lifecycle. Persist the delta or a way to reproduce the exact bytes until the operation commits. An unknown @@ -317,14 +349,16 @@ do not expose identical method shapes. | Surface | Current public capability | Additional boundary | | --- | --- | --- | -| Direct Python `Client` | Create/stat/exists/list/remove/rename, byte/file publish, stable append and its operation status, whole/range/batch reads, query/aggregate/catalog/find, commit/restore, snapshot lifecycle, materialize, and collect | Direct SDK, not a tool-for-tool copy of the 18-name CLI facade | +| Direct Python `Client` | Create/stat/exists/list/remove/rename, byte/file publish, stable append with status/inspect/recover, whole/range/batch reads, query/aggregate/catalog/find, commit/restore, snapshot lifecycle, materialize, and collect | Direct SDK, not a tool-for-tool copy of the 18-name CLI facade | | Python adapters | Workbench-scoped fsspec, checkpoint helpers, and optional torch Distributed Checkpoint reader/writer | Bounded to an explicit Workbench; not arbitrary-root POSIX or FUSE | | Rust `WorkspaceClient` | Lower-level typed workspace, publication, query, lifecycle, routing, and batch-range workflows | Recommended when the caller must own typed retry and recovery integration | | Rust-only extensions | Polling change feed, generic custom-index registration, raw non-append operation status, and phased publish/restore primitives | No native CLI or Python method today; change feed is polling, not push | | Metadata and server lifecycle | snapshot reap, commit/tag holds, reference-fenced GC, quarantine reconciliation, Holt reopen, and optional shared recovery records | Internal/operator mechanisms, not independent end-user commands | -Python `append_bytes` and `operation_status` share the native stable append -contract. The SDK does not expose the legacy 18-tool append facade, exact-string +Python `append_bytes`, `operation_status`, `operation_inspect`, and +`operation_recover` share the native stable append contract. Python inspection +cursors are opaque bytes; recovery state digests are lowercase hex strings. +The SDK does not expose the legacy 18-tool append facade, exact-string edit, or body grep methods; use the CLI for those exact behaviors. Custom SDK compositions are caller-owned and are not equivalent qualification evidence. diff --git a/crates/nokv-client/src/append_operations.rs b/crates/nokv-client/src/append_operations.rs new file mode 100644 index 000000000..dacecc8aa --- /dev/null +++ b/crates/nokv-client/src/append_operations.rs @@ -0,0 +1,309 @@ +// Copyright 2024-2026 The NoKV Authors. +// SPDX-License-Identifier: Apache-2.0 + +//! Metadata-only inspection and owner-executed recovery of logical appends. + +use nokv_protocol::{ + AppendCleanupInspection, AppendCleanupRetryResult, Digest, ErrorCode, + InspectAppendCleanupRequest, OperationIdentity, OperationStatus, OperationToken, PageRequest, + RetryAppendCleanupRequest, RootIdentity, RpcFailure, WorkspaceCapability, WorkspaceRequest, + WorkspaceResult, +}; +use sha2::{Digest as _, Sha256}; + +use crate::{ + append_operation_recovery, AppendRecoveryState, ClientCall, ClientError, RouteResolver, + RpcTransport, WorkspaceClient, +}; + +pub const DEFAULT_APPEND_INSPECTION_LIMIT: u32 = 32; +pub const MAX_APPEND_INSPECTION_LIMIT: u32 = nokv_protocol::MAX_ARTIFACT_PUBLISH_BATCH_ROWS as u32; + +// A versioned SDK cursor binds the root, logical state token and last ordinal. +// The checksum detects damaged cursors; authorization remains with the server's +// exact token/namespace/ledger validation, not this unkeyed checksum. +const CURSOR_TAG: &[u8; 8] = b"NKAPIN01"; +const CURSOR_PAYLOAD_BYTES: usize = 76; +const CURSOR_BYTES: usize = CURSOR_PAYLOAD_BYTES + 32; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AppendOperationInspection { + pub inspection: AppendCleanupInspection, + /// Continue this exact observation. Restart inspection if its state changes. + pub next_cursor: Option>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AppendRecoveryRequestResult { + /// A fresh observation after admission; not a promise of cleanup completion. + pub operation: OperationStatus, + /// True when this request has a durable cleanup retry receipt, including replay. + pub requested: bool, + /// Historical admission receipt, distinct from the logical append's receipt. + pub receipt: Option, +} + +impl WorkspaceClient +where + Transport: RpcTransport, + Resolver: RouteResolver, +{ + /// Inspect retained staged entries without payloads or object credentials. + /// + /// Every page binds the complete logical/child observation. A stale cursor + /// fails rather than combining ledger rows from different cleanup progress + /// or publication attempts. Retired entries are not historical inventory. + pub fn inspect_append_operation( + &self, + operation_id: OperationIdentity, + page: PageRequest, + ) -> Result, ClientError> { + if !(1..=MAX_APPEND_INSPECTION_LIMIT).contains(&page.limit) { + return Err(invalid_input(format!( + "append inspection limit must be between 1 and {MAX_APPEND_INSPECTION_LIMIT}" + ))); + } + let continuation = page + .cursor + .as_deref() + .map(|cursor| decode_cursor(self.root_id(), operation_id, cursor)) + .transpose()?; + let namespace = self + .preflight([WorkspaceCapability::ArtifactAppendRecoveryV1])? + .value + .route + .object_namespace_id; + let (token, start_after) = match continuation { + Some((token, after)) => (token, Some(after)), + None => (self.get_append_operation(operation_id)?.value.token, None), + }; + let call = self.execute_read(WorkspaceRequest::InspectAppendCleanup( + InspectAppendCleanupRequest { + token, + start_after, + limit: page.limit, + }, + ))?; + call.map(|result| { + let WorkspaceResult::AppendCleanupInspection(inspection) = result else { + return Err(ClientError::ResponseMismatch( + "expected append cleanup inspection".to_owned(), + )); + }; + validate_inspection(&inspection, token, start_after, page.limit)?; + if inspection.object_namespace_id != namespace { + return Err(ClientError::ResponseMismatch( + "append inspection returned a different object namespace".to_owned(), + )); + } + let next_cursor = inspection + .next_after + .map(|after| encode_cursor(self.root_id(), token, after)); + Ok(AppendOperationInspection { + inspection, + next_cursor, + }) + }) + } + + /// Ask the fenced owner to retry quarantined cleanup once. + /// + /// Persist the token from inspection and provide it on every retry of the + /// same recovery request, including after process death. Omitting a token + /// explicitly starts a request against the current observation; it is a + /// no-op for active, cleaned or committed attempts. This method never chooses + /// another token after a race or an unknown acknowledgement. It does not + /// upload, seal objects itself, admit a successor, or reconstruct the delta. + pub fn recover_append_operation( + &self, + operation_id: OperationIdentity, + expected_token: Option, + ) -> Result, ClientError> { + if expected_token.is_some_and(|token| token.operation_id != operation_id) { + return Err(invalid_input( + "append recovery token belongs to a different logical operation", + )); + } + self.preflight([WorkspaceCapability::ArtifactAppendRecoveryV1])?; + let (token, expected_receipt) = match expected_token { + Some(token) => (token, None), + None => { + let call = self.get_append_operation(operation_id)?; + if append_operation_recovery(&call.value)?.state != AppendRecoveryState::Quarantined + { + return call.map(|operation| { + Ok(AppendRecoveryRequestResult { + operation, + requested: false, + receipt: None, + }) + }); + } + let preparation = call.value.append_preparation.as_ref().ok_or_else(|| { + ClientError::ResponseMismatch("append status has no preparation".to_owned()) + })?; + let count = preparation + .cleanup_retry_count + .checked_add(1) + .ok_or_else(|| invalid_input("append cleanup retry counter is exhausted"))?; + ( + call.value.token, + Some((preparation.publication_operation_id, count)), + ) + } + }; + let retry = self + .execute( + self.new_request_id(), + WorkspaceRequest::RetryAppendCleanup(RetryAppendCleanupRequest { token }), + ) + .and_then(|call| { + call.map(|result| { + let WorkspaceResult::AppendCleanupRetried(receipt) = result else { + return Err(ClientError::ResponseMismatch( + "expected append cleanup retry receipt".to_owned(), + )); + }; + if receipt.operation_id != operation_id + || receipt.expected_state_digest != token.state_digest + || receipt.publication_operation_id == operation_id + || receipt.cleanup_retry_count == 0 + || expected_receipt.is_some_and(|(child, count)| { + receipt.publication_operation_id != child + || receipt.cleanup_retry_count != count + }) + { + return Err(ClientError::ResponseMismatch( + "append cleanup receipt does not match the requested observation" + .to_owned(), + )); + } + Ok(receipt) + }) + }) + .map_err(|source| cleanup_unresolved(token, None, source))?; + let operation = self + .get_append_operation(operation_id) + .map_err(|source| cleanup_unresolved(token, Some(retry.value.clone()), source))? + .value; + retry.map(|receipt| { + Ok(AppendRecoveryRequestResult { + operation, + requested: true, + receipt: Some(receipt), + }) + }) + } +} + +fn validate_inspection( + inspection: &AppendCleanupInspection, + token: OperationToken, + start_after: Option, + limit: u32, +) -> Result<(), ClientError> { + let mismatch = || { + ClientError::ResponseMismatch( + "append inspection does not match its exact token and retained ledger interval" + .to_owned(), + ) + }; + append_operation_recovery(&inspection.operation)?; + let preparation = inspection + .operation + .append_preparation + .as_ref() + .ok_or_else(mismatch)?; + if inspection.operation.token != token + || inspection.publication_token.operation_id != preparation.publication_operation_id + || inspection.cleanup_cursor > inspection.registered_count + || inspection.remaining_count != inspection.registered_count - inspection.cleanup_cursor + { + return Err(mismatch()); + } + let start = match start_after { + Some(after) => after.checked_add(1).ok_or_else(mismatch)?, + None => inspection.cleanup_cursor, + }; + if start < inspection.cleanup_cursor || start > inspection.registered_count { + return Err(mismatch()); + } + let count = (inspection.registered_count - start).min(limit); + if inspection.entries.len() != count as usize { + return Err(mismatch()); + } + for (index, entry) in inspection.entries.iter().enumerate() { + if entry.sequence != start + index as u32 { + return Err(mismatch()); + } + } + let end = start + count; + let expected_next = (end < inspection.registered_count).then(|| end - 1); + if inspection.next_after != expected_next { + return Err(mismatch()); + } + Ok(()) +} + +fn encode_cursor(root: RootIdentity, token: OperationToken, after: u32) -> Vec { + let mut bytes = Vec::with_capacity(CURSOR_BYTES); + bytes.extend_from_slice(CURSOR_TAG); + bytes.extend_from_slice(&root.0); + bytes.extend_from_slice(&token.operation_id.0); + bytes.extend_from_slice(&token.state_digest.0); + bytes.extend_from_slice(&after.to_be_bytes()); + let checksum = Sha256::digest(&bytes); + bytes.extend_from_slice(&checksum); + bytes +} + +fn decode_cursor( + root: RootIdentity, + operation_id: OperationIdentity, + bytes: &[u8], +) -> Result<(OperationToken, u32), ClientError> { + let invalid = || invalid_input("invalid append inspection cursor; restart inspection"); + if bytes.len() != CURSOR_BYTES + || &bytes[..8] != CURSOR_TAG + || bytes[8..24] != root.0 + || bytes[24..40] != operation_id.0 + || bytes[CURSOR_PAYLOAD_BYTES..] != Sha256::digest(&bytes[..CURSOR_PAYLOAD_BYTES])[..] + { + return Err(invalid()); + } + Ok(( + OperationToken { + operation_id, + state_digest: Digest(bytes[40..72].try_into().map_err(|_| invalid())?), + }, + u32::from_be_bytes(bytes[72..76].try_into().map_err(|_| invalid())?), + )) +} + +fn invalid_input(message: impl Into) -> ClientError { + ClientError::Rpc(RpcFailure { + code: ErrorCode::InvalidArgument, + message: message.into(), + retryable: false, + conflict: None, + current_generation: None, + route_hint: None, + }) +} + +fn cleanup_unresolved( + expected_token: OperationToken, + receipt: Option, + source: ClientError, +) -> ClientError { + ClientError::AppendCleanupUnresolved { + operation_id: expected_token.operation_id, + expected_token, + receipt: receipt.map(Box::new), + source: Box::new(source), + } +} + +#[cfg(test)] +#[path = "append_operations_tests.rs"] +mod tests; diff --git a/crates/nokv-client/src/append_operations_tests.rs b/crates/nokv-client/src/append_operations_tests.rs new file mode 100644 index 000000000..9b50d0826 --- /dev/null +++ b/crates/nokv-client/src/append_operations_tests.rs @@ -0,0 +1,367 @@ +// Copyright 2024-2026 The NoKV Authors. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::{ClientOptions, StaticRouteResolver, TransportError}; +use nokv_protocol::{ + decode_request, encode_response, sha256_digest_uri, stable_append_attempt_identities, + AppendAttemptPhase, AppendPreparation, ConflictKind, LogicalShardIdentity, ObjectIdentity, + ObjectNamespaceIdentity, OperationKind, OperationProgress, OperationState, RelativePath, + RootRoute, StagedObject, WorkbenchName, WorkspaceIdentity, WorkspacePath, + WorkspacePreflightResult, WorkspaceRpcOutcome, WorkspaceRpcResponse, +}; +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, +}; + +fn route() -> RootRoute { + RootRoute { + root_id: RootIdentity([1; 16]), + logical_shard_id: LogicalShardIdentity([2; 16]), + object_namespace_id: ObjectNamespaceIdentity([3; 16]), + placement_generation: 1, + owner_epoch: 1, + } +} + +fn status(phase: AppendAttemptPhase, count: u64) -> OperationStatus { + let operation_id = OperationIdentity([4; 16]); + let (publication_operation_id, artifact_revision_id) = + stable_append_attempt_identities(route().root_id, operation_id, 0); + OperationStatus { + token: OperationToken { + operation_id, + state_digest: Digest([count as u8 + 10; 32]), + }, + kind: OperationKind::ArtifactAppend, + append_preparation: Some(Box::new(AppendPreparation { + intent_digest: Digest([5; 32]), + target: WorkspacePath { + workbench: WorkbenchName::new("queue").unwrap(), + path: RelativePath::new("logs/events.jsonl").unwrap(), + }, + workspace_incarnation_id: WorkspaceIdentity([6; 16]), + attempt: 0, + publication_operation_id, + artifact_revision_id, + attempt_phase: phase, + activity_deadline_ms: 1_000, + cleanup_retry_count: count, + attempt_failure: matches!( + phase, + AppendAttemptPhase::Quarantined | AppendAttemptPhase::Cleaned + ) + .then(|| RpcFailure { + code: ErrorCode::Quarantined, + message: "provider outcome could not be proved".to_owned(), + retryable: false, + conflict: Some(ConflictKind::OperationState), + current_generation: None, + route_hint: None, + }), + })), + publish_preparation: None, + commit_preparation: None, + restore_preparation: None, + state: if phase == AppendAttemptPhase::Quarantined { + OperationState::Quarantined + } else { + OperationState::Running + }, + progress: OperationProgress { + completed_rows: 2, + total_rows: Some(4), + completed_bytes: 8, + total_bytes: Some(16), + }, + result: None, + failure: (phase == AppendAttemptPhase::Quarantined).then(|| RpcFailure { + code: ErrorCode::Quarantined, + message: "provider outcome could not be proved".to_owned(), + retryable: false, + conflict: Some(ConflictKind::OperationState), + current_generation: None, + route_hint: None, + }), + } +} + +type ScriptedReply = Result<(WorkspaceResult, bool), TransportError>; + +#[derive(Clone, Default)] +struct ScriptedTransport { + replies: Arc>>, + requests: Arc>>, +} + +impl RpcTransport for ScriptedTransport { + fn round_trip(&self, _: std::net::SocketAddr, bytes: &[u8]) -> Result, TransportError> { + let request = decode_request(bytes).unwrap(); + let (result, replayed) = if matches!(request.operation, WorkspaceRequest::Preflight(_)) { + ( + WorkspaceResult::Preflight(WorkspacePreflightResult::new( + request.route, + WorkspaceCapability::ALL, + )), + false, + ) + } else { + self.requests.lock().unwrap().push(request.operation); + self.replies + .lock() + .unwrap() + .pop_front() + .expect("unexpected business RPC")? + }; + encode_response(&WorkspaceRpcResponse { + route: request.route, + request_id: request.request_id, + commit_version: matches!(result, WorkspaceResult::AppendCleanupRetried(_)) + .then_some(19), + replayed, + outcome: WorkspaceRpcOutcome::Success(Box::new(result)), + }) + .map_err(|error| TransportError::new(error.to_string(), false)) + } +} + +fn client( + transport: &ScriptedTransport, +) -> WorkspaceClient { + WorkspaceClient::new( + route().root_id, + transport.clone(), + StaticRouteResolver::new(route(), ([127, 0, 0, 1], 4100).into()).unwrap(), + ClientOptions::default(), + ) + .unwrap() +} + +fn retry_receipt(observation: &OperationStatus) -> AppendCleanupRetryResult { + let preparation = observation.append_preparation.as_ref().unwrap(); + AppendCleanupRetryResult { + operation_id: observation.token.operation_id, + publication_operation_id: preparation.publication_operation_id, + cleanup_retry_count: preparation.cleanup_retry_count + 1, + expected_state_digest: observation.token.state_digest, + } +} + +fn inspection(observation: OperationStatus, start: u32, end: u32) -> AppendCleanupInspection { + AppendCleanupInspection { + publication_token: OperationToken { + operation_id: observation + .append_preparation + .as_ref() + .unwrap() + .publication_operation_id, + state_digest: Digest([44; 32]), + }, + operation: Box::new(observation), + object_namespace_id: route().object_namespace_id, + registered_count: 5, + cleanup_cursor: 2, + remaining_count: 3, + entries: (start..end) + .map(|sequence| StagedObject { + sequence, + object_identity: ObjectIdentity::new(format!("object-{sequence}")).unwrap(), + expected_length: 4, + expected_digest: sha256_digest_uri(Digest([7; 32])), + multipart_token: None, + }) + .collect(), + next_after: (end < 5).then(|| end - 1), + } +} + +#[test] +fn retained_ledger_pages_keep_one_exact_token_and_reject_wrong_scope_before_io() { + let transport = ScriptedTransport::default(); + let observation = status(AppendAttemptPhase::Quarantined, 0); + transport.replies.lock().unwrap().extend([ + Ok((WorkspaceResult::Operation(observation.clone()), false)), + Ok(( + WorkspaceResult::AppendCleanupInspection(inspection(observation.clone(), 2, 4)), + false, + )), + Ok(( + WorkspaceResult::AppendCleanupInspection(inspection(observation.clone(), 4, 5)), + false, + )), + ]); + let client = client(&transport); + let first = client + .inspect_append_operation( + observation.token.operation_id, + PageRequest { + limit: 2, + cursor: None, + }, + ) + .unwrap(); + let cursor = first.value.next_cursor.unwrap(); + let second = client + .inspect_append_operation( + observation.token.operation_id, + PageRequest { + limit: 2, + cursor: Some(cursor.clone()), + }, + ) + .unwrap(); + assert!(second.value.next_cursor.is_none()); + let requests = transport.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert!( + matches!(&requests[2], WorkspaceRequest::InspectAppendCleanup(request) if request.token == observation.token && request.start_after == Some(3)) + ); + drop(requests); + assert!(client + .inspect_append_operation( + OperationIdentity([88; 16]), + PageRequest { + limit: 2, + cursor: Some(cursor.clone()) + } + ) + .is_err()); + let mut damaged = cursor; + damaged[45] ^= 1; + assert!(client + .inspect_append_operation( + observation.token.operation_id, + PageRequest { + limit: 2, + cursor: Some(damaged) + } + ) + .is_err()); + assert_eq!(transport.requests.lock().unwrap().len(), 3); +} + +#[test] +fn inspection_rejects_a_truncated_page_even_when_wire_shape_is_valid() { + let observation = status(AppendAttemptPhase::Quarantined, 0); + let mut page = inspection(observation.clone(), 3, 5); + assert!(validate_inspection(&page, observation.token, None, 3).is_err()); + page = inspection(observation.clone(), 2, 5); + page.operation.token.state_digest = Digest([55; 32]); + assert!(validate_inspection(&page, observation.token, None, 3).is_err()); +} + +#[test] +fn explicit_recovery_replays_old_receipt_before_observing_a_new_quarantine() { + let transport = ScriptedTransport::default(); + let original = status(AppendAttemptPhase::Quarantined, 0); + let current = status(AppendAttemptPhase::Quarantined, 1); + let receipt = retry_receipt(&original); + transport.replies.lock().unwrap().extend([ + Ok((WorkspaceResult::AppendCleanupRetried(receipt.clone()), true)), + Ok((WorkspaceResult::Operation(current.clone()), false)), + ]); + let call = client(&transport) + .recover_append_operation(original.token.operation_id, Some(original.token)) + .unwrap(); + assert!(call.replayed); + assert!(call.value.requested); + assert_eq!(call.value.receipt, Some(receipt)); + assert_eq!(call.value.operation, current); + assert!( + matches!(&transport.requests.lock().unwrap()[0], WorkspaceRequest::RetryAppendCleanup(request) if request.token == original.token) + ); + assert_eq!(transport.requests.lock().unwrap().len(), 2); +} + +#[test] +fn unknown_cleanup_ack_retains_the_original_token_without_selecting_a_new_round() { + let transport = ScriptedTransport::default(); + let observation = status(AppendAttemptPhase::Quarantined, 0); + transport + .replies + .lock() + .unwrap() + .push_back(Err(TransportError::new("acknowledgement lost", false))); + let error = client(&transport) + .recover_append_operation(observation.token.operation_id, Some(observation.token)) + .unwrap_err(); + assert!(!error.retryable()); + assert!( + matches!(error, ClientError::AppendCleanupUnresolved { expected_token, receipt: None, .. } if expected_token == observation.token) + ); + assert_eq!(transport.requests.lock().unwrap().len(), 1); +} + +#[test] +fn implicit_recovery_receipt_must_match_the_observed_child_and_next_counter() { + let observation = status(AppendAttemptPhase::Quarantined, 0); + for wrong_child in [true, false] { + let transport = ScriptedTransport::default(); + let mut receipt = retry_receipt(&observation); + if wrong_child { + receipt.publication_operation_id = OperationIdentity([99; 16]); + } else { + receipt.cleanup_retry_count = 99; + } + transport.replies.lock().unwrap().extend([ + Ok((WorkspaceResult::Operation(observation.clone()), false)), + Ok((WorkspaceResult::AppendCleanupRetried(receipt), false)), + ]); + let error = client(&transport) + .recover_append_operation(observation.token.operation_id, None) + .unwrap_err(); + assert!( + matches!(error, ClientError::AppendCleanupUnresolved { source, .. } + if matches!(*source, ClientError::ResponseMismatch(_))) + ); + assert_eq!(transport.requests.lock().unwrap().len(), 2); + } +} + +#[test] +fn successful_cleanup_admission_receipt_survives_a_failed_status_read() { + let transport = ScriptedTransport::default(); + let observation = status(AppendAttemptPhase::Quarantined, 0); + let receipt = retry_receipt(&observation); + transport.replies.lock().unwrap().extend([ + Ok(( + WorkspaceResult::AppendCleanupRetried(receipt.clone()), + false, + )), + Err(TransportError::new( + "owner unavailable after admission", + false, + )), + ]); + let error = client(&transport) + .recover_append_operation(observation.token.operation_id, Some(observation.token)) + .unwrap_err(); + assert!( + matches!(error, ClientError::AppendCleanupUnresolved { expected_token, receipt: Some(actual), .. } if expected_token == observation.token && *actual == receipt) + ); +} + +#[test] +fn implicit_recovery_cannot_abort_active_or_cleaned_attempts() { + for phase in [ + AppendAttemptPhase::Uploading, + AppendAttemptPhase::Finalizing, + AppendAttemptPhase::Cleaning, + AppendAttemptPhase::Cleaned, + ] { + let transport = ScriptedTransport::default(); + let observation = status(phase, 0); + transport + .replies + .lock() + .unwrap() + .push_back(Ok((WorkspaceResult::Operation(observation.clone()), false))); + let call = client(&transport) + .recover_append_operation(observation.token.operation_id, None) + .unwrap(); + assert!(!call.value.requested); + assert_eq!(call.value.receipt, None); + assert_eq!(transport.requests.lock().unwrap().len(), 1); + } +} diff --git a/crates/nokv-client/src/artifact.rs b/crates/nokv-client/src/artifact.rs index 7c31140d8..518c80f6f 100644 --- a/crates/nokv-client/src/artifact.rs +++ b/crates/nokv-client/src/artifact.rs @@ -2508,7 +2508,8 @@ fn is_append_retry_error(error: &ClientError) -> bool { | ClientError::ArtifactIntegrity(_) | ClientError::Object(_) | ClientError::ArtifactUpload(_) - | ClientError::AppendUnresolved { .. } => false, + | ClientError::AppendUnresolved { .. } + | ClientError::AppendCleanupUnresolved { .. } => false, } } diff --git a/crates/nokv-client/src/error.rs b/crates/nokv-client/src/error.rs index 2ad7b9f21..625d9079a 100644 --- a/crates/nokv-client/src/error.rs +++ b/crates/nokv-client/src/error.rs @@ -7,7 +7,8 @@ use std::fmt; use nokv_object::{ArtifactUploadFailure, ObjectError}; use nokv_protocol::{ - ErrorCode, OperationIdentity, OperationState, ProtocolError, RpcFailure, WorkspaceCapability, + AppendCleanupRetryResult, ErrorCode, OperationIdentity, OperationState, OperationToken, + ProtocolError, RpcFailure, WorkspaceCapability, }; #[derive(Clone, Debug, PartialEq, Eq)] @@ -76,6 +77,14 @@ pub enum ClientError { state: Option, source: Box, }, + /// Cleanup was requested against this exact observation. Retain its token + /// across process retries; a fresh observation may authorize another round. + AppendCleanupUnresolved { + operation_id: OperationIdentity, + expected_token: OperationToken, + receipt: Option>, + source: Box, + }, RetryExhausted { attempts: u32, last_error: Box, @@ -97,7 +106,7 @@ impl ClientError { Self::RetryExhausted { last_error, .. } => last_error.retryable(), // Generic retry loops must not turn an unresolved append into a // fresh operation. The caller can explicitly retry the same intent. - Self::AppendUnresolved { .. } => false, + Self::AppendUnresolved { .. } | Self::AppendCleanupUnresolved { .. } => false, Self::ArtifactReadFenceChanged => true, Self::InvalidOptions(_) | Self::InvalidRoute(_) @@ -113,6 +122,7 @@ impl ClientError { Self::Rpc(failure) => Some(failure), Self::ArtifactPublishFailed { source, .. } => source.rpc_failure(), Self::AppendUnresolved { source, .. } => source.rpc_failure(), + Self::AppendCleanupUnresolved { source, .. } => source.rpc_failure(), Self::RetryExhausted { last_error, .. } => last_error.rpc_failure(), _ => None, } @@ -196,6 +206,16 @@ impl fmt::Display for ClientError { formatter, "append operation {operation_id:?} has state {state:?}: {source}; retain the same operation identity" ), + Self::AppendCleanupUnresolved { + operation_id, + expected_token, + source, + .. + } => write!( + formatter, + "append cleanup for {operation_id:?} is unresolved: {source}; retry with the same expected state digest {:?}", + expected_token.state_digest + ), } } } @@ -222,6 +242,7 @@ impl std::error::Error for ClientError { Self::ArtifactUpload(error) => Some(error.as_ref()), Self::ArtifactPublishFailed { source, .. } => Some(source.as_ref()), Self::AppendUnresolved { source, .. } => Some(source.as_ref()), + Self::AppendCleanupUnresolved { source, .. } => Some(source.as_ref()), Self::RetryExhausted { last_error, .. } => Some(last_error.as_ref()), Self::InvalidOptions(_) | Self::InvalidRoute(_) diff --git a/crates/nokv-client/src/idempotent_append.rs b/crates/nokv-client/src/idempotent_append.rs index 1f1bcf044..21186c8c6 100644 --- a/crates/nokv-client/src/idempotent_append.rs +++ b/crates/nokv-client/src/idempotent_append.rs @@ -100,7 +100,7 @@ pub enum AppendNextAction { None, Poll, ResubmitSame, - OperatorReconcile, + RetryCleanup, } impl AppendNextAction { @@ -109,7 +109,7 @@ impl AppendNextAction { Self::None => "none", Self::Poll => "poll", Self::ResubmitSame => "resubmit_same", - Self::OperatorReconcile => "operator_reconcile", + Self::RetryCleanup => "retry_cleanup", } } } @@ -160,7 +160,7 @@ pub fn append_operation_recovery(status: &OperationStatus) -> Result AppendRecovery { state: AppendRecoveryState::Quarantined, - next_action: AppendNextAction::OperatorReconcile, + next_action: AppendNextAction::RetryCleanup, }, AppendAttemptPhase::Uploading | AppendAttemptPhase::Finalizing @@ -688,6 +688,7 @@ mod tests { artifact_revision_id, attempt_phase: AppendAttemptPhase::Published, attempt_failure: None, + cleanup_retry_count: 0, activity_deadline_ms: 1234, })), publish_preparation: None, diff --git a/crates/nokv-client/src/lib.rs b/crates/nokv-client/src/lib.rs index 76762d7f9..873cdd543 100644 --- a/crates/nokv-client/src/lib.rs +++ b/crates/nokv-client/src/lib.rs @@ -9,6 +9,7 @@ //! result extraction. Metadata layout and shard-owner implementation remain //! outside this crate. +mod append_operations; mod artifact; mod error; mod generic_index; @@ -20,6 +21,10 @@ mod transport; mod workbench_lifecycle; mod workbench_workflow; +pub use append_operations::{ + AppendOperationInspection, AppendRecoveryRequestResult, DEFAULT_APPEND_INSPECTION_LIMIT, + MAX_APPEND_INSPECTION_LIMIT, +}; pub use artifact::{ ArtifactAppendOptions, ArtifactAppendOutcome, ArtifactPublishOptions, ArtifactPublishOutcome, ArtifactRangeBatchItem, ArtifactRangeBatchOutcome, ArtifactRangeBatchRequest, diff --git a/crates/nokv-meta/src/workspace/codec.rs b/crates/nokv-meta/src/workspace/codec.rs index b5cccde93..e4405490c 100644 --- a/crates/nokv-meta/src/workspace/codec.rs +++ b/crates/nokv-meta/src/workspace/codec.rs @@ -24,7 +24,7 @@ const GENERIC_INDEX_APPEND_RECEIPT_DISCRIMINATOR: u8 = 3; const FIXED_ID_BYTES: usize = 16; pub(crate) const SYSTEM_SCHEMA_KEY: &[u8] = b"schema"; -const SYSTEM_FORMAT_VERSION: u32 = 12; +const SYSTEM_FORMAT_VERSION: u32 = 13; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum SchemaMarkerVersion { @@ -1308,7 +1308,7 @@ mod tests { #[test] fn schema_marker_is_exact_and_versioned() { - assert_eq!(SYSTEM_FORMAT_VERSION, 12); + assert_eq!(SYSTEM_FORMAT_VERSION, 13); let marker = encode_schema_marker(); assert_eq!(marker[0], VALUE_FORMAT_VERSION); assert_eq!( @@ -1321,7 +1321,7 @@ mod tests { SchemaMarkerVersion::Current ); - let previous_layout = encode_schema_marker_version(10); + let previous_layout = encode_schema_marker_version(12); assert_eq!( validate_schema_marker(&previous_layout), Err(SchemaMarkerError) diff --git a/crates/nokv-meta/src/workspace/mod.rs b/crates/nokv-meta/src/workspace/mod.rs index 0d80dcc43..9a1b76ed7 100644 --- a/crates/nokv-meta/src/workspace/mod.rs +++ b/crates/nokv-meta/src/workspace/mod.rs @@ -142,12 +142,13 @@ pub use namespace::{ MAX_VISIBLE_PATH_LIST_PAGE_SIZE, }; pub use publication::{ - advance_manifest_rolling_digest, advance_staged_object_rolling_digest, dependency_owner_digest, - manifest_rows_digest, seal_publish_operation, staged_object_ledger_digest, BeginPublishRequest, - CleanupPublishBatchRequest, FinalizePublishOutcome, FinalizePublishRequest, - FinishReconcileQuarantinedPublishRequest, HeartbeatPublishRequest, ManifestRowInput, - MarkObjectsUploadedBatchRequest, PublicationContext, PublicationError, PublicationService, - PublishCommandOutcome, PublishedArtifact, QuarantineReconcileResolution, + advance_manifest_rolling_digest, advance_staged_object_rolling_digest, + append_operation_state_digest, dependency_owner_digest, manifest_rows_digest, + seal_publish_operation, staged_object_ledger_digest, AppendCleanupRetryOutcome, + BeginPublishRequest, CleanupPublishBatchRequest, FinalizePublishOutcome, + FinalizePublishRequest, FinishReconcileQuarantinedPublishRequest, HeartbeatPublishRequest, + ManifestRowInput, MarkObjectsUploadedBatchRequest, PublicationContext, PublicationError, + PublicationService, PublishCommandOutcome, PublishedArtifact, QuarantineReconcileResolution, ReconcileQuarantinedPublishBatchRequest, StageManifestBatchRequest, StageObjectsBatchRequest, StagedObjectUpdate, TakeOverOrphanedPublishRequest, TransitionPublishRequest, MAX_PUBLICATION_BATCH_ROWS, @@ -161,11 +162,12 @@ pub use publication_records::{ MAX_QUARANTINE_EVIDENCE_BYTES, PUBLICATION_VALUE_FORMAT_VERSION, }; pub use publish_operation_records::{ - AppendAttemptBinding, AppendOperationRecord, AppendSegment, ArtifactManifestRow, - ManifestPosition, PublishAuthority, PublishClaim, PublishOperationRecord, PublishRecordError, - PublishResult, PublishTerminalError, PublishTerminalErrorKind, PublishTransition, - StagedObjectRecord, MAX_APPEND_SEGMENTS, MAX_MANIFEST_ROWS, MAX_MULTIPART_ID_BYTES, - MAX_STAGED_OBJECTS, MAX_TERMINAL_ERROR_BYTES, PUBLISH_VALUE_FORMAT_VERSION, + AppendAttemptBinding, AppendCleanupRetryReceipt, AppendOperationRecord, AppendSegment, + ArtifactManifestRow, ManifestPosition, PublishAuthority, PublishClaim, PublishOperationRecord, + PublishRecordError, PublishResult, PublishTerminalError, PublishTerminalErrorKind, + PublishTransition, StagedObjectRecord, MAX_APPEND_SEGMENTS, MAX_MANIFEST_ROWS, + MAX_MULTIPART_ID_BYTES, MAX_STAGED_OBJECTS, MAX_TERMINAL_ERROR_BYTES, + PUBLISH_VALUE_FORMAT_VERSION, }; pub use query::{ aggregate_paths_at, catalog_fields_at, find_workspaces_at, get_workspace_at, read_changes_at, diff --git a/crates/nokv-meta/src/workspace/publication.rs b/crates/nokv-meta/src/workspace/publication.rs index 7e88cf79b..2100174db 100644 --- a/crates/nokv-meta/src/workspace/publication.rs +++ b/crates/nokv-meta/src/workspace/publication.rs @@ -41,10 +41,10 @@ use super::publication_records::{ PublicationRecordCodecError, RevisionRefRecord, WorkspaceRecord, }; use super::publish_operation_records::{ - AppendOperationRecord, ArtifactManifestRow, ManifestPosition, PublishAuthority, PublishClaim, - PublishOperationRecord, PublishRecordError, PublishResult, PublishTerminalError, - PublishTerminalErrorKind, PublishTransition, StagedObjectRecord, MAX_DEPENDENCY_COUNT, - MAX_MANIFEST_ROWS, MAX_STAGED_OBJECTS, + AppendCleanupRetryReceipt, AppendOperationRecord, ArtifactManifestRow, ManifestPosition, + PublishAuthority, PublishClaim, PublishOperationRecord, PublishRecordError, PublishResult, + PublishTerminalError, PublishTerminalErrorKind, PublishTransition, StagedObjectRecord, + MAX_DEPENDENCY_COUNT, MAX_MANIFEST_ROWS, MAX_STAGED_OBJECTS, }; use super::query_records::{ secondary_index_key, ChangeEventKind, ChangeEventRecord, QueryRecordError, @@ -57,6 +57,42 @@ use super::restore_records::{ RESTORE_MANIFEST_CONTENT_TYPE, }; +/// Exact logical token binds both current parent mapping and all mutable child progress. +pub fn append_operation_state_digest( + parent: &AppendOperationRecord, + child: &PublishOperationRecord, +) -> Result<[u8; SHA256_BYTES], PublishRecordError> { + let mut hasher = Sha256::new(); + hasher.update(b"nokv.append.state.v1\0"); + hasher.update(parent.encode()?); + hasher.update(Sha256::digest(child.encode()?)); + Ok(hasher.finalize().into()) +} + +fn append_cleanup_retry_request_id( + root: RootId, + operation: OperationId, + state: [u8; SHA256_BYTES], +) -> RequestId { + let mut hasher = Sha256::new(); + hasher.update(b"nokv.append.cleanup-retry.request.v1\0"); + hasher.update(root.as_bytes()); + hasher.update(operation.as_bytes()); + hasher.update(state); + let digest = hasher.finalize(); + let mut id = [0; 16]; + id.copy_from_slice(&digest[..16]); + RequestId::from_bytes(id) +} + +/// One durable cleanup enqueue receipt plus this invocation's replay metadata. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AppendCleanupRetryOutcome { + pub receipt: AppendCleanupRetryReceipt, + pub commit_version: CommitVersion, + pub replayed: bool, +} + /// Maximum rows admitted by one recoverable publication batch. pub const MAX_PUBLICATION_BATCH_ROWS: usize = 192; const _: () = assert!(MAX_PUBLICATION_BATCH_ROWS + 2 <= MAX_COMMAND_ITEMS); @@ -179,9 +215,6 @@ pub enum QuarantineReconcileResolution { /// was never published. Reconciliation releases the revision identity for /// a fresh begin. RevisionUnpublished, - /// Every failed append key is permanently sealed against delayed immutable - /// creates. The unpublished revision identity remains reserved forever. - RevisionUnpublishedSealed, /// The artifact revision is already published, so the staged provider /// keys are the published revision's live objects and must not be touched. /// Only this operation's private bookkeeping rows are removed. @@ -871,6 +904,7 @@ fn validate_begin_request(request: &BeginPublishRequest) -> Result<(), Publicati || request.operation.manifest_last_position.is_some() || request.operation.cleanup_staged_object_cursor != 0 || request.operation.cleanup_manifest_cursor != 0 + || request.operation.cleanup_retry_count != 0 { return Err(PublicationError::InvalidOperationSeal { seal: "initial cursor", @@ -1415,7 +1449,6 @@ fn finalization_takeover_absence_proof( fn reconcile_resolution_label(resolution: QuarantineReconcileResolution) -> &'static str { match resolution { QuarantineReconcileResolution::RevisionUnpublished => "revision-unpublished", - QuarantineReconcileResolution::RevisionUnpublishedSealed => "revision-unpublished-sealed", QuarantineReconcileResolution::RevisionPublished => "revision-published", } } @@ -1438,7 +1471,6 @@ fn reconcile_evidence_digest( hasher.update(b"nokv.publish.reconcile-evidence.v1\0"); hasher.update([match resolution { QuarantineReconcileResolution::RevisionUnpublished => 1, - QuarantineReconcileResolution::RevisionUnpublishedSealed => 3, QuarantineReconcileResolution::RevisionPublished => 2, }]); match original_evidence_digest { @@ -2899,23 +2931,14 @@ impl PublicationService<'_> { let revision_key = artifact_revision_key(context.root_id, operation.artifact_revision_id); let payload = self.read_payload(context, MetadataFamily::ArtifactRevision, &revision_key)?; - if operation.append_attempt.is_some() - != matches!( - resolution, - QuarantineReconcileResolution::RevisionUnpublishedSealed - ) - { + if operation.append_attempt.is_some() { return Err(PublicationError::ReconcileResolutionMismatch { resolution, revision_published: payload.is_some(), }); } match (resolution, payload) { - ( - QuarantineReconcileResolution::RevisionUnpublished - | QuarantineReconcileResolution::RevisionUnpublishedSealed, - None, - ) => { + (QuarantineReconcileResolution::RevisionUnpublished, None) => { plan.assert_value(MetadataFamily::ArtifactRevision, revision_key, None)?; } (QuarantineReconcileResolution::RevisionPublished, Some(payload)) => { @@ -3453,7 +3476,171 @@ impl PublicationService<'_> { seal: "append revision reservation", }); } - plan.assert_value(MetadataFamily::ArtifactRevision, key, Some(payload)) + if !plan + .exact_keys + .contains(&(MetadataFamily::ArtifactRevision, key.clone())) + { + plan.assert_value(MetadataFamily::ArtifactRevision, key, Some(payload))?; + } + Ok(()) + } + + /// Require permanent ownership and absence of a published revision before + /// abandoned append cleanup. Live workspace bindings are intentionally irrelevant. + fn predicate_append_cleanup_reservation( + &self, + context: PublicationContext, + operation: &PublishOperationRecord, + plan: &mut CommandPlan, + ) -> Result<(), PublicationError> { + self.predicate_append_revision_reservation(context, operation, plan)?; + let key = artifact_revision_key(context.root_id, operation.artifact_revision_id); + if self + .read_payload(context, MetadataFamily::ArtifactRevision, &key)? + .is_some() + { + return Err(PublicationError::InvalidOperationSeal { + seal: "append cleanup requires an unpublished revision", + }); + } + if !plan + .exact_keys + .contains(&(MetadataFamily::ArtifactRevision, key.clone())) + { + plan.assert_value(MetadataFamily::ArtifactRevision, key, None)?; + } + Ok(()) + } + + /// Read-only authority check immediately before an owner seals an abandoned + /// append object. No provider outcome supplied by a caller can bypass this gate. + pub fn validate_append_cleanup_authority( + &self, + context: PublicationContext, + operation: &PublishOperationRecord, + ) -> Result<(), PublicationError> { + validate_operation_seals(operation)?; + require_operation_phase(operation, PublishPhase::Cleaning)?; + if operation.append_attempt.is_none() { + return Err(PublicationError::OperationInputMismatch); + } + self.require_current_operation(context, operation)?; + let mut plan = CommandPlan::default(); + self.predicate_append_authority(context, operation, &mut plan)?; + self.predicate_append_cleanup_reservation(context, operation, &mut plan) + } + + /// Atomically accept one exact quarantined state for owner cleanup. The + /// deterministic request identity and full-token receipt survive ACK loss, + /// owner restart, later failure, and parent advancement without an ABA retry. + pub fn retry_append_cleanup( + &self, + mut context: PublicationContext, + operation_id: OperationId, + expected_state_digest: [u8; SHA256_BYTES], + ) -> Result { + context.request_id = + append_cleanup_retry_request_id(context.root_id, operation_id, expected_state_digest); + if let Some(replay) = self.store.lookup_request_result( + context.root_id, + context.placement_generation, + context.owner_epoch, + context.request_id, + )? { + let receipt = AppendCleanupRetryReceipt::decode(&replay.deterministic_result)?; + if receipt.operation_id != operation_id + || receipt.expected_state_digest != expected_state_digest + { + return Err(PublicationError::OperationInputMismatch); + } + return Ok(AppendCleanupRetryOutcome { + receipt, + commit_version: replay.commit_version, + replayed: true, + }); + } + let parent_payload = self + .read_payload( + context, + MetadataFamily::Operation, + &operation_key(context.root_id, OperationKind::Append, operation_id), + )? + .ok_or(PublicationError::OperationInputMismatch)?; + let parent = AppendOperationRecord::decode(&parent_payload)?; + if parent.operation_id != operation_id { + return Err(PublicationError::OperationInputMismatch); + } + let child_key = operation_key( + context.root_id, + OperationKind::Publish, + parent.publication_operation_id, + ); + let child_payload = self + .read_payload(context, MetadataFamily::Operation, &child_key)? + .ok_or(PublicationError::OperationInputMismatch)?; + let child = PublishOperationRecord::decode(&child_payload)?; + validate_operation_seals(&child)?; + if !parent.matches_publication(&child) + || parent.result.is_some() + || append_operation_state_digest(&parent, &child)? != expected_state_digest + { + return Err(PublicationError::ConcurrentMutation); + } + require_operation_phase(&child, PublishPhase::Quarantined)?; + let mut next = child.clone(); + next.retry_append_cleanup()?; + let receipt = AppendCleanupRetryReceipt { + operation_id, + publication_operation_id: child.operation_id, + expected_state_digest, + cleanup_retry_count: next.cleanup_retry_count, + original_failure: child + .terminal_error + .clone() + .expect("validated quarantine requires failure"), + }; + let mut plan = CommandPlan::default(); + plan.replace( + MetadataFamily::Operation, + child_key, + child_payload, + next.encode()?, + )?; + let mut command = self.seal_plan( + context, + plan, + next.encode()?, + PublicationAuthorityPurpose::Cleanup, + )?; + command.deterministic_result = receipt.encode()?; + let result = match self.store.execute(&command.seal()) { + Ok(result) => result, + Err(error) => { + // A concurrent identical token may commit a command planned at + // another read version. Resolve its closed recovery receipt + // before surfacing a low-level command digest mismatch. + match self.store.lookup_request_result( + context.root_id, + context.placement_generation, + context.owner_epoch, + context.request_id, + )? { + Some(replay) => replay, + None => return Err(error.into()), + } + } + }; + let accepted = AppendCleanupRetryReceipt::decode(&result.deterministic_result)?; + if accepted.operation_id != operation_id + || accepted.expected_state_digest != expected_state_digest + { + return Err(PublicationError::OperationInputMismatch); + } + Ok(AppendCleanupRetryOutcome { + receipt: accepted, + commit_version: result.commit_version, + replayed: result.replayed, + }) } /// Check the complete publication index and logical-parent graph at one @@ -3857,7 +4044,18 @@ impl PublicationService<'_> { let operation = PublishOperationRecord::decode(&deterministic_result)?; self.predicate_append_authority(context, &operation, &mut plan)?; self.synchronize_active_publication(context, &operation, &mut plan)?; - self.predicate_publication_authority(context, &operation, authority_purpose, &mut plan)?; + if operation.append_attempt.is_some() + && authority_purpose == PublicationAuthorityPurpose::Cleanup + { + self.predicate_append_cleanup_reservation(context, &operation, &mut plan)?; + } else { + self.predicate_publication_authority( + context, + &operation, + authority_purpose, + &mut plan, + )?; + } plan.validate_bounds()?; Ok(MetadataCommand { schema_id: SCHEMA_ID.to_owned(), @@ -5099,6 +5297,7 @@ mod tests { dependency_digest: dependency_owner_digest(&dependencies).unwrap(), cleanup_staged_object_cursor: 0, cleanup_manifest_cursor: 0, + cleanup_retry_count: 0, publication_absence_proof: None, result: None, terminal_error: None, @@ -6085,39 +6284,32 @@ mod tests { Err(PublicationError::ReconcileResolutionMismatch { .. }) )); assert_eq!(store.current_read_version().unwrap(), before); - child = service - .reconcile_quarantined_publish_batch(ReconcileQuarantinedPublishBatchRequest { - context: publication_context(&store, &mut counter), - expected_operation: child, - resolution: QuarantineReconcileResolution::RevisionUnpublishedSealed, - staged_object_rows: staged.clone(), - }) - .unwrap() - .operation; - assert!(service - .finish_reconcile_quarantined_publish( - FinishReconcileQuarantinedPublishRequest { - context: publication_context(&store, &mut counter), - expected_operation: child.clone(), - resolution: QuarantineReconcileResolution::RevisionUnpublished, - reason: "absent is insufficient".to_owned(), - operator_evidence_digest: [0xc2; 32], - } - ) - .is_err()); - child = service - .finish_reconcile_quarantined_publish( - FinishReconcileQuarantinedPublishRequest { - context: publication_context(&store, &mut counter), - expected_operation: child, - resolution: QuarantineReconcileResolution::RevisionUnpublishedSealed, - reason: "each immutable key is permanently sealed".to_owned(), - operator_evidence_digest: [0xc3; 32], - }, + let parent = read_append_parent(&service, &store, &mut counter); + let token = append_operation_state_digest(&parent, &child).unwrap(); + let receipt = service + .retry_append_cleanup( + publication_context(&store, &mut counter), + parent.operation_id, + token, ) .unwrap() - .operation; - } else { + .receipt; + assert_eq!(receipt.cleanup_retry_count, 1); + child = PublishOperationRecord::decode( + &service + .read_payload( + publication_context(&store, &mut counter), + MetadataFamily::Operation, + &operation_key(root(), OperationKind::Publish, child.operation_id), + ) + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!(child.cleanup_retry_count, 1); + assert_eq!(child.phase, PublishPhase::Cleaning); + } + { let mut next = staged[0].clone(); next.provider_state = StagedProviderState::Aborted; next.cleanup_state = StagedCleanupState::Deleted; @@ -6198,6 +6390,457 @@ mod tests { } } + fn quarantined_append_fixture( + store: &MetaShard, + counter: &mut u128, + count: usize, + cleaned: usize, + ) -> (PublishOperationRecord, Vec) { + let service = PublicationService::new(store); + let template = logical_append_child(0); + let staged = staged_rows(template.artifact_revision_id, count); + let mut initial = publish_operation( + template.operation_id, + template.artifact_revision_id, + template.path, + PublishClaim::CreateOnly, + &staged, + &manifest_rows(&staged), + ); + initial.append_attempt = template.append_attempt; + initial.append_intent_digest = template.append_intent_digest; + seal_publish_operation(&mut initial); + let child = begin_operation(&service, store, counter, initial); + let mut child = service + .stage_objects_batch(StageObjectsBatchRequest { + context: publication_context(store, counter), + expected_operation: child, + staged_objects: staged.clone(), + }) + .unwrap() + .operation; + for transition in [ + PublishTransition::BeginAbort { + terminal_error: PublishTerminalError { + kind: PublishTerminalErrorKind::ActivityLeaseExpired, + message: "abandoned caller".to_owned(), + evidence_digest: None, + }, + }, + PublishTransition::BeginCleaning, + ] { + child = service + .transition_publish(TransitionPublishRequest { + context: publication_context(store, counter), + expected_operation: child, + transition, + }) + .unwrap() + .operation; + } + if cleaned != 0 { + child = service + .cleanup_publish_batch(CleanupPublishBatchRequest { + context: publication_context(store, counter), + expected_operation: child, + staged_object_updates: staged[..cleaned] + .iter() + .map(|expected| { + let mut next = expected.clone(); + next.provider_state = StagedProviderState::Aborted; + next.cleanup_state = StagedCleanupState::Sealed; + StagedObjectUpdate { + expected: expected.clone(), + next, + } + }) + .collect(), + }) + .unwrap() + .operation; + } + let child = service + .transition_publish(TransitionPublishRequest { + context: publication_context(store, counter), + expected_operation: child, + transition: PublishTransition::Quarantine { + terminal_error: PublishTerminalError { + kind: PublishTerminalErrorKind::CleanupFailed, + message: "seal ACK and HEAD unavailable".to_owned(), + evidence_digest: Some([0xc1; 32]), + }, + }, + }) + .unwrap() + .operation; + (child, staged) + } + + fn read_append_child_at( + service: &PublicationService<'_>, + context: PublicationContext, + id: OperationId, + ) -> PublishOperationRecord { + PublishOperationRecord::decode( + &service + .read_payload( + context, + MetadataFamily::Operation, + &operation_key(context.root_id, OperationKind::Publish, id), + ) + .unwrap() + .unwrap(), + ) + .unwrap() + } + + #[test] + fn append_cleanup_retry_receipt_survives_aba_partial_progress_owner_restart_and_successor() { + let directory = tempdir().unwrap(); + let path = directory.path().join("append-cleanup-retry"); + let mut counter = 1; + let store = ready_file_store(&path, &mut counter); + let service = PublicationService::new(&store); + let (child, staged) = quarantined_append_fixture(&store, &mut counter, 2, 1); + let parent = read_append_parent(&service, &store, &mut counter); + let token = append_operation_state_digest(&parent, &child).unwrap(); + let failure = child.terminal_error.clone().unwrap(); + let first = service + .retry_append_cleanup( + publication_context(&store, &mut counter), + parent.operation_id, + token, + ) + .unwrap() + .receipt; + assert_eq!(first.original_failure, failure); + assert_eq!(first.cleanup_retry_count, 1); + let next = read_append_child_at( + &service, + publication_context(&store, &mut counter), + child.operation_id, + ); + assert_eq!(next.cleanup_staged_object_cursor, 1); + assert_eq!(next.cleanup_manifest_cursor, child.cleanup_manifest_cursor); + assert_eq!(next.terminal_error, child.terminal_error); + assert_eq!(read_append_parent(&service, &store, &mut counter), parent); + let repeated_failure = service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: next, + transition: PublishTransition::Quarantine { + terminal_error: failure, + }, + }) + .unwrap() + .operation; + let newer_token = append_operation_state_digest(&parent, &repeated_failure).unwrap(); + assert_ne!( + newer_token, token, + "same error and cursor must not recreate the original token" + ); + let version = store.current_read_version().unwrap(); + assert_eq!( + service + .retry_append_cleanup( + publication_context(&store, &mut counter), + parent.operation_id, + token + ) + .unwrap() + .receipt, + first + ); + assert_eq!( + store.current_read_version().unwrap(), + version, + "old request cannot enqueue a new retry" + ); + let second = service + .retry_append_cleanup( + publication_context(&store, &mut counter), + parent.operation_id, + newer_token, + ) + .unwrap() + .receipt; + assert_eq!(second.cleanup_retry_count, 2); + drop(store); + + let store = crate::workspace::test_support::open_file(&path, shard()).unwrap(); + store + .advance_owner_epoch(Some(owner()), successor_owner()) + .unwrap(); + let service = PublicationService::new(&store); + let version = store.current_read_version().unwrap(); + assert_eq!( + service + .retry_append_cleanup( + publication_context_for_owner(&store, &mut counter, successor_owner()), + parent.operation_id, + token + ) + .unwrap() + .receipt, + first + ); + assert_eq!(store.current_read_version().unwrap(), version); + assert!( + service + .retry_append_cleanup( + publication_context(&store, &mut counter), + parent.operation_id, + token + ) + .is_err(), + "old owner cannot replay at a new fence" + ); + let child = read_append_child_at( + &service, + publication_context_for_owner(&store, &mut counter, successor_owner()), + child.operation_id, + ); + let mut sealed = staged[1].clone(); + sealed.provider_state = StagedProviderState::Aborted; + sealed.cleanup_state = StagedCleanupState::Sealed; + let child = service + .cleanup_publish_batch(CleanupPublishBatchRequest { + context: publication_context_for_owner(&store, &mut counter, successor_owner()), + expected_operation: child, + staged_object_updates: vec![StagedObjectUpdate { + expected: staged[1].clone(), + next: sealed, + }], + }) + .unwrap() + .operation; + let cleaned = service + .transition_publish(TransitionPublishRequest { + context: publication_context_for_owner(&store, &mut counter, successor_owner()), + expected_operation: child, + transition: PublishTransition::FinishCleanup, + }) + .unwrap() + .operation; + assert_eq!(cleaned.cleanup_staged_object_cursor, 2); + assert_eq!(cleaned.cleanup_retry_count, 2); + let mut successor = logical_append_child(1); + successor.initiating_owner_epoch = successor_owner(); + seal_publish_operation(&mut successor); + service + .begin_publish(BeginPublishRequest { + context: publication_context_for_owner(&store, &mut counter, successor_owner()), + operation: successor, + }) + .unwrap(); + assert_eq!( + service + .retry_append_cleanup( + publication_context_for_owner(&store, &mut counter, successor_owner()), + parent.operation_id, + token + ) + .unwrap() + .receipt, + first + ); + assert_eq!( + service + .retry_append_cleanup( + publication_context_for_owner(&store, &mut counter, successor_owner()), + parent.operation_id, + newer_token + ) + .unwrap() + .receipt, + second + ); + } + + #[test] + fn append_cleanup_retry_refuses_missing_foreign_or_published_ownership_without_mutation() { + for damage in 0..5 { + let mut counter = 1; + let store = ready_store(&mut counter); + let service = PublicationService::new(&store); + let (child, _) = quarantined_append_fixture(&store, &mut counter, 1, 0); + let parent = read_append_parent(&service, &store, &mut counter); + let token = append_operation_state_digest(&parent, &child).unwrap(); + let context = publication_context(&store, &mut counter); + let mut plan = CommandPlan::default(); + match damage { + 0 | 1 => { + let key = artifact_revision_claim_key(root(), child.artifact_revision_id); + let old = service + .read_payload(context, MetadataFamily::ArtifactRevision, &key) + .unwrap() + .unwrap(); + if damage == 0 { + plan.delete(MetadataFamily::ArtifactRevision, key, old) + .unwrap(); + } else { + plan.replace( + MetadataFamily::ArtifactRevision, + key, + old, + ArtifactRevisionClaimRecord { + operation_id: operation_id(999_999), + } + .encode() + .unwrap(), + ) + .unwrap(); + } + } + 2 => { + // Any revision value, including unknown metadata, must prevent sealing. + plan.put_absent( + MetadataFamily::ArtifactRevision, + artifact_revision_key(root(), child.artifact_revision_id), + vec![0xff], + ) + .unwrap(); + } + 3 | 4 => { + let kind = if damage == 3 { + OperationKind::Append + } else { + OperationKind::Publish + }; + let id = if damage == 3 { + parent.operation_id + } else { + child.operation_id + }; + let key = operation_key(root(), kind, id); + let old = service + .read_payload(context, MetadataFamily::Operation, &key) + .unwrap() + .unwrap(); + plan.delete(MetadataFamily::Operation, key, old).unwrap(); + } + _ => unreachable!(), + } + let mut command = + fence_command(&store, context.request_id, RootFenceAction::RequireActive); + command.predicates = plan.predicates; + command.mutations = plan.mutations; + command.history_projection = plan.history; + store.execute(&command.seal()).unwrap(); + let before = store.current_read_version().unwrap(); + assert!(service + .retry_append_cleanup( + publication_context(&store, &mut counter), + parent.operation_id, + token + ) + .is_err()); + assert_eq!(store.current_read_version().unwrap(), before); + assert!(service + .read_payload( + publication_context(&store, &mut counter), + MetadataFamily::Operation, + &operation_key(root(), OperationKind::ActivePublish, child.operation_id) + ) + .unwrap() + .is_none()); + } + } + + #[test] + fn append_cleanup_retry_ignores_retired_workspace_but_preserves_exact_cleanup_authority() { + for rebound in [false, true] { + let mut counter = 1; + let store = ready_store(&mut counter); + let service = PublicationService::new(&store); + let (child, staged) = quarantined_append_fixture(&store, &mut counter, 1, 0); + let parent = read_append_parent(&service, &store, &mut counter); + let token = append_operation_state_digest(&parent, &child).unwrap(); + let key = workspace_current_key(root(), &workbench()); + let context = publication_context(&store, &mut counter); + let old = service + .read_payload(context, MetadataFamily::WorkspaceCurrent, &key) + .unwrap() + .unwrap(); + let replacement = if rebound { + let mut workspace = WorkspaceRecord::decode(&old).unwrap(); + workspace.incarnation_id = incarnation(900); + Some(workspace.encode().unwrap()) + } else { + None + }; + let mut plan = CommandPlan::default(); + match &replacement { + Some(new) => plan + .replace( + MetadataFamily::WorkspaceCurrent, + key.clone(), + old, + new.clone(), + ) + .unwrap(), + None => plan + .delete(MetadataFamily::WorkspaceCurrent, key.clone(), old) + .unwrap(), + } + let mut command = + fence_command(&store, context.request_id, RootFenceAction::RequireActive); + command.predicates = plan.predicates; + command.mutations = plan.mutations; + command.history_projection = plan.history; + store.execute(&command.seal()).unwrap(); + service + .retry_append_cleanup( + publication_context(&store, &mut counter), + parent.operation_id, + token, + ) + .unwrap(); + let child = read_append_child_at( + &service, + publication_context(&store, &mut counter), + child.operation_id, + ); + service + .validate_append_cleanup_authority( + publication_context(&store, &mut counter), + &child, + ) + .unwrap(); + let mut sealed = staged[0].clone(); + sealed.provider_state = StagedProviderState::Aborted; + sealed.cleanup_state = StagedCleanupState::Sealed; + let child = service + .cleanup_publish_batch(CleanupPublishBatchRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + staged_object_updates: vec![StagedObjectUpdate { + expected: staged[0].clone(), + next: sealed, + }], + }) + .unwrap() + .operation; + service + .transition_publish(TransitionPublishRequest { + context: publication_context(&store, &mut counter), + expected_operation: child, + transition: PublishTransition::FinishCleanup, + }) + .unwrap(); + assert_eq!( + service + .read_payload( + publication_context(&store, &mut counter), + MetadataFamily::WorkspaceCurrent, + &key + ) + .unwrap(), + replacement + ); + } + } + #[test] fn append_parent_and_child_admit_and_advance_atomically_without_empty_attempts() { let mut counter = 1; @@ -7980,7 +8623,10 @@ mod tests { assert_eq!(replaced.result.path_generation, Generation::new(2).unwrap()); let replace_bytes = capture.with_last_commit(crate::workspace::test_support::transaction_bytes); - assert_eq!(replace_bytes, 11_799_783); + // The 8-byte retry epoch occurs in the operation check, mutation, + // history value and dedupe result, plus their recovery-command copies. + assert_eq!(replace_bytes, 11_799_847); + assert!(replace_bytes < super::super::store_limits().max_transaction_bytes); let removed = remove_path( &store, @@ -8241,7 +8887,7 @@ mod tests { assert_eq!( capture.with_last_commit(crate::workspace::test_support::transaction_bytes), - 9_861_056 + 9_861_120 ); } diff --git a/crates/nokv-meta/src/workspace/publish_operation_records.rs b/crates/nokv-meta/src/workspace/publish_operation_records.rs index e3b5fc00a..624daac0d 100644 --- a/crates/nokv-meta/src/workspace/publish_operation_records.rs +++ b/crates/nokv-meta/src/workspace/publish_operation_records.rs @@ -19,7 +19,7 @@ use nokv_types::{ }; /// Durable value format for publication-owned payloads. -pub const PUBLISH_VALUE_FORMAT_VERSION: u8 = 6; +pub const PUBLISH_VALUE_FORMAT_VERSION: u8 = 7; /// Hard safety bound for one publish operation's staged-object ledger. pub const MAX_STAGED_OBJECTS: u32 = 1_048_576; @@ -188,6 +188,8 @@ pub struct PublishOperationRecord { /// Number of staged-object and manifest rows durably removed by cleanup. pub cleanup_staged_object_cursor: u32, pub cleanup_manifest_cursor: u32, + /// Accepted owner cleanup retries; monotonic even when the same failure recurs. + pub cleanup_retry_count: u64, /// Required only when cleanup takes over a `Finalizing` operation. pub publication_absence_proof: Option<[u8; SHA256_BYTES]>, @@ -633,6 +635,18 @@ impl PublishOperationRecord { self.cleanup_manifest_cursor, self.manifest_cursor, )?; + if self.cleanup_retry_count != 0 + && (self.append_attempt.is_none() + || matches!( + self.phase, + PublishPhase::Uploading | PublishPhase::Finalizing | PublishPhase::Published + )) + { + return Err(PublishRecordError::InvalidPhasePayload { + phase: self.phase, + reason: "cleanup retries belong only to failed logical append attempts", + }); + } validate_progress_digest( "staged_object", self.staged_object_count, @@ -884,6 +898,7 @@ impl PublishOperationRecord { encoded.extend_from_slice(&self.dependency_digest); encoded.extend_from_slice(&self.cleanup_staged_object_cursor.to_be_bytes()); encoded.extend_from_slice(&self.cleanup_manifest_cursor.to_be_bytes()); + encoded.extend_from_slice(&self.cleanup_retry_count.to_be_bytes()); push_optional_fixed(&mut encoded, &self.publication_absence_proof); push_optional_result(&mut encoded, self.result.as_ref())?; push_optional_terminal_error(&mut encoded, self.terminal_error.as_ref())?; @@ -997,6 +1012,7 @@ impl PublishOperationRecord { let dependency_digest = decoder.fixed("dependency_digest")?; let cleanup_staged_object_cursor = decoder.u32("cleanup_staged_object_cursor")?; let cleanup_manifest_cursor = decoder.u32("cleanup_manifest_cursor")?; + let cleanup_retry_count = decoder.u64("cleanup_retry_count")?; let publication_absence_proof = decoder.optional_fixed("publication_absence_proof")?; let result = decode_optional_result(&mut decoder)?; let terminal_error = decode_optional_terminal_error(&mut decoder)?; @@ -1033,6 +1049,7 @@ impl PublishOperationRecord { dependency_digest, cleanup_staged_object_cursor, cleanup_manifest_cursor, + cleanup_retry_count, publication_absence_proof, result, terminal_error, @@ -1147,6 +1164,33 @@ impl PublishOperationRecord { Ok(()) } + pub(super) fn retry_append_cleanup(&mut self) -> Result<(), PublishRecordError> { + self.validate()?; + if self.phase != PublishPhase::Quarantined { + return Err(PublishRecordError::PhaseMismatch { + expected: PublishPhase::Quarantined, + actual: self.phase, + }); + } + if self.append_attempt.is_none() { + return Err(PublishRecordError::InvalidPhasePayload { + phase: self.phase, + reason: "owner cleanup retry requires a logical append child", + }); + } + let mut next = self.clone(); + next.cleanup_retry_count = next.cleanup_retry_count.checked_add(1).ok_or( + PublishRecordError::InvalidPhasePayload { + phase: self.phase, + reason: "cleanup retry count exhausted", + }, + )?; + next.phase = PublishPhase::Cleaning; + next.validate()?; + *self = next; + Ok(()) + } + /// Applies the operator-owned `Quarantined -> Cleaned` reconciliation /// edge. Every durable staging row must already be removed, and the new /// terminal error must retain reconciliation evidence so the operator @@ -1928,6 +1972,64 @@ impl<'a> Decoder<'a> { } } +/// Durable acceptance receipt for one exact logical append cleanup state. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AppendCleanupRetryReceipt { + pub operation_id: OperationId, + pub publication_operation_id: OperationId, + pub expected_state_digest: [u8; SHA256_BYTES], + pub cleanup_retry_count: u64, + pub original_failure: PublishTerminalError, +} + +impl AppendCleanupRetryReceipt { + pub fn encode(&self) -> Result, PublishRecordError> { + validate_terminal_error(&self.original_failure)?; + if self.cleanup_retry_count == 0 + || self.operation_id == self.publication_operation_id + || self.original_failure.evidence_digest.is_none() + { + return Err(PublishRecordError::InvalidPhasePayload { phase: PublishPhase::Quarantined, + reason: "cleanup retry receipt requires distinct identities, positive count, and original quarantine evidence" }); + } + let mut bytes = b"nokv.append.cleanup-retry.v1\0".to_vec(); + bytes.extend_from_slice(self.operation_id.as_bytes()); + bytes.extend_from_slice(self.publication_operation_id.as_bytes()); + bytes.extend_from_slice(&self.expected_state_digest); + bytes.extend_from_slice(&self.cleanup_retry_count.to_be_bytes()); + push_optional_terminal_error(&mut bytes, Some(&self.original_failure))?; + Ok(bytes) + } + + pub fn decode(bytes: &[u8]) -> Result { + let prefix = b"nokv.append.cleanup-retry.v1\0"; + let mut decoder = Decoder::new(bytes); + if decoder.take("cleanup retry receipt tag", prefix.len())? != prefix { + return Err(PublishRecordError::InvalidPhasePayload { + phase: PublishPhase::Quarantined, + reason: "invalid cleanup retry receipt tag", + }); + } + let receipt = Self { + operation_id: OperationId::from_bytes(decoder.fixed("operation_id")?), + publication_operation_id: OperationId::from_bytes( + decoder.fixed("publication_operation_id")?, + ), + expected_state_digest: decoder.fixed("expected_state_digest")?, + cleanup_retry_count: decoder.u64("cleanup_retry_count")?, + original_failure: decode_optional_terminal_error(&mut decoder)?.ok_or( + PublishRecordError::InvalidPhasePayload { + phase: PublishPhase::Quarantined, + reason: "cleanup retry receipt is missing original failure", + }, + )?, + }; + decoder.finish()?; + receipt.encode()?; + Ok(receipt) + } +} + #[cfg(test)] mod tests { use super::*; @@ -2003,6 +2105,7 @@ mod tests { dependency_digest: [0x18; SHA256_BYTES], cleanup_staged_object_cursor: 0, cleanup_manifest_cursor: 0, + cleanup_retry_count: 0, publication_absence_proof: None, result: None, terminal_error: None, @@ -2090,6 +2193,7 @@ mod tests { &[0x18; SHA256_BYTES], &0_u32.to_be_bytes(), &0_u32.to_be_bytes(), + &0_u64.to_be_bytes(), &[0], &[1], &8_u64.to_be_bytes(), diff --git a/crates/nokv-meta/src/workspace/restore.rs b/crates/nokv-meta/src/workspace/restore.rs index fc353f0ed..a1f4c101f 100644 --- a/crates/nokv-meta/src/workspace/restore.rs +++ b/crates/nokv-meta/src/workspace/restore.rs @@ -10799,6 +10799,7 @@ mod tests { dependency_digest: dependency_owner_digest(&[]).unwrap(), cleanup_staged_object_cursor: 0, cleanup_manifest_cursor: 0, + cleanup_retry_count: 0, publication_absence_proof: None, result: None, terminal_error: None, diff --git a/crates/nokv-protocol/src/append.rs b/crates/nokv-protocol/src/append.rs index 60763aca9..1618a4ef0 100644 --- a/crates/nokv-protocol/src/append.rs +++ b/crates/nokv-protocol/src/append.rs @@ -43,11 +43,26 @@ pub struct AppendPreparation { pub activity_deadline_ms: u64, /// Failure of the current publication attempt, independent of the logical outcome. pub attempt_failure: Option, + /// Number of owner cleanup retries accepted for this publication attempt. + pub cleanup_retry_count: u64, } impl AppendPreparation { pub(crate) fn validate(&self) -> Result<(), ProtocolError> { crate::types::require_generation("append.activity_deadline_ms", self.activity_deadline_ms)?; + if self.cleanup_retry_count != 0 + && matches!( + self.attempt_phase, + AppendAttemptPhase::Uploading + | AppendAttemptPhase::Finalizing + | AppendAttemptPhase::Published + ) + { + return Err(ProtocolError::invalid( + "append.cleanup_retry_count", + "must be zero before an append attempt enters cleanup", + )); + } if let Some(failure) = &self.attempt_failure { failure.validate()?; if matches!( @@ -104,6 +119,122 @@ impl AppendResult { } } +/// Read the actual remaining staged ledger at one exact logical-operation state. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct InspectAppendCleanupRequest { + pub token: crate::OperationToken, + pub start_after: Option, + pub limit: u32, +} + +impl InspectAppendCleanupRequest { + pub(crate) fn validate(&self) -> Result<(), ProtocolError> { + if !(1..=crate::MAX_ARTIFACT_PUBLISH_BATCH_ROWS as u32).contains(&self.limit) { + return Err(ProtocolError::invalid( + "append_cleanup.limit", + "must be between 1 and 192", + )); + } + Ok(()) + } +} + +/// Requeue the quarantined current child. Repeating this exact token returns +/// the original acceptance receipt even after subsequent cleanup progress. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RetryAppendCleanupRequest { + pub token: crate::OperationToken, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AppendCleanupInspection { + pub operation: Box, + pub object_namespace_id: crate::ObjectNamespaceIdentity, + pub publication_token: crate::OperationToken, + /// Registered entries, excluding planned objects never admitted to the ledger. + pub registered_count: u32, + pub cleanup_cursor: u32, + pub remaining_count: u32, + pub entries: Vec, + pub next_after: Option, +} + +impl AppendCleanupInspection { + pub(crate) fn validate(&self) -> Result<(), ProtocolError> { + self.operation.validate()?; + let preparation = self.operation.append_preparation.as_ref().ok_or_else(|| { + ProtocolError::invalid("append_cleanup.operation", "must describe a logical append") + })?; + if self.publication_token.operation_id != preparation.publication_operation_id + || self.registered_count.checked_sub(self.cleanup_cursor) != Some(self.remaining_count) + || self.entries.len() > crate::MAX_ARTIFACT_PUBLISH_BATCH_ROWS + { + return Err(ProtocolError::invalid( + "append_cleanup", + "inconsistent ledger bounds or child identity", + )); + } + let mut previous = None; + for entry in &self.entries { + entry.validate()?; + if entry.sequence < self.cleanup_cursor + || entry.sequence >= self.registered_count + || previous + .is_some_and(|sequence: u32| sequence.checked_add(1) != Some(entry.sequence)) + { + return Err(ProtocolError::invalid( + "append_cleanup.entries", + "must be a contiguous remaining ledger page", + )); + } + previous = Some(entry.sequence); + } + if let Some(next) = self.next_after { + if previous != Some(next) + || next + .checked_add(1) + .is_none_or(|n| n >= self.registered_count) + { + return Err(ProtocolError::invalid( + "append_cleanup.next_after", + "must identify the last entry of a non-final page", + )); + } + } else if previous.is_some_and(|last| last + 1 != self.registered_count) { + return Err(ProtocolError::invalid( + "append_cleanup.entries", + "a non-final page requires a continuation", + )); + } + Ok(()) + } +} + +/// Durable acceptance of one cleanup retry, independent of its eventual outcome. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct AppendCleanupRetryResult { + pub operation_id: OperationIdentity, + pub publication_operation_id: OperationIdentity, + pub cleanup_retry_count: u64, + pub expected_state_digest: Digest, +} + +impl AppendCleanupRetryResult { + pub(crate) fn validate(&self) -> Result<(), ProtocolError> { + if self.operation_id == self.publication_operation_id || self.cleanup_retry_count == 0 { + return Err(ProtocolError::invalid( + "append_cleanup_retry", + "requires distinct identities and a positive retry count", + )); + } + Ok(()) + } +} + /// Derive distinct immutable identities for a logical append's numbered attempt. pub fn stable_append_attempt_identities( root: RootIdentity, @@ -131,6 +262,53 @@ pub fn stable_append_attempt_identities( mod tests { use super::*; + #[test] + fn append_cleanup_wire_is_bounded_and_removed_operator_verdict_is_not_decodable() { + let token = crate::OperationToken { + operation_id: OperationIdentity([1; 16]), + state_digest: Digest([2; 32]), + }; + for limit in [0, 193, u32::MAX] { + assert!(InspectAppendCleanupRequest { + token, + start_after: None, + limit + } + .validate() + .is_err()); + } + for limit in [1, 32, 192] { + let request = InspectAppendCleanupRequest { + token, + start_after: Some(17), + limit, + }; + request.validate().unwrap(); + let encoded = rmp_serde::to_vec_named(&request).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + request + ); + } + let receipt = AppendCleanupRetryResult { + operation_id: token.operation_id, + publication_operation_id: OperationIdentity([3; 16]), + cleanup_retry_count: 1, + expected_state_digest: token.state_digest, + }; + receipt.validate().unwrap(); + let encoded = rmp_serde::to_vec_named(&receipt).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + receipt + ); + let removed = rmp_serde::to_vec_named("provider_objects_sealed").unwrap(); + let error = rmp_serde::from_slice::(&removed).unwrap_err(); + assert!(error + .to_string() + .contains("unknown variant `provider_objects_sealed`")); + } + #[test] fn attempt_failure_is_separate_from_logical_outcome_and_validated_by_phase() { let mut preparation = AppendPreparation { @@ -146,8 +324,19 @@ mod tests { attempt_phase: AppendAttemptPhase::Uploading, activity_deadline_ms: 1_000, attempt_failure: None, + cleanup_retry_count: 0, }; assert!(preparation.validate().is_ok()); + preparation.cleanup_retry_count = 1; + for phase in [ + AppendAttemptPhase::Uploading, + AppendAttemptPhase::Finalizing, + AppendAttemptPhase::Published, + ] { + preparation.attempt_phase = phase; + assert!(preparation.validate().is_err()); + } + preparation.cleanup_retry_count = 0; preparation.attempt_phase = AppendAttemptPhase::Cleaned; assert!(preparation.validate().is_err()); preparation.attempt_failure = Some(RpcFailure { @@ -165,8 +354,12 @@ mod tests { AppendAttemptPhase::Quarantined, ] { preparation.attempt_phase = phase; - assert!(preparation.validate().is_ok()); + for count in [0, 1, u64::MAX] { + preparation.cleanup_retry_count = count; + assert!(preparation.validate().is_ok()); + } } + preparation.cleanup_retry_count = 0; preparation.attempt_phase = AppendAttemptPhase::Published; assert!(preparation.validate().is_err()); preparation.attempt_phase = AppendAttemptPhase::Cleaned; diff --git a/crates/nokv-protocol/src/artifact_publish.rs b/crates/nokv-protocol/src/artifact_publish.rs index be8017840..23d445a35 100644 --- a/crates/nokv-protocol/src/artifact_publish.rs +++ b/crates/nokv-protocol/src/artifact_publish.rs @@ -16,7 +16,7 @@ pub const MAX_ARTIFACT_PUBLISH_OBJECTS: usize = 1_048_576; /// Maximum provider-neutral manifest rows returned by one range-plan page. pub const MAX_ARTIFACT_READ_PLAN_ROWS: usize = 512; -const PUBLICATION_RECORD_ENCODING: u8 = 6; +const PUBLICATION_RECORD_ENCODING: u8 = 7; /// Canonical closure committed by `BeginArtifactPublish`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/nokv-protocol/src/codec.rs b/crates/nokv-protocol/src/codec.rs index f9c6ad3c2..1d698ad73 100644 --- a/crates/nokv-protocol/src/codec.rs +++ b/crates/nokv-protocol/src/codec.rs @@ -10,7 +10,7 @@ use crate::request::WorkspaceRpcRequest; use crate::response::WorkspaceRpcResponse; /// The exact and only accepted wire schema. -pub const WORKSPACE_PROTOCOL_SCHEMA: &str = "nokv.workspace.rpc.v11"; +pub const WORKSPACE_PROTOCOL_SCHEMA: &str = "nokv.workspace.rpc.v12"; /// Exact schema for the versioned workspace RPC preflight exchange. pub const WORKSPACE_PREFLIGHT_SCHEMA: &str = "nokv.workspace.rpc_preflight.v1"; /// Exact schema for the advertised workspace RPC capability set. @@ -273,7 +273,7 @@ mod tests { #[test] fn request_round_trips_with_exact_schema() { - assert_eq!(WORKSPACE_PROTOCOL_SCHEMA, "nokv.workspace.rpc.v11"); + assert_eq!(WORKSPACE_PROTOCOL_SCHEMA, "nokv.workspace.rpc.v12"); let expected = request(); let encoded = encode_request(&expected).unwrap(); assert!(encoded @@ -283,7 +283,7 @@ mod tests { } #[test] - fn get_path_read_version_fence_has_one_exact_v11_encoding() { + fn get_path_read_version_fence_has_one_exact_v12_encoding() { let expected = WorkspaceRpcRequest { route: route(), request_id: RequestIdentity([0x41; 16]), @@ -305,15 +305,15 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(&encoded)), [ - 151, 149, 192, 126, 198, 107, 89, 133, 25, 106, 99, 70, 118, 191, 55, 180, 115, 9, - 187, 76, 58, 192, 124, 202, 204, 164, 144, 69, 134, 119, 155, 116 + 179, 220, 36, 137, 57, 170, 16, 113, 127, 121, 12, 54, 66, 112, 180, 80, 1, 16, 34, + 48, 206, 193, 195, 10, 114, 163, 6, 16, 114, 126, 230, 28 ], - "update only for an intentional GetPath v11 wire change" + "update only for an intentional GetPath v12 wire change" ); } #[test] - fn artifact_v1_query_and_catalog_keep_one_exact_v11_encoding() { + fn artifact_v1_query_and_catalog_keep_one_exact_v12_encoding() { let search = WorkspaceRpcRequest { route: route(), request_id: RequestIdentity([0x3a; 16]), @@ -431,10 +431,10 @@ mod tests { assert_eq!( <[u8; 32]>::from(golden.finalize()), [ - 224, 19, 176, 101, 32, 19, 101, 57, 154, 117, 159, 21, 186, 94, 186, 159, 240, 26, - 238, 69, 44, 255, 243, 228, 71, 30, 182, 159, 24, 169, 164, 44 + 48, 253, 178, 231, 50, 193, 139, 175, 49, 139, 216, 226, 90, 13, 27, 136, 102, 252, + 27, 175, 160, 165, 77, 14, 209, 86, 251, 151, 56, 101, 21, 136 ], - "update only for an intentional ArtifactV1 v11 wire change" + "update only for an intentional ArtifactV1 v12 wire change" ); assert_eq!(decode_request(&encoded_search).unwrap(), search); assert_eq!( @@ -448,7 +448,7 @@ mod tests { } #[test] - fn generic_exact_catalog_path_match_has_one_exact_v11_encoding() { + fn generic_exact_catalog_path_match_has_one_exact_v12_encoding() { let expected = WorkspaceRpcRequest { route: route(), request_id: RequestIdentity([0x42; 16]), @@ -475,15 +475,15 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(&encoded)), [ - 236, 170, 91, 173, 234, 150, 77, 102, 131, 72, 85, 116, 45, 218, 121, 129, 54, 46, - 38, 40, 52, 116, 161, 232, 249, 129, 85, 69, 223, 111, 102, 199 + 202, 3, 155, 179, 204, 51, 168, 138, 4, 124, 170, 103, 61, 71, 150, 159, 227, 65, + 149, 73, 201, 142, 66, 149, 59, 115, 150, 253, 238, 154, 39, 33 ], - "update only for an intentional Catalog Exact v11 wire change" + "update only for an intentional Catalog Exact v12 wire change" ); } #[test] - fn generic_registration_surface_has_one_exact_v11_encoding() { + fn generic_registration_surface_has_one_exact_v12_encoding() { let begin = generic_begin_request(); let append = generic_append_request(); let finalize = WorkspaceRpcRequest { @@ -593,10 +593,10 @@ mod tests { assert_eq!( <[u8; 32]>::from(golden.finalize()), [ - 185, 53, 175, 164, 133, 145, 72, 71, 43, 190, 231, 221, 240, 3, 18, 74, 81, 106, - 55, 71, 148, 46, 123, 205, 194, 11, 46, 44, 167, 235, 163, 164 + 76, 148, 117, 180, 52, 151, 228, 127, 234, 140, 75, 192, 248, 29, 46, 94, 196, 4, + 57, 122, 242, 12, 110, 16, 48, 93, 115, 120, 236, 102, 84, 128 ], - "update only for an intentional Generic registration v11 wire change" + "update only for an intentional Generic registration v12 wire change" ); for request in [&begin, &append, &finalize, &abort, &get] { assert_eq!( @@ -975,7 +975,7 @@ mod tests { } #[test] - fn restore_v11_late_binding_and_source_manifest_read_round_trip() { + fn restore_v12_late_binding_and_source_manifest_read_round_trip() { let restore_identity = RestoreManifestIdentity { publication_operation_id: OperationIdentity([0x21; 16]), artifact_revision_id: ArtifactRevisionIdentity([0x22; 16]), @@ -1003,10 +1003,10 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(&encoded_prepare)), [ - 165, 58, 202, 67, 148, 85, 50, 21, 84, 12, 61, 62, 78, 30, 153, 174, 182, 119, 214, - 16, 29, 98, 204, 66, 241, 225, 130, 254, 58, 121, 97, 83 + 19, 184, 126, 255, 221, 167, 62, 204, 167, 139, 10, 50, 60, 97, 62, 227, 184, 120, + 216, 104, 208, 37, 254, 45, 160, 157, 145, 121, 149, 84, 66, 80 ], - "update only for an intentional restore-v11 wire change" + "update only for an intentional restore-v12 wire change" ); let bind = WorkspaceRpcRequest { @@ -1118,7 +1118,7 @@ mod tests { source_manifest ); - let mut complete_v11_golden = Sha256::new(); + let mut complete_v12_golden = Sha256::new(); for encoded in [ encoded_prepare, encoded_bind, @@ -1126,16 +1126,16 @@ mod tests { encoded_prepared, encoded_source_manifest, ] { - complete_v11_golden.update((encoded.len() as u64).to_be_bytes()); - complete_v11_golden.update(encoded); + complete_v12_golden.update((encoded.len() as u64).to_be_bytes()); + complete_v12_golden.update(encoded); } assert_eq!( - <[u8; 32]>::from(complete_v11_golden.finalize()), + <[u8; 32]>::from(complete_v12_golden.finalize()), [ - 196, 15, 64, 182, 228, 178, 23, 245, 169, 244, 22, 242, 215, 154, 19, 155, 168, - 188, 201, 101, 214, 92, 215, 160, 221, 205, 51, 247, 186, 249, 83, 31 + 67, 211, 249, 84, 52, 53, 130, 176, 253, 198, 101, 67, 121, 119, 167, 220, 59, 152, + 156, 165, 126, 213, 114, 37, 229, 198, 254, 20, 94, 148, 15, 3 ], - "update only for an intentional restore-v11 wire change" + "update only for an intentional restore-v12 wire change" ); } @@ -1254,18 +1254,20 @@ mod tests { #[test] fn schema_mismatch_fails_closed() { - let encoded = rmp_serde::to_vec_named(&Frame { - schema: "nokv.workspace.rpc.v8".to_owned(), - payload: request(), - }) - .unwrap(); - assert_eq!( - decode_request(&encoded), - Err(ProtocolError::SchemaMismatch { - actual: "nokv.workspace.rpc.v8".to_owned(), - expected: "nokv.workspace.rpc.v11", + for previous in ["nokv.workspace.rpc.v8", "nokv.workspace.rpc.v11"] { + let encoded = rmp_serde::to_vec_named(&Frame { + schema: previous.to_owned(), + payload: request(), }) - ); + .unwrap(); + assert_eq!( + decode_request(&encoded), + Err(ProtocolError::SchemaMismatch { + actual: previous.to_owned(), + expected: "nokv.workspace.rpc.v12", + }) + ); + } #[derive(Serialize)] #[serde(deny_unknown_fields)] @@ -1316,7 +1318,7 @@ mod tests { decode_request(&actual_v4), Err(ProtocolError::SchemaMismatch { actual: "nokv.workspace.rpc.v4".to_owned(), - expected: "nokv.workspace.rpc.v11", + expected: "nokv.workspace.rpc.v12", }) ); @@ -1344,7 +1346,7 @@ mod tests { decode_response(&encoded), Err(ProtocolError::SchemaMismatch { actual: "nokv.workspace.rpc.v4".to_owned(), - expected: "nokv.workspace.rpc.v11", + expected: "nokv.workspace.rpc.v12", }) ); } @@ -1458,8 +1460,8 @@ mod tests { assert_eq!( <[u8; 32]>::from(Sha256::digest(encode_request(&expected).unwrap())), [ - 1, 199, 172, 241, 75, 114, 194, 208, 36, 153, 86, 191, 63, 217, 252, 231, 179, 144, - 197, 77, 51, 2, 137, 102, 136, 205, 100, 119, 58, 134, 223, 72 + 136, 147, 125, 201, 247, 93, 58, 26, 17, 129, 123, 181, 179, 183, 26, 230, 10, 178, + 187, 91, 218, 24, 107, 190, 100, 8, 47, 131, 64, 28, 68, 16 ], "update only for an intentional capability-v2 wire change" ); diff --git a/crates/nokv-protocol/src/lib.rs b/crates/nokv-protocol/src/lib.rs index 264afb48b..3286a7736 100644 --- a/crates/nokv-protocol/src/lib.rs +++ b/crates/nokv-protocol/src/lib.rs @@ -24,8 +24,9 @@ pub const MAX_ARTIFACT_DEPENDENCY_OWNERS: u32 = 64; pub const MAX_ARTIFACT_DEPENDENCY_DEPTH: u8 = 8; pub use append::{ - stable_append_attempt_identities, AppendAttemptBinding, AppendAttemptPhase, AppendPreparation, - AppendResult, + stable_append_attempt_identities, AppendAttemptBinding, AppendAttemptPhase, + AppendCleanupInspection, AppendCleanupRetryResult, AppendPreparation, AppendResult, + InspectAppendCleanupRequest, RetryAppendCleanupRequest, }; pub use artifact_publish::{ parse_sha256_digest_uri, seal_artifact_publish_plan, sha256_digest_uri, diff --git a/crates/nokv-protocol/src/request.rs b/crates/nokv-protocol/src/request.rs index 88ed95b64..5591c21ca 100644 --- a/crates/nokv-protocol/src/request.rs +++ b/crates/nokv-protocol/src/request.rs @@ -96,6 +96,8 @@ pub enum WorkspaceRequest { ReadRestoreSourceRunManifest(ReadRestoreSourceRunManifestRequest), FinalizeRestore(FinalizeRestoreRequest), GetOperation(GetOperationRequest), + InspectAppendCleanup(crate::InspectAppendCleanupRequest), + RetryAppendCleanup(crate::RetryAppendCleanupRequest), BeginGenericIndexRegistration(BeginGenericIndexRegistrationRequest), AppendGenericIndexRows(AppendGenericIndexRowsRequest), FinalizeGenericIndexRegistration(FinalizeGenericIndexRegistrationRequest), @@ -136,6 +138,8 @@ impl WorkspaceRequest { Self::ReadRestoreSourceRunManifest(request) => request.validate(), Self::FinalizeRestore(request) => request.validate(), Self::GetOperation(request) => request.validate(), + Self::InspectAppendCleanup(request) => request.validate(), + Self::RetryAppendCleanup(_) => Ok(()), Self::BeginGenericIndexRegistration(request) => request.validate(), Self::AppendGenericIndexRows(request) => request.validate(), Self::FinalizeGenericIndexRegistration(request) => request.validate(), @@ -718,8 +722,6 @@ pub enum QuarantineResolution { /// revision was never published; the revision identity is released for a /// fresh publication. ProviderObjectsAbsent, - /// Failed append keys are permanently sealed against delayed immutable creates. - ProviderObjectsSealed, /// The artifact revision is already published; staged provider keys are /// the published revision's live objects and only this operation's /// private bookkeeping rows are removed. diff --git a/crates/nokv-protocol/src/response.rs b/crates/nokv-protocol/src/response.rs index 855609766..f27fd3c19 100644 --- a/crates/nokv-protocol/src/response.rs +++ b/crates/nokv-protocol/src/response.rs @@ -114,6 +114,8 @@ pub enum WorkspaceResult { Renamed(RenamePathResult), Removed(RemovePathResult), Operation(OperationStatus), + AppendCleanupInspection(crate::AppendCleanupInspection), + AppendCleanupRetried(crate::AppendCleanupRetryResult), Published(PublishResult), Commit(CommitResult), Snapshot(SnapshotResult), @@ -141,6 +143,8 @@ impl WorkspaceResult { Self::Renamed(renamed) => renamed.validate(), Self::Removed(removed) => removed.validate(), Self::Operation(operation) => operation.validate(), + Self::AppendCleanupInspection(page) => page.validate(), + Self::AppendCleanupRetried(receipt) => receipt.validate(), Self::Published(published) => published.validate(), Self::Commit(commit) => commit.validate(), Self::Snapshot(snapshot) => snapshot.validate(), @@ -1551,7 +1555,7 @@ pub struct OperationStatus { } impl OperationStatus { - fn validate(&self) -> Result<(), ProtocolError> { + pub(crate) fn validate(&self) -> Result<(), ProtocolError> { self.progress.validate()?; match (self.kind, self.append_preparation.as_ref()) { (OperationKind::ArtifactAppend, Some(preparation)) => { diff --git a/crates/nokv-protocol/src/types.rs b/crates/nokv-protocol/src/types.rs index b38b7da1f..9ab96cc33 100644 --- a/crates/nokv-protocol/src/types.rs +++ b/crates/nokv-protocol/src/types.rs @@ -139,6 +139,7 @@ impl RootRoute { #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "snake_case")] pub enum WorkspaceCapability { + ArtifactAppendRecoveryV1, ArtifactAppendV1, ArtifactPublishV1, ArtifactRangeReadV1, @@ -153,7 +154,8 @@ pub enum WorkspaceCapability { } impl WorkspaceCapability { - pub const ALL: [Self; 11] = [ + pub const ALL: [Self; 12] = [ + Self::ArtifactAppendRecoveryV1, Self::ArtifactAppendV1, Self::ArtifactPublishV1, Self::ArtifactRangeReadV1, diff --git a/crates/nokv-python/README.md b/crates/nokv-python/README.md index 36393bf01..97beb3132 100644 --- a/crates/nokv-python/README.md +++ b/crates/nokv-python/README.md @@ -89,13 +89,34 @@ not for installing the SDK. lazily on object operations, so a fresh status client works during an S3 outage. The result reports `state` and `next_action` from the shared Rust SDK: `committed/none` with the original `receipt`, `pending/poll`, - `ready_to_retry/resubmit_same`, or `quarantined/operator_reconcile`. It also + `ready_to_retry/resubmit_same`, or `quarantined/retry_cleanup`. It also contains the logical and publication identities, attempt number and phase, activity deadline, original target and incarnation, progress, `cause_code`, `failure_message`, and `attempt_failure` for a failed physical attempt. The logical operation may still be `ready_to_retry` while retaining that failure. A query never advances an attempt. Retain or reproduce the original delta until commitment; follow `resubmit_same` using `append_bytes`. +- `Client.operation_inspect(operation_id, *, cursor=None, limit=32)` returns the + same status plus logical `operation_token`, child `publication_token`, object + namespace, `registered_count`, `cleanup_cursor`, `remaining_count`, and a page + of retained staged `entries`. Each entry has `sequence`, `object_identity`, + `expected_length`, `expected_digest`, and optional `multipart_token`. Retired + keys are not a historical inventory. The limit is 1 to 192; `next_cursor` is + opaque `bytes` or `None`. Continue with the unchanged cursor. A changed state + returns `Conflict`; restart inspection rather than combining different pages. +- `Client.operation_recover(operation_id, expected_state_digest=None)` asks the + owner to retry quarantined cleanup. Both inspection and recovery are metadata + only and work with `Client(root_id, routing)` without S3 credentials. Persist + the inspected `operation_token["state_digest"]` (a lowercase hex string) and + pass it on every retry of one recovery request. `requested=True` means durable + acceptance, including replay; `recovery_receipt` preserves that round's + logical id, publication id, cleanup retry count, and expected state digest. + Top-level status is a fresh observation and may be newer than the receipt. + Acceptance is not cleanup completion. Without a digest, the method targets + the current state once and is a no-op for pending, cleaned, or committed + operations. A later quarantine requires a new intentional recovery request. + When status reaches `ready_to_retry`, redeliver the original append inputs; + recovery does not reconstruct or publish the delta. - Append and status failures carry `operation_id`, observed `state`, `code`, expected incarnation in `expected`, `cause_code`, and `next_action=query_same`. `publication_operation_id` is `None` until status supplies an observed @@ -105,6 +126,10 @@ not for installing the SDK. lifecycle, so recover the original intent instead of blindly resubmitting. `retryable` is false for generic retry handlers. An unknown result or a `NotFound` observation never authorizes a replacement logical identity. + Cleanup errors use `code=AppendCleanupUnresolved` and + `next_action=retry_same_cleanup`, retain `expected_state_digest`, and expose + any known `recovery_receipt` and its publication id. Retry using that exact + digest; do not automatically choose a fresh token after a lost reply. - `WorkbenchFileSystem` is an fsspec compatibility adapter bound to one explicit Workbench. Paths must be one of `input`, `scripts`, `outputs`, `logs`, or `metadata`, optionally followed by an artifact-relative path. Sections and diff --git a/crates/nokv-python/python/nokv/__init__.py b/crates/nokv-python/python/nokv/__init__.py index 77863d0c1..46c6e50e3 100644 --- a/crates/nokv-python/python/nokv/__init__.py +++ b/crates/nokv-python/python/nokv/__init__.py @@ -38,8 +38,12 @@ class AppendError(RuntimeError): published. Retry the same intent with the same ``operation_id``; do not generate a replacement identity to bypass an unresolved operation. Query ``Client.operation_status`` and follow its ``next_action``; the exception's - ``next_action`` is conservatively ``query_same``. A publication identity is - not inferred from an error and remains ``None`` until observed by a query. + ``next_action`` is conservatively ``query_same`` for append errors. + ``AppendCleanupUnresolved`` instead supplies ``retry_same_cleanup`` and an + ``expected_state_digest``: pass that same digest to ``operation_recover`` + after an uncertain reply. An already observed ``recovery_receipt`` and its + publication identity are retained even when the later status query fails. + Without an observed receipt, the publication identity remains ``None``. ``cause_code`` preserves the underlying RPC code, for example ``RequestReplayMismatch`` for a permanently mismatched intent, or ``None`` when there is no RPC failure. ``retryable`` is always false: generic retry @@ -56,6 +60,8 @@ def __init__( cause_code: str | None = None, next_action: str = "query_same", publication_operation_id: str | None = None, + expected_state_digest: str | None = None, + recovery_receipt: dict | None = None, ) -> None: super().__init__(message) self.operation_id = operation_id @@ -66,6 +72,8 @@ def __init__( self.retryable = False self.next_action = next_action self.publication_operation_id = publication_operation_id + self.expected_state_digest = expected_state_digest + self.recovery_receipt = recovery_receipt class WorkspaceIncarnationMismatch(RuntimeError): diff --git a/crates/nokv-python/src/client.rs b/crates/nokv-python/src/client.rs index cc6ebbb2a..893b46537 100644 --- a/crates/nokv-python/src/client.rs +++ b/crates/nokv-python/src/client.rs @@ -39,11 +39,12 @@ use crate::local_adapter::{ }; use crate::object_store::{ConfiguredObjectStore, PythonObjectStoreConfig}; use crate::python_value::{ - aggregate_result_to_py, append_result_to_py, catalog_result_to_py, - find_workspaces_result_to_py, hex, parse_aggregates, parse_field_specs, parse_fixed_hex, - parse_predicates, parse_sort, path_metadata_to_py, path_page_to_py, publish_outcome_to_py, - read_outcome_to_py, search_result_to_py, snapshot_result_to_py, workspace_summary_to_py, - PythonAggregateSpec, PythonFieldSpec, PythonPredicateSpec, PythonSortSpec, + aggregate_result_to_py, append_result_to_py, catalog_result_to_py, cleanup_retry_receipt_to_py, + find_workspaces_result_to_py, hex, operation_token_to_py, parse_aggregates, parse_field_specs, + parse_fixed_hex, parse_predicates, parse_sort, path_metadata_to_py, path_page_to_py, + publish_outcome_to_py, read_outcome_to_py, search_result_to_py, snapshot_result_to_py, + workspace_summary_to_py, PythonAggregateSpec, PythonFieldSpec, PythonPredicateSpec, + PythonSortSpec, }; use crate::routing::PythonRoutingConfig; @@ -220,7 +221,7 @@ impl PythonWorkspaceClient { } let origin = match (at_snapshot, at_commit) { (Some(_), Some(_)) => { - return Err(value_error("give at_snapshot or at_commit, not both")) + return Err(value_error("give at_snapshot or at_commit, not both")); } (None, None) => return Err(value_error("give at_snapshot or at_commit")), (Some(value), None) => { @@ -646,71 +647,128 @@ impl PythonWorkspaceClient { let call = py .detach(move || client.get_append_operation(operation_id)) .map_err(|error| append_error(py, error, operation_id, None))?; - let recovery = nokv_client::append_operation_recovery(&call.value) + let dict = append_status_to_py(py, &call.value)?; + set_call_metadata(&dict, call.commit_version, call.replayed)?; + Ok(dict) + } + + /// Inspect one bounded page of the current append's remaining staged objects. + /// Save the opaque cursor unchanged; it is fenced to this exact logical state. + #[pyo3(signature = (operation_id, *, cursor=None, limit=32))] + fn operation_inspect<'py>( + &self, + py: Python<'py>, + operation_id: &str, + cursor: Option<&Bound<'py, PyBytes>>, + limit: u32, + ) -> PyResult> { + let operation_id = OperationIdentity(parse_fixed_hex("operation_id", operation_id)?); + let cursor = cursor.map(|value| value.as_bytes()); + if cursor.is_some_and(|value| value.len() > PageRequest::MAX_CURSOR_BYTES) { + return Err(append_local_error( + py, + "inspection cursor exceeds its byte limit".to_owned(), + operation_id, + None, + "InvalidArgument", + None, + )); + } + let page = PageRequest { + cursor: cursor.map(<[u8]>::to_vec), + limit, + }; + let client = Arc::clone(&self.client); + let call = py + .detach(move || client.inspect_append_operation(operation_id, page)) .map_err(|error| append_error(py, error, operation_id, None))?; - let status = &call.value; - let preparation = status - .append_preparation - .as_ref() - .ok_or_else(|| runtime_error("append status has no preparation"))?; - let dict = PyDict::new(py); - dict.set_item("status", "success")?; - dict.set_item("operation", "append")?; - dict.set_item("operation_id", hex(&status.token.operation_id.0))?; - dict.set_item("state", recovery.state.as_str())?; - dict.set_item("next_action", recovery.next_action.as_str())?; - dict.set_item("observed_state", format!("{:?}", status.state))?; + let inspection = &call.value.inspection; + let dict = append_status_to_py(py, &inspection.operation)?; + dict.set_item("action", "inspect")?; dict.set_item( - "publication_operation_id", - hex(&preparation.publication_operation_id.0), + "operation_token", + operation_token_to_py(py, inspection.operation.token)?, )?; - dict.set_item("attempt", preparation.attempt)?; dict.set_item( - "attempt_phase", - format!("{:?}", preparation.attempt_phase).to_ascii_lowercase(), + "publication_token", + operation_token_to_py(py, inspection.publication_token)?, )?; - dict.set_item("activity_deadline_ms", preparation.activity_deadline_ms)?; dict.set_item( - "workspace_incarnation_id", - hex(&preparation.workspace_incarnation_id.0), + "object_namespace_id", + hex(&inspection.object_namespace_id.0), )?; - dict.set_item("workbench_id", preparation.target.workbench.as_str())?; - dict.set_item("path", preparation.target.path.as_str())?; - let progress = PyDict::new(py); - progress.set_item("completed_rows", status.progress.completed_rows)?; - progress.set_item("total_rows", status.progress.total_rows)?; - progress.set_item("completed_bytes", status.progress.completed_bytes)?; - progress.set_item("total_bytes", status.progress.total_bytes)?; - dict.set_item("progress", progress)?; - let attempt_failure = preparation.attempt_failure.as_ref(); + dict.set_item("registered_count", inspection.registered_count)?; + dict.set_item("cleanup_cursor", inspection.cleanup_cursor)?; + dict.set_item("remaining_count", inspection.remaining_count)?; + let entries = PyList::empty(py); + for entry in &inspection.entries { + let row = PyDict::new(py); + row.set_item("sequence", entry.sequence)?; + row.set_item("object_identity", entry.object_identity.as_str())?; + row.set_item("expected_length", entry.expected_length)?; + row.set_item("expected_digest", entry.expected_digest.as_str())?; + row.set_item("multipart_token", entry.multipart_token.clone())?; + entries.append(row)?; + } + dict.set_item("entries", entries)?; dict.set_item( - "cause_code", - attempt_failure.map(|failure| format!("{:?}", failure.code)), + "next_cursor", + call.value + .next_cursor + .as_ref() + .map(|cursor| PyBytes::new(py, cursor)), )?; + set_call_metadata(&dict, call.commit_version, call.replayed)?; + Ok(dict) + } + + /// Ask the owner to retry quarantined cleanup without sending object bytes. + /// Reuse the inspected state digest to replay one durable recovery request. + #[pyo3(signature = (operation_id, expected_state_digest=None))] + fn operation_recover<'py>( + &self, + py: Python<'py>, + operation_id: &str, + expected_state_digest: Option<&str>, + ) -> PyResult> { + let operation_id = OperationIdentity(parse_fixed_hex("operation_id", operation_id)?); + let expected = expected_state_digest + .map(|value| { + parse_fixed_hex("expected_state_digest", value).map(|digest| { + nokv_protocol::OperationToken { + operation_id, + state_digest: nokv_protocol::Digest(digest), + } + }) + }) + .transpose() + .map_err(|error| { + append_local_error( + py, + error.to_string(), + operation_id, + None, + "InvalidArgument", + None, + ) + })?; + let client = Arc::clone(&self.client); + let call = py + .detach(move || client.recover_append_operation(operation_id, expected)) + .map_err(|error| append_error(py, error, operation_id, None))?; + let dict = append_status_to_py(py, &call.value.operation)?; + dict.set_item("action", "recover")?; dict.set_item( - "failure_message", - attempt_failure.map(|failure| failure.message.as_str()), + "operation_token", + operation_token_to_py(py, call.value.operation.token)?, )?; - match attempt_failure { - Some(failure) => { - let details = PyDict::new(py); - details.set_item("code", format!("{:?}", failure.code))?; - details.set_item("message", &failure.message)?; - details.set_item("retryable", failure.retryable)?; - details.set_item( - "conflict", - failure.conflict.map(|conflict| format!("{conflict:?}")), - )?; - details.set_item("current_generation", failure.current_generation)?; - dict.set_item("attempt_failure", details)?; - } - None => dict.set_item("attempt_failure", py.None())?, - } - match &status.result { - Some(nokv_protocol::OperationResult::ArtifactAppend(result)) => { - dict.set_item("receipt", append_result_to_py(py, result)?)?; - } - _ => dict.set_item("receipt", py.None())?, + dict.set_item("requested", call.value.requested)?; + match &call.value.receipt { + Some(receipt) => dict.set_item( + "recovery_receipt", + cleanup_retry_receipt_to_py(py, receipt)?, + )?, + None => dict.set_item("recovery_receipt", py.None())?, } set_call_metadata(&dict, call.commit_version, call.replayed)?; Ok(dict) @@ -1700,6 +1758,79 @@ fn validate_list_page_fence( Ok(()) } +fn append_status_to_py<'py>( + py: Python<'py>, + status: &nokv_protocol::OperationStatus, +) -> PyResult> { + let recovery = nokv_client::append_operation_recovery(status) + .map_err(|error| append_error(py, error, status.token.operation_id, None))?; + let preparation = status + .append_preparation + .as_ref() + .ok_or_else(|| runtime_error("append status has no preparation"))?; + let dict = PyDict::new(py); + dict.set_item("status", "success")?; + dict.set_item("operation", "append")?; + dict.set_item("operation_id", hex(&status.token.operation_id.0))?; + dict.set_item("state", recovery.state.as_str())?; + dict.set_item("next_action", recovery.next_action.as_str())?; + dict.set_item("observed_state", format!("{:?}", status.state))?; + dict.set_item( + "publication_operation_id", + hex(&preparation.publication_operation_id.0), + )?; + dict.set_item("attempt", preparation.attempt)?; + dict.set_item("cleanup_retry_count", preparation.cleanup_retry_count)?; + dict.set_item( + "attempt_phase", + format!("{:?}", preparation.attempt_phase).to_ascii_lowercase(), + )?; + dict.set_item("activity_deadline_ms", preparation.activity_deadline_ms)?; + dict.set_item( + "workspace_incarnation_id", + hex(&preparation.workspace_incarnation_id.0), + )?; + dict.set_item("workbench_id", preparation.target.workbench.as_str())?; + dict.set_item("path", preparation.target.path.as_str())?; + let progress = PyDict::new(py); + progress.set_item("completed_rows", status.progress.completed_rows)?; + progress.set_item("total_rows", status.progress.total_rows)?; + progress.set_item("completed_bytes", status.progress.completed_bytes)?; + progress.set_item("total_bytes", status.progress.total_bytes)?; + dict.set_item("progress", progress)?; + let attempt_failure = preparation.attempt_failure.as_ref(); + dict.set_item( + "cause_code", + attempt_failure.map(|failure| format!("{:?}", failure.code)), + )?; + dict.set_item( + "failure_message", + attempt_failure.map(|failure| failure.message.as_str()), + )?; + match attempt_failure { + Some(failure) => { + let details = PyDict::new(py); + details.set_item("code", format!("{:?}", failure.code))?; + details.set_item("message", &failure.message)?; + details.set_item("retryable", failure.retryable)?; + details.set_item( + "conflict", + failure.conflict.map(|conflict| format!("{conflict:?}")), + )?; + details.set_item("current_generation", failure.current_generation)?; + dict.set_item("attempt_failure", details)?; + } + None => dict.set_item("attempt_failure", py.None())?, + } + match &status.result { + Some(nokv_protocol::OperationResult::ArtifactAppend(result)) => { + dict.set_item("receipt", append_result_to_py(py, result)?)?; + } + _ => dict.set_item("receipt", py.None())?, + } + Ok(dict) +} + fn runtime_error(error: impl std::fmt::Display) -> PyErr { PyRuntimeError::new_err(error.to_string()) } @@ -1737,6 +1868,40 @@ fn append_error( operation_id: OperationIdentity, expected: Option, ) -> PyErr { + if let ClientError::AppendCleanupUnresolved { + expected_token, + receipt, + .. + } = &error + { + let message = error.to_string(); + let cause_code = client_error_failure(&error).map(|failure| format!("{:?}", failure.code)); + return py + .import("nokv") + .and_then(|module| module.getattr("AppendError")) + .and_then(|class| { + let recovery_receipt = receipt + .as_deref() + .map(|value| cleanup_retry_receipt_to_py(py, value)) + .transpose()?; + class.call1(( + message.clone(), + hex(&operation_id.0), + None::, + "AppendCleanupUnresolved", + None::, + cause_code, + "retry_same_cleanup", + receipt + .as_ref() + .map(|value| hex(&value.publication_operation_id.0)), + hex(&expected_token.state_digest.0), + recovery_receipt, + )) + }) + .map(PyErr::from_value) + .unwrap_or_else(|_| PyRuntimeError::new_err(message)); + } let identity = hex(&operation_id.0); let expected = expected.map(|value| hex(&value.0)); let state = match &error { @@ -1809,6 +1974,7 @@ fn client_error_failure(error: &ClientError) -> Option<&nokv_protocol::RpcFailur ClientError::Rpc(failure) => Some(failure), ClientError::ArtifactPublishFailed { source, .. } | ClientError::AppendUnresolved { source, .. } + | ClientError::AppendCleanupUnresolved { source, .. } | ClientError::RetryExhausted { last_error: source, .. } => client_error_failure(source), @@ -1831,6 +1997,7 @@ fn client_error_code(error: &ClientError) -> Option { ClientError::Rpc(failure) => Some(failure.code), ClientError::ArtifactPublishFailed { source, .. } | ClientError::AppendUnresolved { source, .. } + | ClientError::AppendCleanupUnresolved { source, .. } | ClientError::RetryExhausted { last_error: source, .. } => client_error_code(source), diff --git a/crates/nokv-python/src/python_value.rs b/crates/nokv-python/src/python_value.rs index 5e281ca8f..ffa324052 100644 --- a/crates/nokv-python/src/python_value.rs +++ b/crates/nokv-python/src/python_value.rs @@ -5,10 +5,11 @@ use nokv_client::{ArtifactPublishOutcome, ArtifactReadOutcome}; use nokv_protocol::{ - AggregateFunction, AggregateResult, AggregateSpec, CatalogResult, CommitResult, FacetResult, - FieldValue, FindWorkspacesResult, PathListEntry, PathMetadata, PathPage, PublishResult, - QueryOperand, QueryOperator, QueryPredicate, ScalarValue, SearchResult, SearchRow, - SnapshotResult, SnapshotStatus, SortDirection, SortField, WorkspaceSummary, + AggregateFunction, AggregateResult, AggregateSpec, AppendCleanupRetryResult, CatalogResult, + CommitResult, FacetResult, FieldValue, FindWorkspacesResult, OperationToken, PathListEntry, + PathMetadata, PathPage, PublishResult, QueryOperand, QueryOperator, QueryPredicate, + ScalarValue, SearchResult, SearchRow, SnapshotResult, SnapshotStatus, SortDirection, SortField, + WorkspaceSummary, }; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; @@ -19,6 +20,34 @@ pub(crate) type PythonPredicateSpec = (String, String, String, String); pub(crate) type PythonSortSpec = (String, String); pub(crate) type PythonAggregateSpec = (String, Option, String); +pub(crate) fn operation_token_to_py<'py>( + py: Python<'py>, + token: OperationToken, +) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("operation_id", hex(&token.operation_id.0))?; + dict.set_item("state_digest", hex(&token.state_digest.0))?; + Ok(dict) +} + +pub(crate) fn cleanup_retry_receipt_to_py<'py>( + py: Python<'py>, + receipt: &AppendCleanupRetryResult, +) -> PyResult> { + let dict = PyDict::new(py); + dict.set_item("operation_id", hex(&receipt.operation_id.0))?; + dict.set_item( + "publication_operation_id", + hex(&receipt.publication_operation_id.0), + )?; + dict.set_item("cleanup_retry_count", receipt.cleanup_retry_count)?; + dict.set_item( + "expected_state_digest", + hex(&receipt.expected_state_digest.0), + )?; + Ok(dict) +} + pub(crate) fn parse_fixed_hex( field: &str, encoded: &str, diff --git a/crates/nokv-python/tests/test_api_surface.py b/crates/nokv-python/tests/test_api_surface.py index 1519f8bca..e95e27849 100644 --- a/crates/nokv-python/tests/test_api_surface.py +++ b/crates/nokv-python/tests/test_api_surface.py @@ -35,6 +35,14 @@ def test_versioned_workbench_surface(): assert list(inspect.signature(nokv.Client.operation_status).parameters) == [ "self", "operation_id" ] + inspection = inspect.signature(nokv.Client.operation_inspect).parameters + assert list(inspection) == ["self", "operation_id", "cursor", "limit"] + assert inspection["cursor"].default is None + assert inspection["cursor"].kind is inspect.Parameter.KEYWORD_ONLY + assert inspection["limit"].default == 32 + recovery = inspect.signature(nokv.Client.operation_recover).parameters + assert list(recovery) == ["self", "operation_id", "expected_state_digest"] + assert recovery["expected_state_digest"].default is None assert "object_store=None" in (nokv.Client.__text_signature__ or "") append = inspect.signature(nokv.Client.append_bytes).parameters assert append["operation_id"].default is inspect.Parameter.empty @@ -109,6 +117,8 @@ def test_append_error_keeps_identity_and_observed_state(): assert error.retryable is False assert error.next_action == "query_same" assert error.publication_operation_id is None + assert error.expected_state_digest is None + assert error.recovery_receipt is None assert str(error) == "reply lost" mismatch = nokv.AppendError( "different intent", "a" * 32, None, "AppendUnresolved", "b" * 32, @@ -136,3 +146,25 @@ def test_append_provider_admission_errors_preserve_recovery_contract(): assert error.next_action == "query_same" assert error.publication_operation_id is None assert error.retryable is False + + +def test_cleanup_error_preserves_one_recovery_request_and_optional_receipt(): + receipt = { + "operation_id": "a" * 32, + "publication_operation_id": "b" * 32, + "cleanup_retry_count": 1, + "expected_state_digest": "c" * 64, + } + for known in (None, receipt): + error = nokv.AppendError( + "cleanup reply unknown", "a" * 32, None, "AppendCleanupUnresolved", + None, "NotOwner", "retry_same_cleanup", + "b" * 32 if known else None, "c" * 64, known, + ) + assert error.code == "AppendCleanupUnresolved" + assert error.cause_code == "NotOwner" + assert error.operation_id == "a" * 32 + assert error.expected_state_digest == "c" * 64 + assert error.next_action == "retry_same_cleanup" + assert error.recovery_receipt == known + assert error.retryable is False diff --git a/crates/nokv-server/src/executor.rs b/crates/nokv-server/src/executor.rs index ba9be5520..5d945f378 100644 --- a/crates/nokv-server/src/executor.rs +++ b/crates/nokv-server/src/executor.rs @@ -36,7 +36,8 @@ const _: () = assert!(protocol::MAX_QUERY_PAGE_LIMIT as usize == meta::MAX_QUERY // reject the legacy 61..=64 gap at DTO-to-domain conversion as main does. const _: () = assert!(protocol::ArtifactDescriptor::MAX_INDEX_FIELDS >= meta::MAX_TYPED_PROJECTION_FIELDS); -const SUPPORTED_WORKSPACE_CAPABILITIES: [protocol::WorkspaceCapability; 11] = [ +const SUPPORTED_WORKSPACE_CAPABILITIES: [protocol::WorkspaceCapability; 12] = [ + protocol::WorkspaceCapability::ArtifactAppendRecoveryV1, protocol::WorkspaceCapability::ArtifactAppendV1, protocol::WorkspaceCapability::ArtifactPublishV1, protocol::WorkspaceCapability::ArtifactRangeReadV1, @@ -284,6 +285,12 @@ impl MetadataWorkspaceRequestExecutor { self.finalize_restore(request, finalize) } protocol::WorkspaceRequest::GetOperation(get) => self.get_operation(request, get), + protocol::WorkspaceRequest::InspectAppendCleanup(inspect) => { + self.inspect_append_cleanup(request, inspect) + } + protocol::WorkspaceRequest::RetryAppendCleanup(retry) => { + self.retry_append_cleanup(request, retry) + } protocol::WorkspaceRequest::BeginGenericIndexRegistration(begin) => { self.begin_generic_index_registration(request, begin) } @@ -2172,6 +2179,7 @@ impl MetadataWorkspaceRequestExecutor { dependency_digest, cleanup_staged_object_cursor: 0, cleanup_manifest_cursor: 0, + cleanup_retry_count: 0, publication_absence_proof: None, result: None, terminal_error: None, @@ -2502,9 +2510,6 @@ impl MetadataWorkspaceRequestExecutor { protocol::QuarantineResolution::ProviderObjectsAbsent => { meta::QuarantineReconcileResolution::RevisionUnpublished } - protocol::QuarantineResolution::ProviderObjectsSealed => { - meta::QuarantineReconcileResolution::RevisionUnpublishedSealed - } protocol::QuarantineResolution::RevisionPublished => { meta::QuarantineReconcileResolution::RevisionPublished } @@ -2793,6 +2798,185 @@ impl MetadataWorkspaceRequestExecutor { }) } + fn inspect_append_cleanup( + &self, + rpc: &protocol::WorkspaceRpcRequest, + request: &protocol::InspectAppendCleanupRequest, + ) -> Result { + let context = self.read_context(rpc.route)?; + let logical_id: types::OperationId = request.token.operation_id.into(); + let key = meta::operation_key(context.root_id, types::OperationKind::Append, logical_id); + let payload = self + .meta + .read_at( + context.root_id, + context.placement_generation, + context.owner_epoch, + meta::MetadataFamily::Operation, + &key, + context.read_version, + ) + .map_err(meta_failure)? + .ok_or_else(|| not_found("logical append operation does not exist"))?; + let parent = meta::AppendOperationRecord::decode(&payload) + .map_err(|error| internal(format!("invalid append operation: {error}")))?; + if parent.operation_id != logical_id { + return Err(internal("append operation key and payload disagree")); + } + let child = self.load_publish_operation( + rpc.route, + context.read_version, + parent.publication_operation_id.into(), + )?; + let operation = append_operation_status(&parent, &child)?; + if child.phase != types::PublishPhase::Published { + let key = + meta::artifact_revision_claim_key(context.root_id, child.artifact_revision_id); + let payload = self + .meta + .read_at( + context.root_id, + context.placement_generation, + context.owner_epoch, + meta::MetadataFamily::ArtifactRevision, + &key, + context.read_version, + ) + .map_err(meta_failure)? + .ok_or_else(|| internal("append revision reservation is missing"))?; + if meta::ArtifactRevisionClaimRecord::decode(&payload) + .map_err(|error| internal(error.to_string()))? + .operation_id + != child.operation_id + { + return Err(internal( + "append revision reservation belongs to another operation", + )); + } + } + if operation.token != request.token { + return Err(conflict( + protocol::ConflictKind::OperationState, + "append state changed; restart inspection using its current logical token", + None, + )); + } + let start = match request.start_after { + None => child.cleanup_staged_object_cursor, + Some(after) + if after >= child.cleanup_staged_object_cursor + && after < child.staged_object_cursor => + { + after + 1 + } + Some(_) => { + return Err(invalid_argument( + "append cleanup cursor is outside the current retained ledger", + )) + } + }; + let end = start + .saturating_add(request.limit) + .min(child.staged_object_cursor); + let mut entries = Vec::with_capacity((end - start) as usize); + let route = route_parts(rpc.route)?; + for sequence in start..end { + let key = + meta::staged_object_key(context.root_id, child.operation_id, u64::from(sequence)); + let payload = self + .meta + .read_at( + context.root_id, + context.placement_generation, + context.owner_epoch, + meta::MetadataFamily::StagedObject, + &key, + context.read_version, + ) + .map_err(meta_failure)? + .ok_or_else(|| internal(format!("retained staged object {sequence} is missing")))?; + let row = meta::StagedObjectRecord::decode(&payload) + .map_err(|error| internal(format!("invalid staged object: {error}")))?; + if row.object_sequence != sequence + || row.artifact_revision_id != child.artifact_revision_id + || row.object_key + != meta::object_block_key( + route.logical_shard_id, + context.root_id, + child.artifact_revision_id, + u64::from(sequence), + ) + { + return Err(internal( + "staged object key, sequence, revision, or root binding disagrees", + )); + } + entries.push(protocol::StagedObject { + sequence, + object_identity: protocol::ObjectIdentity::new(row.object_key) + .map_err(|e| internal(e.to_string()))?, + expected_length: row.expected_length, + expected_digest: protocol::DigestUri::new(row.expected_digest_uri) + .map_err(|e| internal(e.to_string()))?, + multipart_token: row.multipart_upload_id, + }); + } + Ok(ExecutedRequest { + result: protocol::WorkspaceResult::AppendCleanupInspection( + protocol::AppendCleanupInspection { + operation: Box::new(operation), + object_namespace_id: rpc.route.object_namespace_id, + publication_token: protocol::OperationToken { + operation_id: child.operation_id.into(), + state_digest: publish_state_digest(&child)?, + }, + registered_count: child.staged_object_cursor, + cleanup_cursor: child.cleanup_staged_object_cursor, + remaining_count: child.staged_object_cursor + - child.cleanup_staged_object_cursor, + entries, + next_after: (end < child.staged_object_cursor).then(|| end - 1), + }, + ), + commit_version: None, + replayed: false, + }) + } + + fn retry_append_cleanup( + &self, + rpc: &protocol::WorkspaceRpcRequest, + request: &protocol::RetryAppendCleanupRequest, + ) -> Result { + self.claim_mutation(rpc)?; + // The service derives its durable enqueue ID from the full logical token; + // the outer request ID is only the ordinary exact-RPC fencing boundary. + let context = self.publication_context( + rpc.route, + derived_request_id(rpc.request_id, b"append-cleanup-retry-context", 0), + )?; + let outcome = meta::PublicationService::new(&self.meta) + .retry_append_cleanup( + context, + request.token.operation_id.into(), + request.token.state_digest.0, + ) + .map_err(publication_failure)?; + let receipt = outcome.receipt; + Ok(ExecutedRequest { + result: protocol::WorkspaceResult::AppendCleanupRetried( + protocol::AppendCleanupRetryResult { + operation_id: receipt.operation_id.into(), + publication_operation_id: receipt.publication_operation_id.into(), + cleanup_retry_count: receipt.cleanup_retry_count, + expected_state_digest: protocol::Digest(receipt.expected_state_digest), + }, + ), + commit_version: Some(outcome.commit_version.get()), + replayed: outcome.replayed, + }) + } + fn get_operation( &self, rpc: &protocol::WorkspaceRpcRequest, @@ -4312,14 +4496,8 @@ fn append_operation_status( } else { protocol::OperationState::Running }; - let mut hasher = Sha256::new(); - hasher.update(b"nokv.append.state.v1\0"); - hasher.update( - parent - .encode() - .map_err(|error| internal(error.to_string()))?, - ); - hasher.update(child_status.token.state_digest.0); + let state_digest = meta::append_operation_state_digest(parent, child) + .map_err(|error| internal(error.to_string()))?; Ok(protocol::OperationStatus { append_preparation: Some(Box::new(protocol::AppendPreparation { intent_digest: protocol::Digest(parent.intent_digest), @@ -4330,6 +4508,7 @@ fn append_operation_status( artifact_revision_id: parent.artifact_revision_id.into(), attempt_phase: phase, activity_deadline_ms: child.activity_deadline_ms, + cleanup_retry_count: child.cleanup_retry_count, attempt_failure: child_status.failure.clone().or_else(|| { child .terminal_error @@ -4339,7 +4518,7 @@ fn append_operation_status( })), token: protocol::OperationToken { operation_id: parent.operation_id.into(), - state_digest: protocol::Digest(hasher.finalize().into()), + state_digest: protocol::Digest(state_digest), }, kind: protocol::OperationKind::ArtifactAppend, publish_preparation: None, @@ -10546,6 +10725,293 @@ mod tests { } } + #[test] + fn append_cleanup_inspection_pages_only_retained_rows_and_retry_is_token_idempotent() { + let (store, executor) = ready_executor(); + executor + .execute(&create_request(0x81, "stable-append", 0x82, 1)) + .unwrap(); + let mut begin = stable_append_begin_request(); + let logical = begin.append_attempt.unwrap().operation_id; + let staged: Vec<_> = (0..65) + .map(|sequence| protocol::StagedObject { + sequence, + object_identity: protocol::ObjectIdentity::new(meta::object_block_key( + shard(), + root(), + begin.artifact_revision_id.into(), + u64::from(sequence), + )) + .unwrap(), + expected_length: 1, + expected_digest: protocol::sha256_digest_uri(protocol::Digest( + [sequence as u8; 32], + )), + multipart_token: None, + }) + .collect(); + let seals = + protocol::seal_artifact_publish_plan(begin.artifact_revision_id, &staged, &[]).unwrap(); + begin.staged_object_count = seals.staged_object_count; + begin.staged_object_seal = seals.staged_object_seal; + let begun = executor + .execute(&restore_rpc( + 0x86, + protocol::WorkspaceRequest::BeginArtifactPublish(begin.clone()), + )) + .unwrap(); + let protocol::WorkspaceResult::Operation(begun) = begun.result else { + panic!("expected publication status") + }; + executor + .execute(&restore_rpc( + 0x87, + protocol::WorkspaceRequest::StageArtifactObjects( + protocol::StageArtifactObjectsRequest { + token: begun.token, + objects: staged.clone(), + }, + ), + )) + .unwrap(); + let query = || { + let result = executor + .execute(&restore_rpc( + 0x88, + protocol::WorkspaceRequest::GetOperation(protocol::GetOperationRequest { + operation_id: logical, + }), + )) + .unwrap(); + let protocol::WorkspaceResult::Operation(status) = result.result else { + panic!("expected logical status") + }; + status + }; + let active = query(); + let active_failure = executor + .execute(&restore_rpc( + 0x89, + protocol::WorkspaceRequest::RetryAppendCleanup( + protocol::RetryAppendCleanupRequest { + token: active.token, + }, + ), + )) + .unwrap_err(); + assert_eq!( + active_failure.conflict, + Some(protocol::ConflictKind::OperationState) + ); + assert_eq!(query().token, active.token); + let service = meta::PublicationService::new(&store); + let mut child = executor + .load_publish_operation( + route(1), + store.current_read_version().unwrap(), + begin.operation_id, + ) + .unwrap(); + for (id, transition) in [ + ( + 0x90, + meta::PublishTransition::BeginAbort { + terminal_error: meta::PublishTerminalError { + kind: meta::PublishTerminalErrorKind::ActivityLeaseExpired, + message: "caller exited".to_owned(), + evidence_digest: None, + }, + }, + ), + (0x91, meta::PublishTransition::BeginCleaning), + ] { + child = service + .transition_publish(meta::TransitionPublishRequest { + context: executor + .publication_context(route(1), types::RequestId::from_bytes([id; 16])) + .unwrap(), + expected_operation: child, + transition, + }) + .unwrap() + .operation; + } + let rows = executor + .reconcile_staged_batch_rows(route(1), &child) + .unwrap(); + child = service + .cleanup_publish_batch(meta::CleanupPublishBatchRequest { + context: executor + .publication_context(route(1), types::RequestId::from_bytes([0x92; 16])) + .unwrap(), + expected_operation: child, + staged_object_updates: rows[..32] + .iter() + .map(|expected| { + let mut next = expected.clone(); + next.provider_state = types::StagedProviderState::Aborted; + next.cleanup_state = types::StagedCleanupState::Sealed; + meta::StagedObjectUpdate { + expected: expected.clone(), + next, + } + }) + .collect(), + }) + .unwrap() + .operation; + let quarantine_error = meta::PublishTerminalError { + kind: meta::PublishTerminalErrorKind::CleanupFailed, + message: "seal outcome unknown".to_owned(), + evidence_digest: Some([0x93; 32]), + }; + child = service + .transition_publish(meta::TransitionPublishRequest { + context: executor + .publication_context(route(1), types::RequestId::from_bytes([0x94; 16])) + .unwrap(), + expected_operation: child, + transition: meta::PublishTransition::Quarantine { + terminal_error: quarantine_error.clone(), + }, + }) + .unwrap() + .operation; + let token = query().token; + let inspect = |after| { + restore_rpc( + 0x95, + protocol::WorkspaceRequest::InspectAppendCleanup( + protocol::InspectAppendCleanupRequest { + token, + start_after: after, + limit: 32, + }, + ), + ) + }; + let version = store.current_read_version().unwrap(); + let first = executor.execute(&inspect(None)).unwrap(); + let protocol::WorkspaceResult::AppendCleanupInspection(first) = first.result else { + panic!("expected inspection") + }; + assert_eq!( + ( + first.registered_count, + first.cleanup_cursor, + first.remaining_count + ), + (65, 32, 33) + ); + assert_eq!(first.entries, staged[32..64]); + assert_eq!(first.next_after, Some(63)); + assert_eq!(first.operation.token, token); + assert_eq!(first.publication_token.operation_id, begin.operation_id); + let last = executor.execute(&inspect(first.next_after)).unwrap(); + let protocol::WorkspaceResult::AppendCleanupInspection(last) = last.result else { + panic!("expected inspection") + }; + assert_eq!(last.entries, staged[64..]); + assert_eq!(last.next_after, None); + assert_eq!( + store.current_read_version().unwrap(), + version, + "inspection must be metadata-only and read-only" + ); + let mut invalid = inspect(None); + let protocol::WorkspaceRequest::InspectAppendCleanup(request) = &mut invalid.operation + else { + unreachable!() + }; + request.limit = 193; + assert_eq!( + executor.execute(&invalid).unwrap_err().code, + protocol::ErrorCode::InvalidArgument + ); + let old_verdict = executor + .execute(&restore_rpc( + 0x96, + protocol::WorkspaceRequest::ReconcileQuarantinedArtifactPublish( + protocol::ReconcileQuarantinedArtifactPublishRequest { + token: first.publication_token, + resolution: protocol::QuarantineResolution::ProviderObjectsAbsent, + reason: "external claim is not owner sealing".to_owned(), + evidence_digest: protocol::Digest([0x97; 32]), + }, + ), + )) + .unwrap_err(); + assert_eq!(old_verdict.code, protocol::ErrorCode::PreconditionFailed); + assert_eq!(query().token, token); + let retry = restore_rpc( + 0x98, + protocol::WorkspaceRequest::RetryAppendCleanup(protocol::RetryAppendCleanupRequest { + token, + }), + ); + let mut redelivery = retry.clone(); + redelivery.request_id = protocol::RequestIdentity([0x99; 16]); + let (left, right) = std::thread::scope(|scope| { + let left = scope.spawn(|| executor.execute(&retry)); + let right = scope.spawn(|| executor.execute(&redelivery)); + ( + left.join().unwrap().unwrap(), + right.join().unwrap().unwrap(), + ) + }); + assert_ne!( + left.replayed, right.replayed, + "concurrent same-token requests must enqueue once" + ); + assert_eq!(left.result, right.result); + assert_eq!(left.commit_version, right.commit_version); + let accepted = if left.replayed { right } else { left }; + assert!(accepted.commit_version.is_some()); + let replay = executor.execute(&redelivery).unwrap(); + assert!(replay.replayed); + assert_eq!(replay.result, accepted.result); + assert_eq!(replay.commit_version, accepted.commit_version); + let stale = executor.execute(&inspect(first.next_after)).unwrap_err(); + assert_eq!(stale.code, protocol::ErrorCode::Conflict); + assert_eq!(stale.conflict, Some(protocol::ConflictKind::OperationState)); + let current = executor + .load_publish_operation( + route(1), + store.current_read_version().unwrap(), + child.operation_id.into(), + ) + .unwrap(); + assert_eq!(current.cleanup_staged_object_cursor, 32); + assert_eq!(current.cleanup_retry_count, 1); + service + .transition_publish(meta::TransitionPublishRequest { + context: executor + .publication_context(route(1), types::RequestId::from_bytes([0x9a; 16])) + .unwrap(), + expected_operation: current, + transition: meta::PublishTransition::Quarantine { + terminal_error: quarantine_error, + }, + }) + .unwrap(); + assert_ne!(query().token, token); + assert!(executor.execute(&redelivery).unwrap().replayed); + assert_eq!( + query().state, + protocol::OperationState::Quarantined, + "redelivery cannot restart a later quarantine" + ); + let mut misuse = retry; + let protocol::WorkspaceRequest::RetryAppendCleanup(request) = &mut misuse.operation else { + unreachable!() + }; + request.token = query().token; + assert_eq!( + executor.execute(&misuse).unwrap_err().code, + protocol::ErrorCode::RequestReplayMismatch + ); + } + #[test] fn stable_append_rejects_pending_replans_and_intent_reuse_without_cleanup() { let (store, executor) = ready_executor(); diff --git a/crates/nokv-server/src/lifecycle.rs b/crates/nokv-server/src/lifecycle.rs index 3534ceacd..9838e0854 100644 --- a/crates/nokv-server/src/lifecycle.rs +++ b/crates/nokv-server/src/lifecycle.rs @@ -731,8 +731,9 @@ impl LifecycleRunner { let sequence = operation.cleanup_staged_object_cursor + u32::try_from(offset).expect("bounded batch offset fits u32"); let expected = self.read_staged_object(operation.operation_id, sequence)?; - if matches!(expected.provider_state, StagedProviderState::Ambiguous) - || matches!(expected.cleanup_state, StagedCleanupState::Quarantined) + if operation.append_attempt.is_none() + && (matches!(expected.provider_state, StagedProviderState::Ambiguous) + || matches!(expected.cleanup_state, StagedCleanupState::Quarantined)) { return self.quarantine_publication( operation, @@ -741,6 +742,21 @@ impl LifecycleRunner { ); } let append = operation.append_attempt.is_some(); + if expected.object_sequence != sequence + || expected.artifact_revision_id != operation.artifact_revision_id + || expected.object_key + != meta::object_block_key( + self.route.logical_shard_id.into(), + self.root_id(), + operation.artifact_revision_id, + u64::from(sequence), + ) + { + return Err(corrupt( + "staged object", + "cleanup key is outside the admitted root/revision/sequence", + )); + } let completed_cleanup = if append { StagedCleanupState::Sealed } else { @@ -754,6 +770,23 @@ impl LifecycleRunner { #[cfg(test)] self.run_before_provider_delete_test_hook(); let _operation_permit = self.enter_destructive_operation()?; + if append { + let encoded = operation + .encode() + .map_err(|error| corrupt("publish operation", error.to_string()))?; + let context = self + .publication_context(b"append-cleanup-provider-authority", &encoded)?; + match service.validate_append_cleanup_authority(context, &operation) { + Ok(()) => {} + Err(error) if publication_concurrent(&error) => { + report.deferred_operations += 1; + return Ok(()); + } + Err(error) => { + return Err(state("authorize append object sealing", error)) + } + } + } let request = LifecycleCleanupRequest { purpose: if append { LifecycleCleanupPurpose::AbortedAppend @@ -3106,6 +3139,7 @@ mod tests { dependency_digest: meta::dependency_owner_digest(&[]).unwrap(), cleanup_staged_object_cursor: 0, cleanup_manifest_cursor: 0, + cleanup_retry_count: 0, publication_absence_proof: None, result: None, terminal_error: None, @@ -3417,6 +3451,7 @@ mod tests { dependency_digest: meta::dependency_owner_digest(&[]).unwrap(), cleanup_staged_object_cursor: 0, cleanup_manifest_cursor: 0, + cleanup_retry_count: 0, publication_absence_proof: None, result: None, terminal_error: None, @@ -3704,6 +3739,7 @@ mod tests { dependency_digest: meta::dependency_owner_digest(&[]).unwrap(), cleanup_staged_object_cursor: 0, cleanup_manifest_cursor: 0, + cleanup_retry_count: 0, publication_absence_proof: None, result: None, terminal_error: None, @@ -4002,6 +4038,7 @@ mod tests { dependency_digest: meta::dependency_owner_digest(&[]).unwrap(), cleanup_staged_object_cursor: 0, cleanup_manifest_cursor: 0, + cleanup_retry_count: 0, publication_absence_proof: None, result: None, terminal_error: None, diff --git a/crates/nokv/src/backend.rs b/crates/nokv/src/backend.rs index 428c7f34a..2e7783e0c 100644 --- a/crates/nokv/src/backend.rs +++ b/crates/nokv/src/backend.rs @@ -2568,7 +2568,7 @@ fn decode_list_cursor( _ => { return Err(invalid_backend_input( "list cursor has an unknown fence kind", - )) + )); } }; let (scope_digest, anchor) = payload @@ -2783,10 +2783,37 @@ pub(crate) fn map_append_client_error(error: ClientError) -> agent::BackendError json!({}), ); } + if matches!(&error, ClientError::Rpc(failure) if failure.code == wire::ErrorCode::InvalidArgument) + { + let mut mapped = map_client_error(error); + mapped.kind = agent::BackendErrorKind::Other("InvalidArgument".to_owned()); + return mapped; + } map_client_error(error) } fn map_client_error(error: ClientError) -> agent::BackendError { + if let ClientError::AppendCleanupUnresolved { + operation_id, + expected_token, + receipt, + source, + } = error + { + let mut mapped = map_client_error(*source); + mapped.kind = agent::BackendErrorKind::Other("AppendCleanupUnresolved".to_owned()); + mapped.retryable = false; + mapped.details["operation_id"] = json!(super::encode_lowercase_hex(&operation_id.0)); + mapped.details["expected_state_digest"] = + json!(super::encode_lowercase_hex(&expected_token.state_digest.0)); + mapped.details["next_action"] = json!("retry_same_cleanup"); + mapped.details["publication_operation_id"] = json!(receipt + .as_ref() + .map(|value| super::encode_lowercase_hex(&value.publication_operation_id.0))); + mapped.details["recovery_receipt"] = + json!(receipt.as_deref().map(super::cleanup_retry_receipt_json)); + return mapped; + } if let ClientError::AppendUnresolved { operation_id, state, @@ -2835,7 +2862,9 @@ fn map_client_error(error: ClientError) -> agent::BackendError { ClientError::ArtifactPublishFailed { .. } | ClientError::RetryExhausted { .. } => { agent::BackendErrorKind::Other("ClientFailure".to_owned()) } - ClientError::AppendUnresolved { .. } => unreachable!("append failure returned above"), + ClientError::AppendUnresolved { .. } | ClientError::AppendCleanupUnresolved { .. } => { + unreachable!("append failure returned above") + } ClientError::Rpc(_) => unreachable!("RPC failures returned above"), } }; @@ -7138,6 +7167,74 @@ mod tests { assert!(!error.retryable); } + #[test] + fn unresolved_cleanup_retains_exact_request_and_known_historical_receipt() { + let operation_id = wire::OperationIdentity([0xaa; 16]); + let token = wire::OperationToken { + operation_id, + state_digest: wire::Digest([0xbb; 32]), + }; + let source = || { + ClientError::Rpc(wire::RpcFailure { + code: wire::ErrorCode::NotOwner, + message: "owner changed".to_owned(), + retryable: true, + conflict: None, + current_generation: None, + route_hint: None, + }) + }; + for receipt in [ + None, + Some(Box::new(wire::AppendCleanupRetryResult { + operation_id, + publication_operation_id: wire::OperationIdentity([0xcc; 16]), + cleanup_retry_count: 1, + expected_state_digest: token.state_digest, + })), + ] { + let known = receipt.is_some(); + let mapped = map_append_client_error(ClientError::AppendCleanupUnresolved { + operation_id, + expected_token: token, + receipt, + source: Box::new(source()), + }); + assert_eq!( + mapped.kind, + agent::BackendErrorKind::Other("AppendCleanupUnresolved".to_owned()) + ); + assert_eq!(mapped.details["expected_state_digest"], "bb".repeat(32)); + assert_eq!(mapped.details["next_action"], "retry_same_cleanup"); + assert_eq!(mapped.details["code"], "NotOwner"); + assert_eq!(mapped.details["recovery_receipt"].is_null(), !known); + assert!(!mapped.retryable); + let error = super::super::append_error_json( + operation_id, + super::super::agent_error(mapped.into()), + ); + let json: Value = serde_json::from_str(&error).unwrap(); + assert_eq!(json["details"]["next_action"], "retry_same_cleanup"); + assert_eq!(json["details"]["cause_code"], "NotOwner"); + assert_eq!( + json["details"]["publication_operation_id"].is_null(), + !known + ); + } + let invalid = map_append_client_error(ClientError::Rpc(wire::RpcFailure { + code: wire::ErrorCode::InvalidArgument, + message: "invalid inspection cursor".to_owned(), + retryable: false, + conflict: None, + current_generation: None, + route_hint: None, + })); + assert_eq!( + invalid.kind, + agent::BackendErrorKind::Other("InvalidArgument".to_owned()) + ); + } + #[test] fn exhausted_sdk_append_conflict_remains_a_retryable_workbench_conflict() { let mapped = map_client_error(ClientError::RetryExhausted { diff --git a/crates/nokv/src/cli.rs b/crates/nokv/src/cli.rs index a1fccd876..2c8505c2e 100644 --- a/crates/nokv/src/cli.rs +++ b/crates/nokv/src/cli.rs @@ -10,6 +10,9 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::str::FromStr; +use base64::engine::general_purpose::STANDARD; +use base64::Engine as _; + pub const DEFAULT_METADATA_ADDRESS: &str = "127.0.0.1:7750"; pub const DEFAULT_SERVER_BIND: &str = "127.0.0.1:7750"; pub const DEFAULT_MAX_ARTIFACT_BYTES: usize = 16 * 1024 * 1024; @@ -138,6 +141,15 @@ pub enum Command { OperationStatus { operation_id: [u8; 16], }, + OperationInspect { + operation_id: [u8; 16], + limit: u32, + cursor: Option>, + }, + OperationRecover { + operation_id: [u8; 16], + expected_state_digest: Option<[u8; 32]>, + }, Provision { logical_shard_id: String, adopt_legacy_object_namespace: bool, @@ -203,10 +215,23 @@ pub enum CliError { UnknownOption(String), UnknownCommand(String), UnexpectedArgument(String), - InvalidNumber { option: &'static str, value: String }, - InvalidAddress { option: &'static str, value: String }, - InvalidOption { option: &'static str, value: String }, + InvalidNumber { + option: &'static str, + value: String, + }, + InvalidAddress { + option: &'static str, + value: String, + }, + InvalidOption { + option: &'static str, + value: String, + }, InvalidRequestId(String), + OperationInput { + operation_id: [u8; 16], + message: String, + }, MixedRoutingOptions, MixedMetadataStoreOptions, LocalOnlyRecoverLog, @@ -237,6 +262,7 @@ impl fmt::Display for CliError { formatter, "--request-id must be exactly 32 lowercase hexadecimal characters, got {value:?}" ), + Self::OperationInput { message, .. } => formatter.write_str(message), Self::MixedRoutingOptions => formatter.write_str( "static metadata routing options and etcd routing options cannot be combined", ), @@ -507,7 +533,7 @@ pub fn parse(arguments: impl IntoIterator) -> Result) -> Result) -> Result, +) -> Result { + let mut limit = None; + let mut cursor = None; + let mut expected_state_digest = None; + while let Some(argument) = arguments.next() { + match argument.as_str() { + "--limit" if operation == "inspect" && limit.is_none() => { + let value = next_value(arguments, &argument)?; + let parsed = parse_number::("--limit", value.clone())?; + if !(1..=nokv_client::MAX_APPEND_INSPECTION_LIMIT).contains(&parsed) { + return Err(CliError::InvalidOption { + option: "--limit", + value, + }); + } + limit = Some(parsed); + } + "--cursor" if operation == "inspect" && cursor.is_none() => { + let value = next_value(arguments, &argument)?; + let maximum = nokv_protocol::PageRequest::MAX_CURSOR_BYTES; + if value.len() > maximum.div_ceil(3) * 4 { + return Err(CliError::InvalidOption { + option: "--cursor", + value, + }); + } + let decoded = STANDARD + .decode(&value) + .map_err(|_| CliError::InvalidOption { + option: "--cursor", + value: value.clone(), + })?; + if decoded.len() > maximum { + return Err(CliError::InvalidOption { + option: "--cursor", + value, + }); + } + cursor = Some(decoded); + } + "--expected-state-digest" + if operation == "recover" && expected_state_digest.is_none() => + { + let value = next_value(arguments, &argument)?; + expected_state_digest = + Some(decode_fixed_hex(&value).ok_or(CliError::InvalidOption { + option: "--expected-state-digest", + value, + })?); + } + _ => return Err(CliError::UnexpectedArgument(argument)), + } + } + if operation == "inspect" { + Ok(Command::OperationInspect { + operation_id, + limit: limit.unwrap_or(nokv_client::DEFAULT_APPEND_INSPECTION_LIMIT), + cursor, + }) + } else { + Ok(Command::OperationRecover { + operation_id, + expected_state_digest, + }) + } +} + fn parse_mcp(arguments: &mut impl Iterator) -> Result { let mut profile = None; while let Some(argument) = arguments.next() { @@ -710,7 +816,7 @@ fn parse_workspace_path(arguments: &mut impl Iterator) -> Result< _ => { return Err(CliError::UnknownCommand(format!( "workspace-path {operation}" - ))) + ))); } }; @@ -945,20 +1051,24 @@ fn parse_address(option: &'static str, value: String) -> Result Result<[u8; 16], CliError> { - if value.len() != 32 + decode_fixed_hex(&value).ok_or(CliError::InvalidRequestId(value)) +} + +fn decode_fixed_hex(value: &str) -> Option<[u8; WIDTH]> { + if value.len() != WIDTH * 2 || !value .as_bytes() .iter() .copied() .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) { - return Err(CliError::InvalidRequestId(value)); + return None; } - let mut decoded = [0_u8; 16]; + let mut decoded = [0_u8; WIDTH]; for (index, pair) in value.as_bytes().as_chunks::<2>().0.iter().enumerate() { decoded[index] = (hex_nibble(pair[0]) << 4) | hex_nibble(pair[1]); } - Ok(decoded) + Some(decoded) } fn hex_nibble(byte: u8) -> u8 { @@ -1154,6 +1264,83 @@ mod tests { .is_err()); } + #[test] + fn operation_inspection_and_recovery_preserve_cursor_and_exact_token() { + let parse_command = |tail: &[&str]| { + let mut input = vec![ + "--agent-id", + "44444444444444444444444444444444", + "operation", + ]; + input.extend_from_slice(tail); + parse(args(&input)) + }; + let identity = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let inspected = parse_command(&["inspect", identity]).unwrap(); + assert_eq!( + inspected.command, + Command::OperationInspect { + operation_id: [0xaa; 16], + limit: 32, + cursor: None, + } + ); + assert!(inspected.client.object.bucket.is_none()); + assert!(inspected.workbench_root.is_none()); + assert_eq!( + parse_command(&["inspect", identity, "--limit", "192", "--cursor", "AQID"]) + .unwrap() + .command, + Command::OperationInspect { + operation_id: [0xaa; 16], + limit: 192, + cursor: Some(vec![1, 2, 3]) + } + ); + assert_eq!( + parse_command(&["recover", identity]).unwrap().command, + Command::OperationRecover { + operation_id: [0xaa; 16], + expected_state_digest: None + } + ); + assert_eq!( + parse_command(&[ + "recover", + identity, + "--expected-state-digest", + &"bb".repeat(32) + ]) + .unwrap() + .command, + Command::OperationRecover { + operation_id: [0xaa; 16], + expected_state_digest: Some([0xbb; 32]) + } + ); + for tail in [ + vec!["inspect", identity, "--limit", "0"], + vec!["inspect", identity, "--limit", "193"], + vec!["inspect", identity, "--cursor", "%%%"], + vec!["inspect", identity, "--limit", "1", "--limit", "2"], + vec!["recover", identity, "--expected-state-digest", "aa"], + vec!["recover", identity, "--limit", "1"], + ] { + assert!( + matches!(parse_command(&tail), Err(CliError::OperationInput { operation_id, .. }) if operation_id == [0xaa; 16]), + "{tail:?}" + ); + } + assert!(parse_command(&["inspect", identity, "--cursor", &"A".repeat(6000)]).is_err()); + assert!(parse_command(&[ + "recover", + identity, + "--expected-state-digest", + &"BB".repeat(32) + ]) + .is_err()); + } + #[test] fn workspace_append_requires_one_payload_and_stable_identity() { let prefix = ["append", "run-42", "logs", "events.jsonl"]; diff --git a/crates/nokv/src/main.rs b/crates/nokv/src/main.rs index 24dcd03e1..98edab4bd 100644 --- a/crates/nokv/src/main.rs +++ b/crates/nokv/src/main.rs @@ -61,7 +61,16 @@ fn main() -> ExitCode { } fn run() -> Result<(), String> { - let invocation = cli::parse(std::env::args().skip(1)).map_err(|error| error.to_string())?; + let invocation = cli::parse(std::env::args().skip(1)).map_err(|error| match error { + cli::CliError::OperationInput { + operation_id, + message, + } => append_error_json( + nokv_protocol::OperationIdentity(operation_id), + append_input_error(message), + ), + error => error.to_string(), + })?; if matches!(invocation.command, Command::Mcp { .. }) { // stderr only: stdout carries the line-delimited JSON-RPC stream. eprintln!( @@ -149,6 +158,24 @@ fn run() -> Result<(), String> { Command::OperationStatus { operation_id } => { run_operation_status(&invocation, nokv_protocol::OperationIdentity(*operation_id)) } + Command::OperationInspect { + operation_id, + limit, + cursor, + } => run_operation_inspect( + &invocation, + nokv_protocol::OperationIdentity(*operation_id), + *limit, + cursor.clone(), + ), + Command::OperationRecover { + operation_id, + expected_state_digest, + } => run_operation_recover( + &invocation, + nokv_protocol::OperationIdentity(*operation_id), + *expected_state_digest, + ), } } @@ -548,9 +575,15 @@ fn run_operation_status( fn append_status_json( call: &nokv_client::ClientCall, ) -> Result { - let recovery = nokv_client::append_operation_recovery(&call.value) + let mut result = append_operation_json(&call.value)?; + result["commit_version"] = json!(call.commit_version); + result["replayed"] = json!(call.replayed); + Ok(result) +} + +fn append_operation_json(status: &nokv_protocol::OperationStatus) -> Result { + let recovery = nokv_client::append_operation_recovery(status) .map_err(|error| agent_error(backend::map_append_client_error(error).into()))?; - let status = &call.value; let preparation = status .append_preparation .as_ref() @@ -571,6 +604,7 @@ fn append_status_json( "observed_state": format!("{:?}", status.state), "publication_operation_id": encode_lowercase_hex(&preparation.publication_operation_id.0), "attempt": preparation.attempt, + "cleanup_retry_count": preparation.cleanup_retry_count, "attempt_phase": preparation.attempt_phase, "activity_deadline_ms": preparation.activity_deadline_ms, "workspace_incarnation_id": encode_lowercase_hex(&preparation.workspace_incarnation_id.0), @@ -587,11 +621,96 @@ fn append_status_json( "current_generation": failure.current_generation, })), "receipt": receipt, - "commit_version": call.commit_version, - "replayed": call.replayed, })) } +fn operation_token_json(token: nokv_protocol::OperationToken) -> Value { + json!({ + "operation_id": encode_lowercase_hex(&token.operation_id.0), + "state_digest": encode_lowercase_hex(&token.state_digest.0), + }) +} + +fn cleanup_retry_receipt_json(receipt: &nokv_protocol::AppendCleanupRetryResult) -> Value { + json!({ + "operation_id": encode_lowercase_hex(&receipt.operation_id.0), + "publication_operation_id": encode_lowercase_hex(&receipt.publication_operation_id.0), + "cleanup_retry_count": receipt.cleanup_retry_count, + "expected_state_digest": encode_lowercase_hex(&receipt.expected_state_digest.0), + }) +} + +fn append_inspection_json( + call: &nokv_client::ClientCall, +) -> Result { + let inspection = &call.value.inspection; + let mut result = append_operation_json(&inspection.operation)?; + result["action"] = json!("inspect"); + result["operation_token"] = operation_token_json(inspection.operation.token); + result["publication_token"] = operation_token_json(inspection.publication_token); + result["object_namespace_id"] = json!(encode_lowercase_hex(&inspection.object_namespace_id.0)); + result["registered_count"] = json!(inspection.registered_count); + result["cleanup_cursor"] = json!(inspection.cleanup_cursor); + result["remaining_count"] = json!(inspection.remaining_count); + result["entries"] = json!(inspection.entries); + result["next_cursor"] = json!(call + .value + .next_cursor + .as_ref() + .map(|cursor| STANDARD.encode(cursor))); + result["commit_version"] = json!(call.commit_version); + result["replayed"] = json!(call.replayed); + Ok(result) +} + +fn append_recovery_json( + call: &nokv_client::ClientCall, +) -> Result { + let mut result = append_operation_json(&call.value.operation)?; + result["action"] = json!("recover"); + result["operation_token"] = operation_token_json(call.value.operation.token); + result["requested"] = json!(call.value.requested); + result["recovery_receipt"] = json!(call.value.receipt.as_ref().map(cleanup_retry_receipt_json)); + result["commit_version"] = json!(call.commit_version); + result["replayed"] = json!(call.replayed); + Ok(result) +} + +fn run_operation_inspect( + invocation: &Invocation, + operation_id: nokv_protocol::OperationIdentity, + limit: u32, + cursor: Option>, +) -> Result<(), String> { + (|| { + let client = build_workspace_path_client(invocation)?; + let call = client + .inspect_append_operation(operation_id, nokv_protocol::PageRequest { cursor, limit }) + .map_err(|error| agent_error(backend::map_append_client_error(error).into()))?; + print_json(&append_inspection_json(&call)?) + })() + .map_err(|message| append_error_json(operation_id, message)) +} + +fn run_operation_recover( + invocation: &Invocation, + operation_id: nokv_protocol::OperationIdentity, + expected_state_digest: Option<[u8; 32]>, +) -> Result<(), String> { + (|| { + let client = build_workspace_path_client(invocation)?; + let expected = expected_state_digest.map(|digest| nokv_protocol::OperationToken { + operation_id, + state_digest: nokv_protocol::Digest(digest), + }); + let call = client + .recover_append_operation(operation_id, expected) + .map_err(|error| agent_error(backend::map_append_client_error(error).into()))?; + print_json(&append_recovery_json(&call)?) + })() + .map_err(|message| append_error_json(operation_id, message)) +} + fn append_input_error(message: String) -> String { agent_error(nokv_agent::AgentError::backend( "InvalidArgument", @@ -618,9 +737,13 @@ fn append_error_json(operation_id: nokv_protocol::OperationIdentity, message: St nokv_agent::AgentError::backend("AppendFailed", message, false, json!({})).as_value() }); error["details"]["operation_id"] = json!(encode_lowercase_hex(&operation_id.0)); - error["details"]["next_action"] = json!("query_same"); + if error["details"]["next_action"].is_null() { + error["details"]["next_action"] = json!("query_same"); + } error["details"]["cause_code"] = error["details"]["code"].clone(); - error["details"]["publication_operation_id"] = Value::Null; + if error["details"]["publication_operation_id"].is_null() { + error["details"]["publication_operation_id"] = Value::Null; + } serde_json::to_string(&error).expect("append error contains only JSON values") } @@ -843,6 +966,8 @@ USAGE: nokv [connection/object options] materialize
nokv [connection/object options] collect
[--replace] [--expected-generation N] [--content-type TYPE] nokv [route/agent options] operation status + nokv [route/agent options] operation inspect [--limit N] [--cursor BASE64] + nokv [route/agent options] operation recover [--expected-state-digest HEX64] nokv [route/agent options] workspace-path rename
--expected-generation N --request-id HEX32 nokv [route/agent options] workspace-path remove
--expected-generation N --request-id HEX32 nokv [route/agent/object options] workspace-path append
--operation-id HEX32 (--text TEXT | --base64 BASE64 | --file PATH) [--content-type TYPE] [--max-logical-size N] [--block-size N] [--expected-workspace-incarnation-id HEX32] @@ -854,7 +979,7 @@ USAGE: AGENT CONTROL ROUTING: --root-id HEX32 - --agent-id HEX32 is required by provision, workbench, mcp, materialize, collect, workspace-path, and operation status + --agent-id HEX32 is required by provision, workbench, mcp, materialize, collect, workspace-path, and operation commands AgentId is a durable deployment identity used to prevent root misconfiguration; it is not an authentication credential --metadata-address HOST:PORT --logical-shard-id HEX32 --object-namespace-id HEX32 --placement-generation N --owner-epoch N @@ -875,6 +1000,9 @@ AGENT PRESENTATION: workspace-path rename/remove require an explicit lowercase HEX32 request id for exact cross-process replay workspace-path append requires a stable lowercase HEX32 logical operation id operation status observes append recovery without a payload, workspace, or object-store connection + operation inspect returns retained staged entries, default 32 per page (maximum 192); reuse its opaque cursor + save inspect.operation_token.state_digest before operation recover and reuse it after an unknown reply + operation recover asks the owner to retry quarantined cleanup; it does not resubmit the append payload the first append requires an existing workspace; replays resolve its original incarnation append delta and default total body limits are 16 MiB; --max-logical-size raises the bound body limit append returns the original publication receipt on replay; retain the id and incarnation across restarts @@ -1509,6 +1637,7 @@ mod tests { attempt_phase: nokv_protocol::AppendAttemptPhase::Finalizing, activity_deadline_ms: 123_000, attempt_failure: None, + cleanup_retry_count: 0, })), publish_preparation: None, commit_preparation: None, @@ -1562,8 +1691,82 @@ mod tests { .unwrap() .attempt_phase = nokv_protocol::AppendAttemptPhase::Quarantined; let quarantined = append_status_json(&call).unwrap(); - assert_eq!(quarantined["next_action"], "operator_reconcile"); + assert_eq!(quarantined["next_action"], "retry_cleanup"); assert!(quarantined["receipt"].is_null()); + + let inspected = append_inspection_json(&nokv_client::ClientCall { + value: nokv_client::AppendOperationInspection { + inspection: nokv_protocol::AppendCleanupInspection { + operation: Box::new(call.value.clone()), + object_namespace_id: nokv_protocol::ObjectNamespaceIdentity([0xee; 16]), + publication_token: nokv_protocol::OperationToken { + operation_id: nokv_protocol::OperationIdentity([0xcc; 16]), + state_digest: nokv_protocol::Digest([0x33; 32]), + }, + registered_count: 3, + cleanup_cursor: 1, + remaining_count: 2, + entries: vec![nokv_protocol::StagedObject { + sequence: 1, + object_identity: nokv_protocol::ObjectIdentity::new("objects/retained-1") + .unwrap(), + expected_length: 4, + expected_digest: nokv_protocol::sha256_digest_uri(nokv_protocol::Digest( + [0x44; 32], + )), + multipart_token: None, + }], + next_after: Some(1), + }, + next_cursor: Some(vec![1, 2, 3]), + }, + commit_version: None, + replayed: false, + }) + .unwrap(); + assert_eq!( + inspected["operation_token"]["state_digest"], + "11".repeat(32) + ); + assert_eq!( + inspected["publication_token"]["operation_id"], + "cc".repeat(16) + ); + assert_eq!(inspected["remaining_count"], 2); + assert_eq!(inspected["cleanup_cursor"], 1); + assert_eq!(inspected["entries"][0]["expected_length"], 4); + assert_eq!(inspected["next_cursor"], "AQID"); + + // A replayed cleanup receipt can be older than the current operation. + call.value + .append_preparation + .as_mut() + .unwrap() + .cleanup_retry_count = 2; + let recovery = append_recovery_json(&nokv_client::ClientCall { + value: nokv_client::AppendRecoveryRequestResult { + operation: call.value.clone(), + requested: true, + receipt: Some(nokv_protocol::AppendCleanupRetryResult { + operation_id: call.value.token.operation_id, + publication_operation_id: nokv_protocol::OperationIdentity([0xcc; 16]), + cleanup_retry_count: 1, + expected_state_digest: nokv_protocol::Digest([0x55; 32]), + }), + }, + commit_version: None, + replayed: true, + }) + .unwrap(); + assert_eq!(recovery["cleanup_retry_count"], 2); + assert_eq!(recovery["recovery_receipt"]["cleanup_retry_count"], 1); + assert_eq!( + recovery["recovery_receipt"]["expected_state_digest"], + "55".repeat(32) + ); + assert_eq!(recovery["requested"], true); + assert_eq!(recovery["replayed"], true); + assert!(recovery["receipt"].is_null()); } #[test] diff --git a/docs/architecture.md b/docs/architecture.md index f60bec1b2..f999e1e80 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -107,7 +107,7 @@ This reserves both markers, places a child's delimiter rollup before its exact artifact and both before longer siblings, and prevents an exact key from being a strict prefix of another valid path key. The same normalizer/codec owns storage keys, request identities, index identities, and restore member ids. -System format version 9 retains and gates this layout. +System format version 13 retains and gates this layout. Directories are implicit. The Workbench root and five standard sections are virtual. A file stat is a point read; an implicit-directory stat is a prefix @@ -159,7 +159,9 @@ continuations retain one exact root read version. Live continuations may move to a newer root read version only while the target workspace incarnation and revision remain unchanged; target drift fails closed, and an initial bounded collection may restart in full but never merges workspace revisions. This -contract is gated by operation schema `nokv.workspace.rpc.v11`; v11 adds durable logical append parents, +contract is gated by operation schema `nokv.workspace.rpc.v12`; v12 adds +metadata-only append ledger inspection and exact-token owner cleanup retry. +V11 added durable logical append parents, predecessor-fenced publication attempts, and metadata-only append status. The initial exact append-intent binding was introduced by v10. v7 added the exact read-version fence used by path point reads, v8 added the required typed @@ -170,13 +172,13 @@ added the provider-neutral object namespace identity to every root route. Each TCP connection starts with the fixed-width, schema-neutral transport handshake v1. The client offers one exact operation schema, and the server -accepts only `nokv.workspace.rpc.v11`; the handshake version remains stable when +accepts only `nokv.workspace.rpc.v12`; the handshake version remains stable when the operation schema changes. For upgrade diagnostics, the server recognizes only operation-first envelopes from the public v2 client and the post-tag v3 client. It parses a bounded route/request header, ignores the operation, emits one schema-readable failure, and closes without dispatch. The v2 client gets -its exact v2 failure envelope; the v3 client gets a v11 failure envelope so its -decoder reports the v11/v3 schema mismatch. Unknown legacy schemas, malformed +its exact v2 failure envelope; the v3 client gets a v12 failure envelope so its +decoder reports the v12/v3 schema mismatch. Unknown legacy schemas, malformed envelopes, and malformed handshakes fail closed. This is a read-only rejection path, not a legacy response decoder or operation fallback. diff --git a/docs/development/append-product-spec.md b/docs/development/append-product-spec.md index 206563fbc..bf3f86f3d 100644 --- a/docs/development/append-product-spec.md +++ b/docs/development/append-product-spec.md @@ -23,6 +23,11 @@ They share the Rust client state machine. Owner preflight advertises `artifact_append_v1` so downstream clients can require the logical append contract explicitly. `nokv operation status` and Python `Client.operation_status` query the logical operation using metadata only. +`nokv operation inspect` / `Client.operation_inspect` inspect the current child's +retained staged ledger. `nokv operation recover` / `Client.operation_recover` +request owner-executed cleanup through `artifact_append_recovery_v1`. Neither +operation requires the delta, a current workspace binding, object credentials +on the caller, or a transcript captured before the failure. The frozen Workbench tool schemas remain unchanged. The caller must durably retain the following before its first submission: @@ -96,6 +101,12 @@ active child cannot be replaced by a newly planned child. retiring their staging rows. Its revision claim remains reserved so that neither delayed uploads nor later reuse of that revision can revive its payload. Parent receipts do not pin historical artifact bytes. +8. Retrying quarantined cleanup increments the child's durable + `cleanup_retry_count` exactly once. The exact expected logical state token + identifies a recovery request, whose original admission receipt remains + replayable after another quarantine, owner restart, or logical successor. + The counter prevents a cleanup failure from returning the child to the same + state bytes and accepting a delayed old request as a new recovery round. A definitive generation conflict during completion can put the exact losing child into fenced abort/cleanup after proving it did not publish. The client @@ -110,7 +121,7 @@ that this order matches caller start times or external queue order. | `committed` | Parent and child have an atomic successful receipt. | `none`; acknowledge the queued action. | | `pending` | The current child is uploading, finalizing, aborting, or cleaning. | `poll`; keep the same action and payload. | | `ready_to_retry` | The predecessor is durably cleaned and cannot publish. | `resubmit_same`; send the original logical ID, delta, and options. | -| `quarantined` | Automatic recovery cannot prove a safe transition. | `operator_reconcile`; preserve evidence and the original ID. | +| `quarantined` | Automatic recovery cannot prove a safe transition. | `retry_cleanup`; inspect the operation, resolve the dependency failure, and request owner cleanup using the saved state token. | | Query error or unknown transport outcome | No authoritative recovery observation was obtained. | Query the same ID again; do not infer absence or allocate a replacement action. | | Intent mismatch | The ID belongs to a different intent or lifecycle. | Correct the caller's persisted action mapping; do not mutate that ID's intent. | @@ -119,7 +130,7 @@ code, and observed wire state where known. An error envelope's conservative `query_same` action does not claim knowledge of the current child. The metadata status result supplies authoritative attempt, child ID, phase, activity deadline, original incarnation/target, current attempt failure when present, -and receipt. An attempt failure describes its child, not a terminal failure of +cleanup retry count, and receipt. An attempt failure describes its child, not a terminal failure of the logical action. Applications must use structured fields, not parse human-readable messages. @@ -155,25 +166,10 @@ exact empty-body replay still cannot authorize publication by a terminal child. revision claim remains permanently associated with its terminal operation, including after a logical successor succeeds. The ordinary collector for published revisions cannot claim that unpublished reserved revision. The -operator reconciliation API requires the append-specific `provider_objects_sealed` -verdict; its older absence verdict cannot bypass this contract. An ambiguous -provider result cannot be converted into an unproved successful cleanup. - -For a quarantined append, the operator first captures the logical status and -exact quarantined child token, then inspects that child's persisted staged -records and reserved revision in the metadata snapshot. Every registered key -must be addressed in its bound object namespace; guessing keys from a live -file listing is insufficient. For each key, record HEAD evidence. If absent, -conditionally PUT an empty body with `If-None-Match: *`; otherwise use its ETag -with `If-Match`. Reobserve on a conditional conflict, and require a final HEAD -showing an existing zero-byte object. Do not DELETE during this procedure. -Preserve the key-by-key transcript and its SHA-256 digest. Submit -`ReconcileQuarantinedArtifactPublish` with that exact child token, -`provider_objects_sealed`, a reason, and the transcript's `evidence_digest`. -The RPC checks the metadata half of the verdict; it does not execute the -provider repair or independently certify the operator's transcript. A changed -token or any unproved key requires a fresh inspection. Finally query the -logical ID and follow its returned action using the original intent. +generic publication reconciliation API cannot resolve a stable append by +accepting a caller's claim of absence or publication. The earlier unreleased +append-specific sealed-object verdict is removed. An ambiguous provider result +cannot be converted into an unproved successful cleanup. Object-provider admission for stable append includes sealing conformance in addition to immutable creation. Provider receipt verification is bound to the @@ -195,6 +191,75 @@ feature do not certify deletion of historical provider versions. Ordinary publication cleanup remains a separate lifecycle contract; this feature's seal semantics apply to stable append attempts. +## Public inspection and recovery + +Inspection starts with the logical operation ID and returns its exact logical +token, current publication token, object namespace, phase, and counters. The +`registered_count` is the actual staged cursor, not the publication plan's +intended object count. The `cleanup_cursor` counts rows retired after proven +sealing; `remaining_count = registered_count - cleanup_cursor`. Each entry +contains its sequence, complete object identity, expected length and digest, +and multipart token when applicable. These entries describe currently retained +staging metadata, not a history of every object ever used by the logical action. +Published attempts can still retain their staging rows: an inspection result +alone does not authorize cleanup of those live objects. + +The default page limit is 32 and the maximum is 192. A page has exactly the +bounded contiguous portion of `[cleanup_cursor, registered_count)` selected by +its cursor. The opaque continuation binds the root, logical ID, exact state +token and last ordinal. Every page checks the current parent, current child +and ledger rows. Unpublished children also require their exact revision +reservation; successful publication has already released that reservation. +State or child changes invalidate the +continuation; the caller must restart from the first page. Missing or mismatched +rows fail closed instead of producing an incomplete page described as complete. +Cursor checksums detect accidental damage; server-side fencing supplies the +authority. Inspection performs no provider I/O or metadata mutation. + +For example, an operator investigates action `ab...` after a provider outage: + +```text +nokv operation inspect ab... --limit 32 +nokv operation recover ab... --expected-state-digest +nokv operation status ab... +``` + +Use full 32-character operation IDs and the full 64-character state digest. +Persist `operation_token.state_digest` from inspection before requesting +recovery. Repeating `recover` with that same digest replays the same durable +cleanup admission, even if the first response was lost and the owner has since +failed cleanup again. The response's `recovery_receipt` identifies that original +round, while its operation fields report the current observation. `requested` +means the request has a durable admission receipt, including replay; +`replayed` distinguishes replay from fresh admission. Neither field means the +append committed or cleanup finished. + +Omitting the expected digest explicitly requests recovery against the currently +observed state. Active, cleaned and committed attempts return an observation +with `requested=false` and no recovery receipt. A quarantined attempt can start +one new round. One invocation never silently selects another state token after +a race or uncertain response. For queue-driven operator automation, retain the +explicit digest so retrying a request cannot accidentally start a later round. +An unresolved request error preserves the logical ID, expected digest and any +known admission receipt, with `retry_same_cleanup` as the next action. + +The owner verifies the exact parent/child binding, permanent revision claim, +revision absence and root/owner fences before atomically re-enqueuing cleanup. +The durable command receipt retains the original failure for audit. The +existing lifecycle worker performs and verifies conditional seals, advances +cleanup cursors, and records durable proof. Operator clients do not write seals +or supply a provider verdict. Cleanup of an abandoned old incarnation uses +these operation/revision fences even after the workspace name disappears or +is rebound; it cannot publish into the replacement workspace. Generic +publication cleanup retains its existing authorization rules. + +After the owner reports `ready_to_retry`, the caller redelivers the original +append ID, delta and complete intent. Recovery without the payload only makes +that safe redelivery possible; it cannot synthesize the logical append result. +Repeated provider ambiguity returns the same child to quarantine with a new +retry count. A new inspection and a deliberate new token can request another +round after the underlying problem is corrected. + ## Example recovery timelines | Failure | Durable observation | Recovery | @@ -277,6 +342,8 @@ functional demand, not a benchmark target or a promised commercial SLA. | Delete failed objects and forget their keys | Rejected for stable append: a delayed PUT can recreate an untracked payload after deletion. | | Periodically rescan all failed keys forever | Can eventually reclaim finite delayed writes, but retains an ongoing history-sized scan and leaves a reopening window after every DELETE. | | Monotonic zero-byte seals plus revision reservations | Selected for failed append objects. Requires qualified conditional replacement and permanent small reservations; avoids delayed DELETE/PUT reopening races. | +| Caller seals keys and submits a trusted provider verdict | Rejected for the public append recovery workflow. Requires callers to execute storage repair correctly and cannot independently verify their claimed evidence. | +| Owner-executed cleanup with exact-token retry receipts | Selected. Reuses fenced lifecycle proofs, requires no caller payload or provider configuration, and distinguishes replaying one recovery request from starting another. | | Separate parent advance and child admission | Creates a missing-child interval and requires another recovery protocol. Atomic Begin keeps the predecessor proof and successor admission together. | | Logical parent plus fenced immutable attempts | Selected. Reuses publication durability, cleanup, reference lifetime, and owner fencing while adding durable redelivery completion. | | Content hash as the identity | Suppresses two intentional actions with the same bytes. Hashes authenticate intent; they do not identify business actions. | @@ -284,8 +351,10 @@ functional demand, not a benchmark target or a promised commercial SLA. ## Version and deployment boundary -The workspace RPC schema is `nokv.workspace.rpc.v11`, the system format is 12, -and the publication value format is 6. A parent row is a new operation kind. +The workspace RPC schema is `nokv.workspace.rpc.v12`, the system format is 13, +and the publication value format is 7. A parent row is a new operation kind; +the child includes the monotonic cleanup retry counter. These version gates +also reject the preceding unreleased append candidate's v11/system-12 layout. Older stores and protocol clients are rejected explicitly. Existing Holt stores are inspected through the locked backend's read-only open path before writable recovery, so a format rejection does not rewrite their files. There is no silent @@ -303,8 +372,10 @@ backend linked into the tested NoKV binary. ## Functional acceptance requirements -The product gate is +The core product gate is [`append_product_acceptance_gate.py`](../../scripts/workbench/append_product_acceptance_gate.py). +Public operator workflows are covered by +[`append_operations_acceptance_gate.py`](../../scripts/workbench/append_operations_acceptance_gate.py). Run its `--help` for the exact invocation. Preserve the previous frozen binary for red tests and use a separately frozen candidate binary and matching Python wheel for green tests. Do not replace a red failure oracle with a weaker green @@ -320,8 +391,8 @@ The functional matrix must include: - Staged-object and manifest boundaries, object-provider faults, actual delayed PUT arrival after cleanup, monotonic zero-byte seals, and recovery from proven failed attempts without an additional user action ID. A seal with an - unprovable outcome must quarantine, reject the old absence verdict, and - complete only after exact-token reconciliation with sealed-object evidence. + unprovable outcome must quarantine, reject caller-supplied provider verdicts, + and complete only after exact-token recovery through the fenced owner. - Metadata-only status with the delta file gone; historical append replay with its original bytes while the object provider is unavailable, after a newer live generation, removal, and workspace name reuse. @@ -330,6 +401,20 @@ The functional matrix must include: - CLI/Python/Rust semantic parity, incompatible-client/store rejection without mutation, and deterministic metadata tests for predecessor holes, parent receipt atomicity, cross-kind collisions, and maximum legal command shape. +- Public metadata-only inspection across multiple pages, a genuinely sealed + prefix, missing-row rejection, stale/cross-root/cross-child cursors, and + inspection without access to earlier transport transcripts. +- Cleanup admission response loss, fresh processes, owner reopen, concurrent + requests and repeated quarantine. The same saved token must retain its + original receipt and counter; a deliberate new token may start another round. + Active and committed attempts must remain untouched. +- A durable downstream queue retains the complete intent before dispatch and + acknowledges only the stored NoKV receipt. Kill its actual consumer after + append commit but before queue acknowledgement, then redeliver through its + actual storage adapter. Identify the harness and integration actually run. +- Old-incarnation cleanup after workspace deletion/rebinding, using metadata + integration tests where no corresponding public lifecycle entry point exists. + Do not mislabel fixture-only transitions as public black-box coverage. Each scenario checks both safety and completion. Compare the entire ordered content, expected generation increments, stable historical receipt fields, diff --git a/docs/metadata-schema.md b/docs/metadata-schema.md index 967208962..e909673ee 100644 --- a/docs/metadata-schema.md +++ b/docs/metadata-schema.md @@ -19,16 +19,16 @@ Every logical-shard store has one authoritative marker: System("schema") -> value_format_version = 1 schema_id = "nokv_workspace" - format_version = 12 + format_version = 13 ``` Startup is fail-closed: - an empty store is initialized with the exact supported marker and logical keyspace catalog; -- format-11 and older stores are rejected without writes; there is no marker-only - upgrade because format 12 adds logical append records and changes the durable - publication operation codec; +- format-12 and older stores are rejected without writes; there is no marker-only + upgrade because format 13 adds the monotonic append cleanup retry counter and + changes the durable publication operation codec; - a nonempty current store opens only when its marker, value format, and configured adapter catalog match this contract; - a missing, malformed, unknown-version, or inconsistent store is rejected. @@ -85,7 +85,7 @@ exact key is a strict prefix of another valid path key. A child/subtree prefix appends NUL, so `a` cannot match `ab`. The empty path has no `PathCurrent` record; the workspace root is synthesized from `WorkspaceCurrent`. This path key layout was introduced by system format version 8 and is retained by -version 12. +version 13. The one shared normalizer enforces: @@ -106,21 +106,24 @@ float, timestamp, bytes, and string values. ## Durable Format Registry -`System.format_version` is `12`. Version 12 adds logical append parents, +`System.format_version` is `13`. Version 13 adds the monotonic append cleanup +retry counter and durable exact-token retry receipts in CommandDedupe. +Version 12 introduced logical append parents, publication-attempt bindings, and the active-publication recovery index in the existing Operation family. It retains the format-11 full SHA-256 append intent, format-10 Generic index families, and format-9 RecoveryOutbox fixed-width decimal LSN keys. Existing logical recovery and object formats are unchanged. -Ordinary open does not migrate a format-11 or older marker. A marker-only +Ordinary open does not migrate a format-12 or older marker. A marker-only upgrade would reinterpret publication records without their required codec and omit the active index. Migration remains not qualified; every older or unknown marker is fail-closed and unchanged. Durable codecs are independently versioned: publish operation, staged-object, and manifest-row records use value version -`6` and reject version `5`; an append publication binds both the full 256-bit -intent and logical parent/attempt. `AppendOperationRecord` uses value version +`7` and reject version `6`; an append publication binds both the full 256-bit +intent and logical parent/attempt, with a cleanup retry counter that prevents +repeated quarantine from recreating an earlier state token. `AppendOperationRecord` uses value version `1` and retains the current attempt and optional compact receipt. An `Operation(ActivePublish)` marker has the exact one-byte value `[1]`. Other publication rows retain their existing layouts. @@ -707,7 +710,8 @@ The mutually exclusive operation transitions are: ```text Uploading -> Finalizing -> Published Uploading -> Aborting -> Cleaning -> Cleaned - -> Quarantined -> Cleaned # operator reconcile + -> Quarantined -> Cleaned # generic publication reconcile + Quarantined -> Cleaning # append owner cleanup retry Finalizing -> Aborting # fenced proof of no path/dedupe publication ``` @@ -726,9 +730,14 @@ through conditional creation or ETag-conditional replacement, without DELETE. Only after sealing may cleanup remove their staging rows and reach `Cleaned`. The child's `ArtifactRevisionClaim` is retained permanently, including when a successor becomes current, to reserve and explain those provider keys. A late -immutable PUT cannot replace the seal. Reconciliation of a quarantined append -requires the distinct `ProviderObjectsSealed` verdict and preserves that claim; -`ProviderObjectsAbsent` is insufficient. Generic publications and published +immutable PUT cannot replace the seal. Recovery of a quarantined append +requires exact-token `RetryAppendCleanup`, which increments its durable counter +and re-enqueues owner cleanup. The stable admission receipt retains the original +failure and is replayable after later cleanup progress or another quarantine. +Only the fenced owner supplies sealing proof; caller-supplied absence, +publication or sealed-object verdicts cannot bypass this contract. +Metadata-only `InspectAppendCleanup` returns bounded contiguous pages of actual +retained staged rows, bound to the exact current logical/child state. Generic publications and published revision GC retain their existing deletion semantics. A late upload completion must observe the operation state; after abort it joins @@ -736,7 +745,7 @@ cleanup instead of publishing. Ambiguous multipart completion, late PUT, or DELETE remains ledger-owned and `Quarantined` until reconciled. Object listing is never used to discover staged ownership. -Reconciliation is operator-driven, never scanner-driven. The operator verifies +For generic publications, reconciliation is operator-driven, never scanner-driven. The operator verifies provider-side object state for the operation's staged keys out-of-band and presents one of two verdicts through `ReconcileQuarantinedArtifactPublish`: every staged key verified absent with diff --git a/scripts/workbench/append_operations_acceptance_gate.py b/scripts/workbench/append_operations_acceptance_gate.py new file mode 100644 index 000000000..54978f846 --- /dev/null +++ b/scripts/workbench/append_operations_acceptance_gate.py @@ -0,0 +1,1374 @@ +#!/usr/bin/env python3 +# Copyright 2024-2026 The NoKV Authors. +# SPDX-License-Identifier: Apache-2.0 +"""Public, ID-based append cleanup inspection and recovery acceptance. + +The fault proxy proves real transport boundaries. Operational recovery obtains +its ledger only from public CLI/Python inspection, never a captured Stage RPC. +""" + +from __future__ import annotations + +import argparse +import base64 +import concurrent.futures +import dataclasses +import hashlib +import json +from pathlib import Path +import shutil +import sys +import time +import traceback + +import append_product_acceptance_gate as product + +base = product.base +require = product.require +BLOCK_SIZE = 4096 +BODY_LIMIT = 4 * 1024 * 1024 + + +def operation( + stack, + action, + operation_id, + label, + *, + limit=None, + cursor=None, + expected_state_digest=None, + config=None, +): + cfg = config or stack.config + command = [ + str(stack.binary), + *base.live.control_args(cfg), + "--agent-id", + cfg.agent_id, + "operation", + action, + operation_id, + ] + if limit is not None: + command += ["--limit", str(limit)] + if cursor is not None: + command += ["--cursor", cursor] + if expected_state_digest is not None: + command += ["--expected-state-digest", expected_state_digest] + response = stack.run(command, label=label, check=False) + if response.returncode == -9: + result = {"status": "caller_killed", "operation_id": operation_id} + else: + raw = ( + response.stdout + if response.returncode == 0 + else response.stderr.strip().removeprefix("nokv: ") + ) + try: + result = json.loads(raw) + except json.JSONDecodeError as error: + raise product.ContractViolation( + f"Public operation {action} returned no structured JSON: {raw}" + ) from error + require( + (response.returncode == 0) == (result.get("status") == "success"), + "operation exit and structured receipt agree", + result, + ) + stack.record( + "public-operation-transcript.jsonl", + { + "at": base.live.now(), + "action": action, + "operation_id": operation_id, + "label": label, + "cursor": cursor, + "expected_state_digest": expected_state_digest, + "object_configuration_supplied": False, + "result": result, + }, + ) + return result + + +PYTHON_OPERATIONS_PROGRAM = r""" +import base64, hashlib, json, sys +from pathlib import Path +import nokv +from nokv import _native +c = json.load(sys.stdin) +client = nokv.Client(c["root_id"], nokv.RoutingConfig.etcd([c["etcd_endpoint"]], key_prefix=c["etcd_prefix"]), object_store=None, workbench_root=c["workbench_root"]) +try: + if c["action"] == "inspect": + cursor = None if c["cursor"] is None else base64.b64decode(c["cursor"], validate=True) + result = client.operation_inspect(c["operation_id"], cursor=cursor, limit=c["limit"]) + if result.get("next_cursor") is not None: + result["next_cursor"] = base64.b64encode(result["next_cursor"]).decode() + else: + result = client.operation_recover(c["operation_id"], expected_state_digest=c["expected_state_digest"]) + print(json.dumps({"result":result,"native_sha256":hashlib.sha256(Path(_native.__file__).read_bytes()).hexdigest(),"object_store":None})) +except Exception as error: + print(json.dumps({"result":{"status":"error","type":type(error).__name__,"message":str(error),**{field:getattr(error,field,None) for field in ("code","cause_code","operation_id","expected_state_digest","recovery_receipt","next_action","retryable")}}})) +""" + + +def python_operation( + stack, + action, + operation_id, + label, + *, + limit=32, + cursor=None, + expected_state_digest=None, +): + if stack.python_executable is None: + raise base.infra.NotQualified( + "An installed matching Python extension is required for public operations interop." + ) + config = { + "root_id": stack.root_id, + "etcd_endpoint": stack.etcd_endpoint, + "etcd_prefix": stack.etcd_prefix, + "workbench_root": stack.config.workbench_root, + "action": action, + "operation_id": operation_id, + "limit": limit, + "cursor": cursor, + "expected_state_digest": expected_state_digest, + } + response = stack.run( + [stack.python_executable, "-c", PYTHON_OPERATIONS_PROGRAM], + label=label, + input_text=json.dumps(config), + ) + value = json.loads(response.stdout) + stack.record("python-operation-transcript.jsonl", {"label": label, "result": value}) + return value["result"] + + +def await_state(stack, operation_id, predicate, label, deadline): + until, observations = time.monotonic() + deadline, [] + while True: + status = product.operation_status( + stack, operation_id, f"{label}-{len(observations):03d}" + ) + observations.append(status) + if predicate(status): + stack.json(label + ".json", observations) + return status + require( + time.monotonic() < until, + "operation reaches declared state within the recovery window", + observations, + ) + time.sleep(0.5) + + +def fixture_body(count): + return b"".join( + sequence.to_bytes(4, "big") + bytes([sequence % 251]) * (BLOCK_SIZE - 4) + for sequence in range(count) + ) + + +def create_quarantine(stack, *, label, count, fault_sequence, deadline): + workbench = "operations-" + label + operation_id = hashlib.sha256(workbench.encode()).hexdigest()[:32] + payload = fixture_body(count) + stack.cli("workbench_create", {"id": workbench}) + owner_pid = stack.owner.pid + with stack.object_proxy.lock: + require(stack.object_proxy.rule is None, "no previous provider fault is armed") + first_fault = len(stack.object_proxy.fired) + stack.object_proxy.rule = { + "method": "PUT", + "action": "seal_ack_loss_head_failure", + "path_suffix": f"/blocks/{fault_sequence:016x}", + } + caller_label = label + "-uploaded-caller" + stack.proxy.arm("stage_artifact_manifest", lambda: stack.kill_caller(caller_label)) + first = product.append( + stack, + workbench, + operation_id, + payload, + label=caller_label, + max_size=BODY_LIMIT, + block_size=BLOCK_SIZE, + ) + stack.proxy.wait_dropped() + require( + first.get("status") == "caller_killed", + "the real uploaded caller was killed", + first, + ) + active_noop = operation(stack, "recover", operation_id, label + "-active-noop") + require( + stack.owner.pid == owner_pid and stack.owner.poll() is None, + "the same healthy owner survives the real caller SIGKILL", + ) + require( + active_noop.get("status") == "success" + and active_noop.get("state") == "pending" + and active_noop.get("requested") is False + and active_noop.get("recovery_receipt") is None + and active_noop.get("cleanup_retry_count") == 0, + "fresh recovery of an active publication is a metadata-only no-op", + active_noop, + ) + status = await_state( + stack, + operation_id, + lambda s: s.get("state") == "quarantined", + label + "-quarantine", + deadline, + ) + faults = stack.object_proxy.fired[first_fault:] + require( + len(faults) == 1 + and faults[0]["fault"] == "seal_ack_loss_head_failure" + and faults[0]["request_bytes"] == 0 + and faults[0]["if_match"] + and 200 <= faults[0]["upstream_status"] < 300, + "real empty conditional seal persisted before ACK loss", + faults, + ) + require( + any( + item["method"] == "HEAD" and item["path"] == faults[0]["path"] + for item in stack.object_proxy.rejected_requests + ), + "the exact sealed key's follow-up HEAD was rejected", + ) + require( + status.get("receipt") is None + and status.get("cleanup_retry_count") == 0 + and status.get("cause_code"), + "initial quarantine has no append receipt or retry generation", + status, + ) + return { + "workbench": workbench, + "operation_id": operation_id, + "payload": payload, + "registered_count": count, + "cleanup_cursor": fault_sequence // 32 * 32, + "quarantine": status, + "fault": faults[0], + "active_noop": active_noop, + } + + +def inspect_all( + stack, fixture, label, *, limit=32, python_pages=False, reopen_after_first=False +): + operation_id, cursor, pages, entries = fixture["operation_id"], None, [], [] + while True: + caller = python_operation if python_pages and len(pages) % 2 else operation + page = caller( + stack, + "inspect", + operation_id, + f"{label}-page-{len(pages):03d}", + limit=limit, + cursor=cursor, + ) + require( + page.get("status") == "success" and page.get("action") == "inspect", + "public ledger page succeeds", + page, + ) + require( + page.get("operation_id") == operation_id, + "public page belongs to the requested logical operation", + page, + ) + require( + page["operation_token"]["operation_id"] == operation_id + and page["publication_token"]["operation_id"] + == fixture["quarantine"]["publication_operation_id"], + "public tokens bind the logical parent and admitted publication child", + page, + ) + if pages: + require( + all( + page[field] == pages[0][field] + for field in ( + "operation_token", + "publication_token", + "registered_count", + "cleanup_cursor", + "remaining_count", + "object_namespace_id", + ) + ), + "all pages bind the same exact logical/child state and ledger interval", + page, + ) + require( + len(page["entries"]) <= limit, + "public page respects its requested bound", + page, + ) + pages.append(page) + entries.extend(page["entries"]) + cursor = page.get("next_cursor") + if cursor is None: + break + require( + page["entries"] and len(pages) <= fixture["registered_count"] + 1, + "pagination advances and remains bounded", + page, + ) + if reopen_after_first and len(pages) == 1: + stack.restart() + first = pages[0] + require( + first["registered_count"] == fixture["registered_count"] + and first["cleanup_cursor"] == fixture["cleanup_cursor"] + and first["remaining_count"] + == first["registered_count"] - first["cleanup_cursor"], + "public interval describes actual registered rows and cleaned prefix", + first, + ) + require( + [row["sequence"] for row in entries] + == list(range(first["cleanup_cursor"], first["registered_count"])), + "every remaining sequence occurs exactly once across public pages", + entries, + ) + for row in entries: + block = fixture["payload"][ + row["sequence"] * BLOCK_SIZE : (row["sequence"] + 1) * BLOCK_SIZE + ] + require( + row["expected_length"] == len(block) + and row["expected_digest"] == "sha256:" + hashlib.sha256(block).hexdigest() + and row["object_identity"].endswith(f"/blocks/{row['sequence']:016x}") + and row["multipart_token"] is None, + "public ledger preserves exact expected object identity, bytes, and digest", + row, + ) + result = {"pages": pages, "entries": entries} + stack.json(label + ".json", result) + return result + + +def release_provider(stack): + with stack.object_proxy.lock: + stack.object_proxy.reject_matching = None + stack.object_proxy.rule = None + + +def cleanup_receipt(value, fixture, digest, count): + receipt = value.get("recovery_receipt") + expected = { + "operation_id": fixture["operation_id"], + "publication_operation_id": fixture["quarantine"]["publication_operation_id"], + "cleanup_retry_count": count, + "expected_state_digest": digest, + } + require( + value.get("status") == "success" + and value.get("requested") is True + and receipt == expected, + "complete cleanup acceptance receipt binds its original exact state", + value, + ) + return receipt + + +def reject_trusted_verdict(stack, fixture, inspected, label): + # This deliberate raw request is only a negative test of the surviving + # generic verdict API; all inspection/recovery inputs come from public CLI. + token = inspected["publication_token"] + wrong = base.raw_rpc( + stack, + { + "operation": "reconcile_quarantined_artifact_publish", + "request": { + "token": { + "operation_id": list(bytes.fromhex(token["operation_id"])), + "state_digest": list(bytes.fromhex(token["state_digest"])), + }, + "resolution": "provider_objects_absent", + "reason": "Acceptance verifies that append cleanup cannot trust a caller verdict.", + "evidence_digest": list(hashlib.sha256(fixture["payload"]).digest()), + }, + }, + label, + ) + require( + wrong.get("status") == "failure" + and wrong.get("body", {}).get("code") == "precondition_failed" + and wrong.get("body", {}).get("conflict") == "operation_state" + and wrong.get("body", {}).get("retryable") is False, + "trusted generic provider verdict cannot reconcile an append", + wrong, + ) + after = operation(stack, "inspect", fixture["operation_id"], label + "-unchanged") + require( + all( + after.get(field) == inspected[field] + for field in ( + "operation_token", + "publication_token", + "registered_count", + "cleanup_cursor", + "remaining_count", + ) + ), + "rejected caller verdict leaves the inspected durable state unchanged", + after, + ) + return wrong + + +def exact_input_error( + value, + operation_id, + *, + code="InvalidArgument", + cause="InvalidArgument", + conflict=None, +): + details = value.get("details", {}) + require( + value.get("status") == "error" + and value.get("code") == code + and details.get("operation_id") == operation_id + and details.get("cause_code") == cause + and details.get("next_action") == "query_same" + and (conflict is None or details.get("conflict") == conflict), + "public operation error has the precise actionable classification", + value, + ) + + +def inspection_input_rejections(stack, fixture, first_page, label): + operation_id, cursor = fixture["operation_id"], first_page["next_cursor"] + require( + cursor is not None, "negative pagination tests require a real non-final cursor" + ) + results = [] + for limit in (0, 193): + result = operation( + stack, "inspect", operation_id, f"{label}-limit-{limit}", limit=limit + ) + exact_input_error(result, operation_id, cause=None) + results.append(result) + malformed = operation( + stack, "inspect", operation_id, label + "-malformed-base64", cursor="%%%" + ) + exact_input_error(malformed, operation_id, cause=None) + results.append(malformed) + decoded = bytearray(base64.b64decode(cursor, validate=True)) + decoded[-1] ^= 1 + corrupted = operation( + stack, + "inspect", + operation_id, + label + "-corrupt-cursor", + cursor=base64.b64encode(decoded).decode(), + ) + exact_input_error(corrupted, operation_id) + results.append(corrupted) + other_id = hashlib.sha256((operation_id + "other-operation").encode()).hexdigest()[ + :32 + ] + other = operation( + stack, "inspect", other_id, label + "-wrong-logical-id", cursor=cursor + ) + exact_input_error(other, other_id) + results.append(other) + root_b = hashlib.sha256( + (stack.root_id + "inspection-other-root").encode() + ).hexdigest()[:32] + config_b = dataclasses.replace(stack.config, root_id=root_b) + stack.run( + base.live.provision_command(config_b), label=label + "-provision-other-root" + ) + stack.restart() + other_root = operation( + stack, + "inspect", + operation_id, + label + "-wrong-root", + cursor=cursor, + config=config_b, + ) + exact_input_error(other_root, operation_id) + require( + "entries" not in other_root, + "wrong-root error exposes no ledger entries", + other_root, + ) + results.append(other_root) + for action in ("inspect", "recover"): + missing = operation(stack, action, other_id, label + "-missing-" + action) + exact_input_error(missing, other_id, code="NotFound", cause="NotFound") + results.append(missing) + unchanged = operation( + stack, "inspect", operation_id, label + "-first-page-after-errors" + ) + require( + all( + unchanged.get(field) == first_page[field] + for field in ( + "operation_token", + "publication_token", + "registered_count", + "cleanup_cursor", + "remaining_count", + ) + ), + "invalid scope, cursors and limits do not mutate the original ledger", + unchanged, + ) + return results + + +def finish_append(stack, fixture, label): + operation_id = fixture["operation_id"] + result = product.append( + stack, + fixture["workbench"], + operation_id, + fixture["payload"], + label=label, + max_size=BODY_LIMIT, + block_size=BLOCK_SIZE, + ) + require( + result.get("status") == "success", + "same logical append completes after public cleanup", + result, + ) + require( + result["publication_operation_id"] + != fixture["quarantine"]["publication_operation_id"], + "append completion uses a successor after the failed child was cleaned", + result, + ) + product.exact_body( + stack, fixture["workbench"], fixture["payload"], label + "-bytes" + ) + replay = product.replay_unchanged( + stack, + fixture["workbench"], + operation_id, + fixture["payload"], + result, + label + "-replay", + max_size=BODY_LIMIT, + block_size=BLOCK_SIZE, + ) + return {"receipt": result, "replay": replay} + + +def public_quarantine_recovery( + stack, + deadline, + *, + label="public-quarantine", + count=1, + fault_sequence=0, + python_pages=False, +): + fixture = create_quarantine( + stack, + label=label, + count=count, + fault_sequence=fault_sequence, + deadline=deadline, + ) + try: + stack.restart() + before = product.inventory(stack, label + "-inventory-before-inspect") + inspected = inspect_all( + stack, fixture, label + "-inspection", python_pages=python_pages + ) + obsolete = reject_trusted_verdict( + stack, fixture, inspected["pages"][0], label + "-trusted-verdict-rejected" + ) + blocked = product.append( + stack, + fixture["workbench"], + fixture["operation_id"], + fixture["payload"], + label=label + "-quarantined-append-blocked", + max_size=BODY_LIMIT, + block_size=BLOCK_SIZE, + ) + require( + blocked.get("status") == "error" + and blocked.get("code") == "AppendUnresolved" + and blocked.get("details", {}).get("operation_id") + == fixture["operation_id"], + "append cannot silently advance a quarantined child", + blocked, + ) + still_quarantined = product.operation_status( + stack, fixture["operation_id"], label + "-still-quarantined" + ) + require( + still_quarantined.get("state") == "quarantined" + and still_quarantined.get("publication_operation_id") + == fixture["quarantine"]["publication_operation_id"] + and still_quarantined.get("cleanup_retry_count") == 0, + "quarantined append replay leaves the same child and retry generation", + still_quarantined, + ) + require( + product.inventory(stack, label + "-inventory-after-inspect") == before, + "metadata-only inspection leaves the real provider inventory unchanged", + ) + release_provider(stack) + digest = inspected["pages"][0]["operation_token"]["state_digest"] + accepted = operation( + stack, + "recover", + fixture["operation_id"], + label + "-recover", + expected_state_digest=digest, + ) + cleanup_receipt(accepted, fixture, digest, 1) + ready = await_state( + stack, + fixture["operation_id"], + lambda s: s.get("state") == "ready_to_retry" + and s.get("next_action") == "resubmit_same", + label + "-ready", + deadline, + ) + require( + ready["cleanup_retry_count"] == 1, + "one cleanup request advances one retry generation", + ready, + ) + ready_noop = operation( + stack, "recover", fixture["operation_id"], label + "-ready-noop" + ) + require( + ready_noop.get("status") == "success" + and ready_noop.get("requested") is False + and ready_noop.get("recovery_receipt") is None + and ready_noop.get("state") == "ready_to_retry" + and ready_noop.get("cleanup_retry_count") == 1, + "fresh recovery of a cleaned attempt is a metadata-only no-op", + ready_noop, + ) + sealed = product.inventory(stack, label + "-sealed-inventory") + require( + all( + sealed.get( + stack.object_root.rstrip("/") + "/" + row["object_identity"], [None] + )[0] + == 0 + for row in inspected["entries"] + ), + "owner retained a zero guard for every publicly inspected remaining key", + sealed, + ) + provider_reads = [] + for row in inspected["entries"]: + key = stack.object_root.rstrip("/") + "/" + row["object_identity"] + target = stack.evidence_dir / f"{label}-sealed-key-{row['sequence']}.data" + read = stack.run( + base.infra.aws_command( + stack.aws, + stack.object_endpoint, + "s3api", + "get-object", + "--bucket", + stack.bucket, + "--key", + key, + str(target), + ), + label=f"{label}-sealed-key-{row['sequence']}-read", + env=stack.aws_env, + ) + metadata = json.loads(read.stdout) + require( + target.read_bytes() == b"" + and metadata["ContentLength"] == 0 + and metadata.get("ETag"), + "independent provider GET proves every inspected guard is empty", + metadata, + ) + provider_reads.append( + {"key": key, "content_length": 0, "etag": metadata["ETag"]} + ) + finished = finish_append(stack, fixture, label + "-same-id-completion") + replay = operation( + stack, + "recover", + fixture["operation_id"], + label + "-same-token-replay", + expected_state_digest=digest, + ) + cleanup_receipt(replay, fixture, digest, 1) + require( + replay.get("replayed") is True, + "historical cleanup acceptance is marked as replay", + replay, + ) + return product.case_result( + safety="PASS", + completion="PASS", + fault=fixture["fault"], + initial_quarantine=fixture["quarantine"], + inspection=inspected, + recovery=accepted, + ready=ready, + active_noop=fixture["active_noop"], + ready_noop=ready_noop, + historical_cleanup_replay=replay, + obsolete_trusted_verdict=obsolete, + blocked_append=blocked, + verified_provider_reads=provider_reads, + **finished, + ) + finally: + release_provider(stack) + + +def paginated_prefix_recovery(stack, deadline): + label = "public-paged-prefix" + fixture = create_quarantine( + stack, label=label, count=257, fault_sequence=32, deadline=deadline + ) + try: + inspected = inspect_all( + stack, + fixture, + label + "-mixed-sdk-pages", + python_pages=True, + reopen_after_first=True, + ) + require( + len(inspected["pages"]) > 1, "more than one actual CLI/Python page was read" + ) + widest = inspect_all(stack, fixture, label + "-maximum-pages", limit=192) + require( + len(widest["pages"]) == 2 and widest["entries"] == inspected["entries"], + "maximum-size pagination agrees with small mixed-SDK pages", + ) + rejected_inputs = inspection_input_rejections( + stack, fixture, inspected["pages"][0], label + "-inputs" + ) + # The public first remaining key locates this controlled fixture's older + # keys; they are checked as provider guards, never fabricated ledger rows. + prefix = inspected["entries"][0]["object_identity"].rsplit("/", 1)[0] + objects = product.inventory(stack, label + "-cleaned-prefix-guards") + require( + all( + objects.get( + stack.object_root.rstrip("/") + "/" + prefix + f"/{index:016x}", + [None], + )[0] + == 0 + for index in range(32) + ), + "the durably retired prefix remains sealed", + objects, + ) + release_provider(stack) + digest = inspected["pages"][0]["operation_token"]["state_digest"] + stack.proxy.begin_rendezvous(2, operation_name="retry_append_cleanup") + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + calls = [ + pool.submit( + caller, + stack, + "recover", + fixture["operation_id"], + label + "-" + name + "-recover", + expected_state_digest=digest, + ) + for name, caller in ( + ("python", python_operation), + ("cli", operation), + ) + ] + concurrent_results = [ + call.result(timeout=stack.timeout) for call in calls + ] + require( + len(stack.proxy.rendezvous_entries) == 2, + "two real recovery RPCs arrived before either was forwarded", + ) + finally: + stack.proxy.rendezvous = None + for value in concurrent_results: + cleanup_receipt(value, fixture, digest, 1) + require( + any(value.get("replayed") is True for value in concurrent_results), + "one concurrent exact-token recovery is durably replayed", + concurrent_results, + ) + accepted = concurrent_results[0] + ready = await_state( + stack, + fixture["operation_id"], + lambda s: s.get("state") == "ready_to_retry", + label + "-ready", + deadline, + ) + guards = product.inventory(stack, label + "-all-guards") + require( + all( + guards.get( + stack.object_root.rstrip("/") + "/" + prefix + f"/{index:016x}", + [None], + )[0] + == 0 + for index in range(257) + ), + "owner seals and retains every old key across all pages", + guards, + ) + finished = finish_append(stack, fixture, label + "-complete") + replay = operation( + stack, + "recover", + fixture["operation_id"], + label + "-cli-replay", + expected_state_digest=digest, + ) + cleanup_receipt(replay, fixture, digest, 1) + require( + replay.get("replayed") is True, + "Python cleanup acceptance replays through CLI", + replay, + ) + return product.case_result( + safety="PASS", + completion="PASS", + fault=fixture["fault"], + cleaned_prefix=32, + registered_count=257, + mixed_pages=inspected, + maximum_pages=widest, + recovery=accepted, + ready=ready, + recovery_replay=replay, + concurrent_recoveries=concurrent_results, + recovery_requests_held_before_forward=2, + rejected_inputs=rejected_inputs, + **finished, + ) + finally: + release_provider(stack) + + +def recovery_ack_loss_and_aba(stack, deadline): + label = "public-recovery-aba" + fixture = create_quarantine( + stack, label=label, count=3, fault_sequence=0, deadline=deadline + ) + operation_id = fixture["operation_id"] + try: + first_pages = inspect_all(stack, fixture, label + "-first-inspection", limit=1) + first_digest = first_pages["pages"][0]["operation_token"]["state_digest"] + caller = label + "-recover-caller" + stack.proxy.arm("retry_append_cleanup", lambda: stack.kill_caller(caller)) + lost = operation( + stack, "recover", operation_id, caller, expected_state_digest=first_digest + ) + stack.proxy.wait_dropped() + require( + lost.get("status") == "caller_killed", + "real recovery acceptance ACK was lost with caller SIGKILL", + lost, + ) + dropped = stack.proxy.last_drop + require( + dropped is not None + and dropped.get("operation") == "retry_append_cleanup" + and dropped.get("delivered_to_client") is False, + "the original cleanup acceptance is captured before response loss", + dropped, + ) + original = dropped["response"]["payload"] + original_body = original["outcome"]["body"] + require( + original["outcome"]["status"] == "success" + and original_body["result"] == "append_cleanup_retried" + and original.get("replayed") is False + and isinstance(original.get("commit_version"), int) + and original["commit_version"] > 0, + "the dropped response contains the first durable cleanup acceptance", + original, + ) + original_receipt = { + field: bytes(value).hex() + if field + in ( + "operation_id", + "publication_operation_id", + "expected_state_digest", + ) + else value + for field, value in original_body["value"].items() + } + cleanup_receipt( + { + "status": "success", + "requested": True, + "recovery_receipt": original_receipt, + }, + fixture, + first_digest, + 1, + ) + again = await_state( + stack, + operation_id, + lambda s: s.get("state") == "quarantined" + and s.get("cleanup_retry_count") == 1, + label + "-quarantine-count-one", + deadline, + ) + stack.restart() + replay = operation( + stack, + "recover", + operation_id, + label + "-old-token-after-reopen", + expected_state_digest=first_digest, + ) + cleanup_receipt(replay, fixture, first_digest, 1) + require( + replay.get("replayed") is True + and replay.get("commit_version") == original["commit_version"] + and replay.get("cleanup_retry_count") == 1 + and replay.get("state") == "quarantined", + "same token cannot start another cleanup round after ABA", + replay, + ) + python_replay = python_operation( + stack, + "recover", + operation_id, + label + "-python-old-token", + expected_state_digest=first_digest, + ) + cleanup_receipt(python_replay, fixture, first_digest, 1) + require( + python_replay.get("replayed") is True + and python_replay.get("commit_version") == original["commit_version"] + and python_replay.get("cleanup_retry_count") == 1, + "Python repeats the original cleanup command receipt without advancing its counter", + python_replay, + ) + next_pages = inspect_all(stack, fixture, label + "-next-inspection", limit=1) + next_digest = next_pages["pages"][0]["operation_token"]["state_digest"] + require( + next_digest != first_digest, + "retry generation makes repeated quarantine tokens distinct", + ) + stale = operation( + stack, + "inspect", + operation_id, + label + "-stale-first-round-cursor", + cursor=first_pages["pages"][0]["next_cursor"], + ) + exact_input_error( + stale, + operation_id, + code="Conflict", + cause="Conflict", + conflict="OperationState", + ) + release_provider(stack) + old_again = operation( + stack, + "recover", + operation_id, + label + "-old-token-provider-restored", + expected_state_digest=first_digest, + ) + cleanup_receipt(old_again, fixture, first_digest, 1) + require( + old_again.get("state") == "quarantined" + and old_again.get("replayed") is True + and old_again.get("commit_version") == original["commit_version"] + and old_again.get("cleanup_retry_count") == 1, + "restoring the provider does not turn an old token replay into a new request", + old_again, + ) + accepted = operation( + stack, + "recover", + operation_id, + label + "-new-token-recovery", + expected_state_digest=next_digest, + ) + cleanup_receipt(accepted, fixture, next_digest, 2) + ready = await_state( + stack, + operation_id, + lambda s: s.get("state") == "ready_to_retry" + and s.get("cleanup_retry_count") == 2, + label + "-ready-count-two", + deadline, + ) + require( + ready.get("publication_operation_id") + == fixture["quarantine"]["publication_operation_id"] + and ready.get("attempt") == fixture["quarantine"]["attempt"] + and ready.get("attempt_phase") == "cleaned" + and ready.get("cleanup_retry_count") == 2 + and ready.get("receipt") is None + and ready.get("next_action") == "resubmit_same", + "both cleanup rounds belong to the original child before successor admission", + ready, + ) + finished = finish_append(stack, fixture, label + "-complete") + historical = operation( + stack, + "recover", + operation_id, + label + "-first-receipt-after-second-round", + expected_state_digest=first_digest, + ) + cleanup_receipt(historical, fixture, first_digest, 1) + require( + historical.get("state") == "committed" + and historical.get("publication_operation_id") + == finished["receipt"]["publication_operation_id"] + and historical.get("publication_operation_id") + != ready["publication_operation_id"] + and historical.get("attempt") == ready["attempt"] + 1 + and historical.get("attempt_phase") == "published" + and historical.get("cleanup_retry_count") == 0 + and historical.get("replayed") is True + and historical.get("commit_version") == original["commit_version"] + and historical.get("recovery_receipt") == original_receipt + and product.canonical_receipt(historical["receipt"]) + == product.canonical_receipt(finished["receipt"]), + "the published successor has its own zero cleanup count while the original recovery receipt replays unchanged", + historical, + ) + no_op = operation(stack, "recover", operation_id, label + "-committed-noop") + require( + no_op.get("status") == "success" + and no_op.get("requested") is False + and no_op.get("recovery_receipt") is None + and product.canonical_receipt(no_op["receipt"]) + == product.canonical_receipt(finished["receipt"]), + "recover of committed append is a metadata-only no-op", + no_op, + ) + return product.case_result( + safety="PASS", + completion="PASS", + initial_fault=fixture["fault"], + lost_recovery=lost, + original_cleanup_receipt=original_receipt, + original_cleanup_commit_version=original["commit_version"], + repeated_quarantine=again, + first_replay=replay, + python_replay=python_replay, + second_recovery=accepted, + ready=ready, + historical_first_receipt=historical, + committed_noop=no_op, + stale_cursor_rejected=stale, + **finished, + ) + finally: + release_provider(stack) + + +def owner_during_cleanup_seal(stack, deadline): + label = "public-owner-mid-seal" + fixture = create_quarantine( + stack, label=label, count=3, fault_sequence=0, deadline=deadline + ) + operation_id = fixture["operation_id"] + try: + inspected = inspect_all(stack, fixture, label + "-inspection") + digest = inspected["pages"][0]["operation_token"]["state_digest"] + release_provider(stack) + with stack.object_proxy.lock: + stack.object_proxy.seal_held.clear() + stack.object_proxy.seal_release.clear() + previous_faults = len(stack.object_proxy.fired) + stack.object_proxy.rule = { + "method": "PUT", + "action": "hold_seal_success", + "path_suffix": "/blocks/0000000000000001", + } + owner_pid = stack.owner.pid + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + future = pool.submit( + operation, + stack, + "recover", + operation_id, + label + "-recover-before-owner-kill", + expected_state_digest=digest, + ) + try: + require( + stack.object_proxy.seal_held.wait(stack.timeout), + "the owner reached a real successful conditional seal before its response was released", + ) + faults = stack.object_proxy.fired[previous_faults:] + require( + len(faults) == 1 + and faults[0]["fault"] == "hold_seal_success" + and faults[0]["request_bytes"] == 0 + and faults[0]["if_match"] + and 200 <= faults[0]["upstream_status"] < 300, + "real provider seal success is held at the owner crash boundary", + faults, + ) + require( + stack.owner.pid == owner_pid and stack.owner.poll() is None, + "SIGKILL targets the owner currently waiting for the seal response", + ) + stack.record( + "owner-cleanup-seal-fault.jsonl", + { + "at": base.live.now(), + "owner_pid": owner_pid, + "operation_id": operation_id, + "provider_response": faults[0], + "delivered_to_old_owner": False, + }, + ) + stack.restart() + finally: + stack.object_proxy.seal_release.set() + first = future.result(timeout=stack.timeout) + if first.get("status") == "success": + cleanup_receipt(first, fixture, digest, 1) + else: + details = first.get("details", {}) + require( + first.get("code") == "AppendCleanupUnresolved" + and details.get("operation_id") == operation_id + and details.get("expected_state_digest") == digest + and details.get("next_action") == "retry_same_cleanup", + "unknown recovery observation preserves the exact cleanup command identity", + first, + ) + if details.get("recovery_receipt") is not None: + cleanup_receipt( + { + "status": "success", + "requested": True, + "recovery_receipt": details["recovery_receipt"], + }, + fixture, + digest, + 1, + ) + replay = operation( + stack, + "recover", + operation_id, + label + "-same-token-after-owner-reopen", + expected_state_digest=digest, + ) + cleanup_receipt(replay, fixture, digest, 1) + require( + replay.get("replayed") is True, + "same cleanup command replays after an owner died mid-seal", + replay, + ) + ready = await_state( + stack, + operation_id, + lambda s: s.get("state") == "ready_to_retry" + and s.get("cleanup_retry_count") == 1, + label + "-ready", + deadline, + ) + guards = product.inventory(stack, label + "-all-old-guards") + require( + all( + guards.get( + stack.object_root.rstrip("/") + "/" + row["object_identity"], [None] + )[0] + == 0 + for row in inspected["entries"] + ), + "new owner finishes sealing all old keys without reopening the completed key", + guards, + ) + finished = finish_append(stack, fixture, label + "-same-id-completion") + return product.case_result( + safety="PASS", + completion="PASS", + inspected=inspected, + killed_owner_pid=owner_pid, + reopened_owner_pid=stack.owner.pid, + held_provider_success=faults[0], + first_recovery_observation=first, + recovery_replay=replay, + ready=ready, + **finished, + ) + finally: + stack.object_proxy.seal_release.set() + release_provider(stack) + + +def main(): + cases = { + "public_quarantine_recovery": public_quarantine_recovery, + "paginated_prefix_recovery": paginated_prefix_recovery, + "recovery_ack_loss_and_aba": recovery_ack_loss_and_aba, + "owner_during_cleanup_seal": owner_during_cleanup_seal, + } + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--nokv-bin", type=Path, required=True) + parser.add_argument("--source-dir", type=Path, required=True) + parser.add_argument("--evidence-dir", type=Path, required=True) + parser.add_argument("--python-executable", type=Path) + parser.add_argument("--append-activity-lease-ms", type=int, default=1000) + parser.add_argument("--completion-timeout-seconds", type=float, default=90) + parser.add_argument("--timeout-seconds", type=int, default=120) + parser.add_argument("--scenario", choices=("all", *cases), default="all") + args = parser.parse_args() + result = { + "schema": "nokv.append_operations_acceptance.v1", + "status": "NOT QUALIFIED", + "started_at": base.live.now(), + "command": sys.argv, + "harness_path": str(Path(__file__).resolve()), + "harness_sha256": base.live.digest_file(Path(__file__)), + "scenarios": {}, + "qualification_scope": "Public CLI/Python inspection and explicit same-token cleanup retries on a real local Holt/RustFS owner.", + "safety": "NOT QUALIFIED", + "completion": "NOT QUALIFIED", + "performance": "NOT QUALIFIED", + "workspace_release": "NOT QUALIFIED", + "unqualified_scopes": [ + "cross-host HA", + "real demo queue redelivery (separate integration)", + "public workspace deletion/reincarnation (no public lifecycle endpoint)", + "long-term retention and high-cardinality GC", + "workload-matched performance SLO", + ], + } + stack, exit_code = None, 2 + try: + require( + sys.flags.optimize == 0, "qualification requires active Python assertions" + ) + stack = product.ProductStack( + args.evidence_dir, + binary=args.nokv_bin, + source=args.source_dir, + label="append-operations", + timeout=args.timeout_seconds, + publication_lease_ms=args.append_activity_lease_ms, + ) + stack.python_executable = args.python_executable + with stack: + shutil.copyfile( + Path(__file__), args.evidence_dir / "executed-operations-harness.py" + ) + result["machine_profile"] = product.record_machine_profile(stack) + selected = ( + cases + if args.scenario == "all" + else {args.scenario: cases[args.scenario]} + ) + for name, case in selected.items(): + print("START " + name, flush=True) + started = time.monotonic() + try: + value = case(stack, args.completion_timeout_seconds) + except Exception as error: + value = product.case_result( + safety="FAIL" + if isinstance(error, product.ContractViolation) + else "NOT QUALIFIED", + completion="NOT QUALIFIED", + failure=repr(error), + traceback=traceback.format_exc(), + ) + value["elapsed_seconds"] = time.monotonic() - started + result["scenarios"][name] = value + stack.json("operations-results.json", result) + print( + json.dumps( + { + "scenario": name, + "safety": value["safety"], + "completion": value["completion"], + } + ), + flush=True, + ) + if value["safety"] != "PASS" or value["completion"] != "PASS": + break + for dimension in ("safety", "completion"): + statuses = [value[dimension] for value in result["scenarios"].values()] + result[dimension] = ( + "FAIL" + if "FAIL" in statuses + else "PASS" + if statuses and all(status == "PASS" for status in statuses) + else "NOT QUALIFIED" + ) + result["all_listed_scenarios_executed"] = set(result["scenarios"]) == set( + cases + ) + result["status"] = ( + "FAIL" + if "FAIL" in (result["safety"], result["completion"]) + else "PASS" + if result["safety"] == result["completion"] == "PASS" + else "NOT QUALIFIED" + ) + result["qualified_scopes"] = [ + name + for name, value in result["scenarios"].items() + if value["safety"] == value["completion"] == "PASS" + ] + exit_code = {"PASS": 0, "FAIL": 1, "NOT QUALIFIED": 2}[result["status"]] + except Exception as error: + result.update( + status="FAIL" + if isinstance(error, product.ContractViolation) + else "NOT QUALIFIED", + error=repr(error), + traceback=traceback.format_exc(), + ) + exit_code = 1 if isinstance(error, product.ContractViolation) else 2 + finally: + result["completed_at"] = base.live.now() + args.evidence_dir.mkdir(parents=True, exist_ok=True) + cleanup_file = args.evidence_dir / "cleanup.json" + if cleanup_file.is_file(): + cleanup = json.loads(cleanup_file.read_text()) + if any(item.get("exit_code") != 0 for item in cleanup["results"]): + result["cleanup_failure"] = cleanup + result["status"], exit_code = "NOT QUALIFIED", 2 + (args.evidence_dir / "operations-results.json").write_text( + json.dumps(result, indent=2) + "\n" + ) + print( + json.dumps( + {"status": result["status"], "evidence": str(args.evidence_dir)} + ), + flush=True, + ) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/workbench/append_product_acceptance_gate.py b/scripts/workbench/append_product_acceptance_gate.py index a6cff1931..fbc6619be 100644 --- a/scripts/workbench/append_product_acceptance_gate.py +++ b/scripts/workbench/append_product_acceptance_gate.py @@ -48,11 +48,13 @@ class PublicationProxy(base.EvidenceDropProxy): def __init__(self, port, timeout, stack): super().__init__(port, timeout, stack) self.rendezvous = None + self.rendezvous_operation = "begin_artifact_publish" self.rendezvous_entries = [] - def begin_rendezvous(self, count): + def begin_rendezvous(self, count, *, operation_name="begin_artifact_publish"): require(self.rendezvous is None, "only one admission rendezvous") self.rendezvous = threading.Barrier(count) + self.rendezvous_operation = operation_name self.rendezvous_entries = [] def _forward(self, client): @@ -76,7 +78,7 @@ def _forward(self, client): barrier = None with self._lock: if ( - operation == "begin_artifact_publish" + operation == self.rendezvous_operation and self.rendezvous is not None ): if ( @@ -161,6 +163,8 @@ def __init__(self, stack, port, upstream_port): self.held = threading.Event() self.release = threading.Event() self.released_response = threading.Event() + self.seal_held = threading.Event() + self.seal_release = threading.Event() self.held_request = None self.reject_matching = None self.rejected_requests = [] @@ -277,7 +281,15 @@ def forward(self): and "/nokv/artifacts/" in self.path and 200 <= response.status < 300 and ( - outer.rule["action"] != "seal_ack_loss_head_failure" + not outer.rule.get("path_suffix") + or self.path.endswith(outer.rule["path_suffix"]) + ) + and ( + outer.rule["action"] + not in ( + "seal_ack_loss_head_failure", + "hold_seal_success", + ) or ( self.headers.get("Content-Length") == "0" and self.headers.get("If-Match") is not None @@ -313,6 +325,12 @@ def forward(self): if rule: with outer.lock: outer.fired.append(event) + if rule["action"] == "hold_seal_success": + outer.seal_held.set() + require( + outer.seal_release.wait(2 * outer.stack.timeout), + "held real seal response was released within the gate deadline", + ) if rule["action"] in ( "drop_success", "seal_ack_loss_head_failure", @@ -403,6 +421,7 @@ def hold_next_put(self): def close(self): self.release.set() + self.seal_release.set() self.server.shutdown() self.server.server_close() self.thread.join(timeout=2) @@ -456,7 +475,9 @@ def run( env=env, ) self.processes.append(process) - if "workspace-path" in command and "append" in command: + if ("workspace-path" in command and "append" in command) or ( + "operation" in command and "recover" in command + ): with self.lock: self.callers[label] = process timed_out = False @@ -1776,281 +1797,12 @@ def late_put_after_cleanup(stack, deadline): def seal_quarantine_operator_recovery(stack, deadline): - """Resolve one real provider ambiguity using an inspected child token.""" - workbench, operation_id = "product-seal-quarantine", "fd" * 16 - payload, label = b"operator-recovered-event\n", "seal-quarantine-caller" - stack.cli("workbench_create", {"id": workbench}) - owner_pid = stack.owner.pid - stack.object_proxy.rule = {"method": "PUT", "action": "seal_ack_loss_head_failure"} - stack.proxy.arm("mark_artifact_objects_uploaded", lambda: stack.kill_caller(label)) - try: - first = append( - stack, - workbench, - operation_id, - payload, - label=label, - max_size=1024 * 1024, - block_size=65536, - ) - stack.proxy.wait_dropped() - require(first.get("status") == "caller_killed", "uploaded caller killed", first) - require( - stack.owner.pid == owner_pid and stack.owner.poll() is None, - "owner survives the uploaded caller", - ) - until, observations = time.monotonic() + deadline, [] - while True: - status = operation_status( - stack, operation_id, f"seal-quarantine-poll-{len(observations):03d}" - ) - observations.append(status) - if status.get("state") == "quarantined": - break - require( - time.monotonic() < until, - "seal ambiguity durably quarantines", - observations, - ) - time.sleep(0.5) - require( - status.get("next_action") == "operator_reconcile" - and status.get("receipt") is None - and status.get("cause_code"), - "quarantine is actionable and has no success receipt", - status, - ) - child = status["publication_operation_id"] - faults = [ - event - for event in stack.object_proxy.fired - if event.get("fault") == "seal_ack_loss_head_failure" - ] - require( - len(faults) == 1 - and faults[0]["request_bytes"] == 0 - and faults[0]["if_match"] - and 200 <= faults[0]["upstream_status"] < 300, - "a conditional empty seal really persisted before its ACK was lost", - faults, - ) - require( - any( - event["method"] == "HEAD" and event["path"] == faults[0]["path"] - for event in stack.object_proxy.rejected_requests - ), - "post-seal HEAD really failed", - ) - query = { - "operation": "get_operation", - "request": {"operation_id": list(bytes.fromhex(child))}, - } - inspected = base.raw_rpc(stack, query, "quarantine-inspected-child") - require( - inspected.get("status") == "success" - and inspected["body"]["value"]["state"] == "quarantined", - "exact quarantined publication child is inspectable", - inspected, - ) - token = inspected["body"]["value"]["token"] - retry = append( - stack, - workbench, - operation_id, - payload, - label="quarantined-same-id-cannot-advance", - max_size=1024 * 1024, - block_size=65536, - ) - require( - retry.get("status") == "error" and retry.get("code") == "AppendUnresolved", - "quarantined append cannot silently create a successor", - retry, - ) - after_retry = operation_status( - stack, operation_id, "quarantine-parent-after-retry" - ) - require( - after_retry.get("state") == "quarantined" - and after_retry.get("publication_operation_id") == child - and after_retry.get("attempt") == status["attempt"], - "same logical identity remains on the quarantined child", - after_retry, - ) - - # This controlled fault uses the acknowledged public staging transcript - # as its complete ledger. Status does not expose a staged-key listing API. - rows, counts = {}, set() - for line in ( - (stack.evidence_dir / "publication-rpc-trace.jsonl") - .read_text() - .splitlines() - ): - event = json.loads(line) - if event["response"]["payload"]["outcome"]["status"] != "success": - continue - request = event["request"]["payload"]["operation"]["request"] - if ( - event["operation"] == "begin_artifact_publish" - and bytes(request["operation_id"]).hex() == child - ): - counts.add(request["staged_object_count"]) - if ( - event["operation"] == "stage_artifact_objects" - and bytes(request["token"]["operation_id"]).hex() == child - ): - for row in request["objects"]: - require( - row["sequence"] not in rows or rows[row["sequence"]] == row, - "acknowledged staged sequence has one immutable identity", - row, - ) - rows[row["sequence"]] = row - require( - counts == {1} and set(rows) == {0}, - "acknowledged Begin and staging prove the complete single-key ledger", - (counts, rows), - ) - stack.json( - "operator-acknowledged-staged-ledger.json", - { - "child": child, - "rows": list(rows.values()), - "source": "Successful Begin and StageArtifactObjects public RPC transcript; controlled one-key fixture.", - }, - ) - with stack.object_proxy.lock: - stack.object_proxy.reject_matching = None - verified = [] - for sequence, row in sorted(rows.items()): - key = stack.object_root.rstrip("/") + "/" + row["object_identity"] - output = stack.evidence_dir / f"operator-sealed-key-{sequence}.data" - read = stack.run( - base.infra.aws_command( - stack.aws, - stack.object_endpoint, - "s3api", - "get-object", - "--bucket", - stack.bucket, - "--key", - key, - str(output), - ), - label=f"operator-provider-read-{sequence}", - env=stack.aws_env, - ) - metadata = json.loads(read.stdout) - require( - output.read_bytes() == b"" and metadata["ContentLength"] == 0, - "every acknowledged staged key is permanently sealed at the real provider", - metadata, - ) - verified.append( - { - "sequence": sequence, - "key": key, - "size": 0, - "etag": metadata["ETag"], - "sha256": hashlib.sha256(b"").hexdigest(), - } - ) - evidence = json.dumps( - {"child": child, "token": token, "verified_keys": verified}, sort_keys=True - ).encode() - (stack.evidence_dir / "operator-provider-verification.json").write_bytes( - evidence - ) - verdict = { - "operation": "reconcile_quarantined_artifact_publish", - "request": { - "token": token, - "resolution": "provider_objects_absent", - "reason": "Controlled acceptance verifies the complete acknowledged key ledger as permanently sealed.", - "evidence_digest": list(hashlib.sha256(evidence).digest()), - }, - } - wrong = base.raw_rpc(stack, verdict, "operator-obsolete-absence-verdict") - require( - wrong.get("status") == "failure" - and wrong.get("body", {}).get("code") == "precondition_failed" - and wrong.get("body", {}).get("conflict") == "operation_state" - and wrong.get("body", {}).get("retryable") is False, - "obsolete absence verdict is precisely rejected for append", - wrong, - ) - unchanged = base.raw_rpc(stack, query, "operator-child-after-wrong-verdict") - require( - unchanged == inspected, - "rejected verdict does not change inspected child/token", - unchanged, - ) - verdict["request"]["resolution"] = "provider_objects_sealed" - resolved = base.raw_rpc(stack, verdict, "operator-exact-sealed-verdict") - require( - resolved.get("status") == "success", - "exact inspected child token accepts sealed proof", - resolved, - ) - ready = operation_status(stack, operation_id, "operator-parent-ready") - require( - ready.get("state") == "ready_to_retry" - and ready.get("next_action") == "resubmit_same", - "operator reconciliation makes the original logical ID resumable", - ready, - ) - # Keep identical block geometry and bound in the immutable append intent. - recovered = append( - stack, - workbench, - operation_id, - payload, - label="operator-same-id-completion", - max_size=1024 * 1024, - block_size=65536, - ) - require( - recovered.get("status") == "success" - and recovered["publication_operation_id"] != child, - "same logical ID completes through its next child", - recovered, - ) - exact_body(stack, workbench, payload, "operator-exact-one-effect") - final_inventory = inventory(stack, "operator-final-inventory") - require( - all(final_inventory.get(row["key"], [None])[0] == 0 for row in verified), - "operator recovery retains all old zero guards", - final_inventory, - ) - replay = replay_unchanged( - stack, - workbench, - operation_id, - payload, - recovered, - "operator-completed-replay", - max_size=1024 * 1024, - block_size=65536, - ) - return case_result( - safety="PASS", - completion="PASS", - caller_sigkill=True, - same_owner_pid=owner_pid, - quarantine_status=status, - fault=faults[0], - status_observations=observations, - inspected_child=inspected, - verified_provider_keys=verified, - obsolete_verdict=wrong, - sealed_verdict=resolved, - terminal_receipt=recovered, - replay=replay, - ) - finally: - with stack.object_proxy.lock: - stack.object_proxy.reject_matching = None - stack.object_proxy.rule = None + """Recover through public inspection and owner-driven sealed cleanup.""" + if __name__ == "__main__": + sys.modules.setdefault("append_product_acceptance_gate", sys.modules[__name__]) + import append_operations_acceptance_gate as operations + + return operations.public_quarantine_recovery(stack, deadline) def identity_errors_and_status(stack): From 13590c1cc12300d79396ea902a06b2dbb010a75b Mon Sep 17 00:00:00 2001 From: wchwawa Date: Wed, 23 Sep 2026 03:39:01 +1000 Subject: [PATCH 5/5] test: qualify append inspection failure boundaries Signed-off-by: wchwawa --- crates/nokv-server/src/executor.rs | 74 ++++++++++++ .../append_operations_acceptance_gate.py | 108 +++++++++++++++++- 2 files changed, 181 insertions(+), 1 deletion(-) diff --git a/crates/nokv-server/src/executor.rs b/crates/nokv-server/src/executor.rs index 5d945f378..65f152b02 100644 --- a/crates/nokv-server/src/executor.rs +++ b/crates/nokv-server/src/executor.rs @@ -11010,6 +11010,80 @@ mod tests { executor.execute(&misuse).unwrap_err().code, protocol::ErrorCode::RequestReplayMismatch ); + + // Remove a real retained row after the first entry of the next page. + // Inspection must discard any entries already read instead of + // returning a plausible but incomplete recovery ledger. + let before_corruption = query(); + let key = meta::staged_object_key(root(), child.operation_id, 33); + let read_version = store.current_read_version().unwrap(); + let retained = store + .read_at( + root(), + placement(), + owner(1), + meta::MetadataFamily::StagedObject, + &key, + read_version, + ) + .unwrap() + .expect("remaining staged row exists before corruption"); + assert_eq!( + meta::StagedObjectRecord::decode(&retained) + .unwrap() + .object_sequence, + 33 + ); + store + .execute( + &meta::MetadataCommand { + schema_id: meta::SCHEMA_ID.to_owned(), + root_id: root(), + logical_shard_id: shard(), + object_namespace_id: Some(types::ObjectNamespaceId::from_bytes( + [10; types::FIXED_ID_BYTES], + )), + placement_generation: placement(), + owner_epoch: owner(1), + request_id: request_id(0x9b), + command_digest: types::CommandDigest::from_bytes([0; types::SHA256_BYTES]), + read_version, + root_fence_action: meta::RootFenceAction::RequireActive, + predicates: vec![meta::CommandPredicate::Value { + family: meta::MetadataFamily::StagedObject, + key: key.clone(), + expected: Some(retained), + }], + mutations: vec![meta::CommandMutation::Delete { + family: meta::MetadataFamily::StagedObject, + key: key.clone(), + }], + history_projection: vec![meta::HistoryProjection { + family: meta::MetadataFamily::StagedObject, + key, + }], + event_projection: Vec::new(), + deterministic_result: Vec::new(), + } + .seal(), + ) + .unwrap(); + let corrupted_version = store.current_read_version().unwrap(); + let mut corrupted_inspection = inspect(None); + let protocol::WorkspaceRequest::InspectAppendCleanup(request) = + &mut corrupted_inspection.operation + else { + unreachable!() + }; + request.token = before_corruption.token; + let failure = executor.execute(&corrupted_inspection).unwrap_err(); + assert_eq!(failure.code, protocol::ErrorCode::Internal); + assert_eq!( + store.current_read_version().unwrap(), + corrupted_version, + "failed inspection must not commit metadata or repair the missing row" + ); + assert_eq!(query(), before_corruption); } #[test] diff --git a/scripts/workbench/append_operations_acceptance_gate.py b/scripts/workbench/append_operations_acceptance_gate.py index 54978f846..1838f51fd 100644 --- a/scripts/workbench/append_operations_acceptance_gate.py +++ b/scripts/workbench/append_operations_acceptance_gate.py @@ -876,6 +876,12 @@ def recovery_ack_loss_and_aba(stack, deadline): try: first_pages = inspect_all(stack, fixture, label + "-first-inspection", limit=1) first_digest = first_pages["pages"][0]["operation_token"]["state_digest"] + first_cursor = first_pages["pages"][0]["next_cursor"] + require( + isinstance(first_cursor, str) and bool(first_cursor), + "the original attempt supplies a real non-final inspection cursor", + first_pages["pages"][0], + ) caller = label + "-recover-caller" stack.proxy.arm("retry_append_cleanup", lambda: stack.kill_caller(caller)) lost = operation( @@ -978,7 +984,7 @@ def recovery_ack_loss_and_aba(stack, deadline): "inspect", operation_id, label + "-stale-first-round-cursor", - cursor=first_pages["pages"][0]["next_cursor"], + cursor=first_cursor, ) exact_input_error( stale, @@ -1032,6 +1038,99 @@ def recovery_ack_loss_and_aba(stack, deadline): ready, ) finished = finish_append(stack, fixture, label + "-complete") + + def successor_observation(stage): + status = product.operation_status( + stack, operation_id, label + "-successor-cursor-status-" + stage + ) + objects = product.inventory( + stack, label + "-successor-cursor-objects-" + stage + ) + # This independent public read includes the root metadata clock, + # so even an unintended write that preserves visible fields fails. + paths = base.raw_rpc( + stack, + { + "operation": "list_paths", + "request": { + "workbench": fixture["workbench"], + "prefix": None, + "recursive": True, + "view": "live", + "expected_read_version": None, + "workspace_continuation_fence": None, + "page": {"cursor": None, "limit": 32}, + }, + }, + label + "-successor-cursor-paths-" + stage, + ) + require( + paths.get("status") == "success" + and paths["body"]["result"] == "paths" + and paths["body"]["value"]["read_version"] > 0 + and len(paths["body"]["value"]["entries"]) == 1 + and paths["body"]["value"]["next_cursor"] is None, + "a complete public path listing supplies the metadata read version", + paths, + ) + return {"status": status, "objects": objects, "paths": paths["body"]} + + before_rejections = successor_observation("before") + require( + before_rejections["status"]["state"] == "committed" + and before_rejections["status"]["attempt"] == ready["attempt"] + 1 + and before_rejections["status"]["publication_operation_id"] + == finished["receipt"]["publication_operation_id"], + "old-cursor rejection is exercised after the successor has published", + before_rejections["status"], + ) + successor_cli_rejection = operation( + stack, + "inspect", + operation_id, + label + "-old-cursor-after-successor-cli", + limit=1, + cursor=first_cursor, + ) + exact_input_error( + successor_cli_rejection, + operation_id, + code="Conflict", + cause="Conflict", + conflict="OperationState", + ) + require( + successor_cli_rejection.get("retryable") is False + and "entries" not in successor_cli_rejection, + "CLI rejects the old attempt cursor without returning successor ledger rows", + successor_cli_rejection, + ) + successor_python_rejection = python_operation( + stack, + "inspect", + operation_id, + label + "-old-cursor-after-successor-python", + limit=1, + cursor=first_cursor, + ) + require( + successor_python_rejection.get("status") == "error" + and successor_python_rejection.get("type") == "AppendError" + and successor_python_rejection.get("code") == "Conflict" + and successor_python_rejection.get("cause_code") == "Conflict" + and successor_python_rejection.get("operation_id") == operation_id + and successor_python_rejection.get("next_action") == "query_same" + and successor_python_rejection.get("retryable") is False + and "entries" not in successor_python_rejection, + "Python rejects the same old attempt cursor with the exact typed conflict", + successor_python_rejection, + ) + after_rejections = successor_observation("after") + require( + after_rejections == before_rejections, + "old-cursor rejections preserve the metadata clock, append status, paths and object inventory", + {"before": before_rejections, "after": after_rejections}, + ) historical = operation( stack, "recover", @@ -1082,6 +1181,13 @@ def recovery_ack_loss_and_aba(stack, deadline): historical_first_receipt=historical, committed_noop=no_op, stale_cursor_rejected=stale, + successor_cursor_rejections={ + "cursor": first_cursor, + "cli": successor_cli_rejection, + "python": successor_python_rejection, + "before": before_rejections, + "after": after_rejections, + }, **finished, ) finally: