diff --git a/Cargo.lock b/Cargo.lock index e4d9fd4..89642bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "graphql-orm-ai" -version = "0.95.14" +version = "0.96.0" dependencies = [ "agql-auth", "async-graphql", diff --git a/crates/graphql-orm-ai/CHANGELOG.md b/crates/graphql-orm-ai/CHANGELOG.md index f52733c..9e1175a 100644 --- a/crates/graphql-orm-ai/CHANGELOG.md +++ b/crates/graphql-orm-ai/CHANGELOG.md @@ -3,7 +3,7 @@ title: "Changelog" kind: reference status: active owner: graphql-orm-ai-maintainers -last_reviewed: 2026-09-01 +last_reviewed: 2026-09-02 review_by: 2027-02-01 supersedes: [] --- @@ -18,6 +18,33 @@ checkpoint facts. For the current workspace baseline and active gates, use the [implementation status](docs/implementation-status.md) and the central [AI production-readiness plan](../../docs/plans/active/ai-production-readiness/README.md). +## [0.96.0] - 2026-09-02 + +Persistent schema module: **0.64.0** (unchanged from 0.95.14). + +### Fixed + +- An active retained provider-session binding whose exact server-authored + descriptor, transcript fingerprint, watermark, or lifetime no longer + matches the current turn is now atomically fenced into cleanup. The run + defers before provider dispatch, exact absence is recorded, and a compatible + generation may rebind instead of leaving the application session stranded. +- A `RecoveryRequired` run may now be retried when committed rows prove both + that no assistant output exists and that no provider dispatch was possible. + Missing reservations and exclusively released/expired reservations are the + only no-dispatch proofs. + +### Security + +- Reserved, committed, uncertain, and unknown reservation states continue to + refuse replay. Same-run replacement after cleanup is authorized only when + exact provider absence and committed no-dispatch evidence both exist; an + incompatible live cursor is never resumed or silently replaced. + +`AiRunRetryEvidence` adds the required `provider_dispatch_possible` field. +There is no database, data, table, column, index, constraint, backfill, +protected-payload, GraphQL SDL, backup, or restore migration. + ## [0.95.14] - 2026-09-01 Persistent schema module: **0.64.0** (unchanged from 0.95.13). diff --git a/crates/graphql-orm-ai/Cargo.toml b/crates/graphql-orm-ai/Cargo.toml index 08eb9fc..6e977ba 100644 --- a/crates/graphql-orm-ai/Cargo.toml +++ b/crates/graphql-orm-ai/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "graphql-orm-ai" -version = "0.95.14" +version = "0.96.0" edition = "2024" authors = ["Toby Martin "] description = "Project-agnostic AI agent runtime for graphql-orm applications" diff --git a/crates/graphql-orm-ai/MIGRATION.md b/crates/graphql-orm-ai/MIGRATION.md index 344e678..0c3f750 100644 --- a/crates/graphql-orm-ai/MIGRATION.md +++ b/crates/graphql-orm-ai/MIGRATION.md @@ -3,7 +3,7 @@ title: "Migration Guide" kind: reference status: active owner: graphql-orm-ai-maintainers -last_reviewed: 2026-09-01 +last_reviewed: 2026-09-02 review_by: 2027-02-01 supersedes: [] --- @@ -19,6 +19,25 @@ they describe. For the current workspace baseline and active delivery gates, use [implementation status](docs/implementation-status.md) and the central [AI production-readiness plan](../../docs/plans/active/ai-production-readiness/README.md). +## 0.95.14 to 0.96.0: retained-plan recovery and no-dispatch retry proof + +Adopt `graphql-orm-ai` 0.96.0 from one reviewed full monorepo revision. Hosts +that construct `AiRunRetryEvidence` must add `provider_dispatch_possible` from +committed reservation rows. Set it to `false` only when the exact run has no +reservation or every reservation is durably `released`/`expired`; use `true` +for reserved, committed, uncertain, unrecognized, or unavailable evidence. + +The ORM provider-session service now atomically moves an incompatible active +binding into cleanup during disposition. Provider execution treats unavailable +bindings as a pre-dispatch deferral, allowing the ordinary cleanup worker to +record exact absence and the retry scheduler to bind the current descriptor. +Do not bypass deletion or copy an old cursor into a new descriptor. A deleted +generation may authorize the same run only when its committed budget rows also +prove that provider dispatch was impossible. + +The AI schema module remains **0.64.0**. There is no database, data, GraphQL +SDL, protected-payload, backup, restore, or data backfill migration. + ## 0.95.13 to 0.95.14: retained-resume warning interleaving Adopt `graphql-orm-ai` 0.95.14 from one reviewed full monorepo revision. A host diff --git a/crates/graphql-orm-ai/README.md b/crates/graphql-orm-ai/README.md index da1cbb7..10d7ef5 100644 --- a/crates/graphql-orm-ai/README.md +++ b/crates/graphql-orm-ai/README.md @@ -28,7 +28,7 @@ for AI, ORM, storage, backup, and tool-profile packages: ```toml [dependencies] -graphql-orm-ai = { git = "https://github.com/Dastari/graphql-orm.git", rev = "", version = "0.95.14", default-features = false, features = ["sqlite"] } +graphql-orm-ai = { git = "https://github.com/Dastari/graphql-orm.git", rev = "", version = "0.96.0", default-features = false, features = ["sqlite"] } ``` Exactly one persistence backend is required: `sqlite` (default), `postgres`, @@ -150,15 +150,21 @@ depend solely on the process-local wakeup channel. Terminal `run_failed` and `run_recovery_required` events carry a bounded, content-free failure record with a stable code and a retryable flag computed from committed rows. `retryAiRun` authors a new run over the same durable user -message under current policy where re-execution is provably safe; +message under current policy where re-execution is provably safe. A +recovery-required run is retryable only when committed rows prove no assistant +output and no possible provider dispatch: no reservation, or exclusively +released/expired reservations. Reserved, committed, uncertain, and unknown +states remain non-retryable; `acknowledgeAiRunFailure` dismisses a failure without deleting audit history. Invalidating a retained provider thread emits `provider_session_reset` or `provider_session_rebound` so a host can tell the user the model's context was reset even though the durable transcript reads as continuous. Once exact -provider absence is persisted, a later run may replace the deleted generation -under its current server-authored descriptor; the historical descriptor grants -no replacement authority. A lost rebind fence discards the fresh empty -provider session and defers safely before any business input or tool request. +provider absence is persisted, a later run—or the same provably pre-dispatch +run—may replace the deleted generation under its current server-authored +descriptor; the historical descriptor grants no replacement authority. An +incompatible active descriptor/transcript is atomically fenced into that +cleanup path. A lost rebind fence discards the fresh empty provider session and +defers safely before any business input or tool request. A retained Codex host must not treat the `thread/resume` response as sufficient readiness. Continue feeding the strict protocol actor until diff --git a/crates/graphql-orm-ai/docs/implementation-status.md b/crates/graphql-orm-ai/docs/implementation-status.md index aaea903..262aa95 100644 --- a/crates/graphql-orm-ai/docs/implementation-status.md +++ b/crates/graphql-orm-ai/docs/implementation-status.md @@ -10,7 +10,7 @@ supersedes: [] # Implementation Status -`graphql-orm-ai` is at crate version `0.95.14` with AI schema module +`graphql-orm-ai` is at crate version `0.96.0` with AI schema module `0.64.0`. It uses workspace `graphql-orm` `0.30.0`, backend-neutral `graphql-orm-ai-tool-profiles` `0.10.4`, and external `agql-auth` `0.19.0` at `1d2e9fe2e1576105212a7b340a11abf8cad0382d`. @@ -75,10 +75,13 @@ verification evidence belongs in the focused guides. - The provider-neutral durable session service protects opaque retained-thread cursors under exact owner/scope/run/descriptor/transcript fencing and an exact deletion/absence lifecycle. An absence-proven deleted generation may - be replaced once through a crate-issued short-lived rebind authorization; + be replaced once through a crate-issued short-lived rebind authorization. + An incompatible active descriptor/transcript is atomically fenced into + cleanup before dispatch, and a recovery-required run is re-authored only + when committed output/reservation rows prove no provider dispatch occurred; exact cleanup may use a fresh initialized provider process without resuming - the retained thread. Cleanup/backoff, expiry, descriptor drift, and restore - quarantine remain unavailable. Cursor state is separate from warm processes, + the retained thread. Cleanup/backoff and restore quarantine remain + unavailable. Cursor state is separate from warm processes, private from GraphQL, backup-redacted, and readiness-blocking on portable restore until drained. Provider failures may additionally emit only a closed content-free operational category without changing conservative run diff --git a/crates/graphql-orm-ai/src/orm_coordinator.rs b/crates/graphql-orm-ai/src/orm_coordinator.rs index f123b4f..ab9171a 100644 --- a/crates/graphql-orm-ai/src/orm_coordinator.rs +++ b/crates/graphql-orm-ai/src/orm_coordinator.rs @@ -3914,6 +3914,7 @@ mod tests { crate::AiRunRetryEvidence { terminal: crate::AiRunTerminalEvent::Failed, produced_assistant_output: false, + provider_dispatch_possible: false, }, Some("provider_budget_denied"), ), @@ -3972,6 +3973,7 @@ mod tests { crate::AiRunRetryEvidence { terminal: crate::AiRunTerminalEvent::Failed, produced_assistant_output: false, + provider_dispatch_possible: false, }, Some("provider_pre_transport_failed"), ), @@ -4029,6 +4031,7 @@ mod tests { crate::AiRunRetryEvidence { terminal: crate::AiRunTerminalEvent::Failed, produced_assistant_output: false, + provider_dispatch_possible: false, }, Some("provider_native_item_rejected"), ), @@ -4604,6 +4607,7 @@ mod tests { crate::AiRunRetryEvidence { terminal: crate::AiRunTerminalEvent::Failed, produced_assistant_output: false, + provider_dispatch_possible: false, }, Some("provider_session_cleanup_unavailable"), ), diff --git a/crates/graphql-orm-ai/src/orm_provider_session.rs b/crates/graphql-orm-ai/src/orm_provider_session.rs index 3e1da00..e43da15 100644 --- a/crates/graphql-orm-ai/src/orm_provider_session.rs +++ b/crates/graphql-orm-ai/src/orm_provider_session.rs @@ -570,8 +570,74 @@ mod service { .map(AiProviderSessionRunDisposition::Resume) .map_err(ai_error_to_orm); } + if state == AiProviderSessionState::Active { + // An active cursor that cannot resume the exact + // server-authored descriptor/transcript must not + // strand the durable application session. Fence it + // into deletion before returning to the caller; + // only exact provider-absence proof may authorize a + // fresh binding generation. + let descriptor_matches = descriptor_from_record(binding) + .map_err(ai_error_to_orm)? + == *planned.descriptor(); + let reason_code = if !descriptor_matches { + "provider_session_descriptor_changed" + } else if binding.transcript_fingerprint + != planned.transcript_fingerprint() + || binding.through_message_sequence.checked_add(1) + != Some(input.sequence) + { + "provider_session_transcript_changed" + } else { + "provider_session_expired" + } + .to_owned(); + let outcome = tx + .compare_and_swap::( + &binding.id, + binding.row_version, + AiProviderSessionBindingRecordWhereInput::default(), + cleanup_required_update(reason_code.clone(), lease.run_id().0), + ) + .await + .map_err(OrmPublicError::from)?; + if !matches!(outcome, ConditionalUpdateOutcome::Updated(_)) { + return Err(OrmPublicError::new(OrmErrorCode::Conflict)); + } + append_audit( + tx, + "ai.provider_session.cleanup_required", + binding.id, + &reason_code, + lease.run_id().0, + now, + ) + .await?; + append_provider_session_disclosure( + tx, + crate::orm_runs::PROVIDER_SESSION_RESET_EVENT, + binding.session_id, + binding.id, + &reason_code, + Some(lease.run_id().0), + now, + ) + .await?; + return Ok(AiProviderSessionRunDisposition::Unavailable( + AiProviderSessionState::CleanupRequired, + )); + } if state == AiProviderSessionState::Deleted - && binding.last_run_id != Some(lease.run_id().0) + && (binding.last_run_id != Some(lease.run_id().0) + || binding + .cleanup_reason_code + .as_deref() + .is_some_and(is_pre_dispatch_rebind_cleanup_reason) + && !crate::orm_runs::run_provider_dispatch_possible( + tx, + lease.run_id().0, + ) + .await?) { // Exact provider absence severs the old cursor from // the next generation. The tombstone's historical @@ -2542,14 +2608,21 @@ mod service { cleanup_next_attempt_at: Some(Some( next_attempt_at.unix_timestamp(), )), - cleanup_reason_code: Some(Some(reason_code)), ..Default::default() }, ) .await .map_err(OrmPublicError::from)?; if matches!(outcome, ConditionalUpdateOutcome::Updated(_)) { - Ok(()) + append_audit( + tx, + "ai.provider_session.cleanup_retry_scheduled", + record.id, + &reason_code, + record.last_run_id.unwrap_or(record.session_id), + now, + ) + .await } else { Err(OrmPublicError::new(OrmErrorCode::Conflict)) } @@ -2580,6 +2653,15 @@ mod service { } } + fn is_pre_dispatch_rebind_cleanup_reason(reason_code: &str) -> bool { + matches!( + reason_code, + "provider_session_descriptor_changed" + | "provider_session_transcript_changed" + | "provider_session_expired" + ) + } + async fn load_and_validate_completed_run( tx: &mut MutationContext<'_, DefaultWriteBackend>, lease: &AiRunLease, diff --git a/crates/graphql-orm-ai/src/orm_run_disposition.rs b/crates/graphql-orm-ai/src/orm_run_disposition.rs index 29210fa..a71382f 100644 --- a/crates/graphql-orm-ai/src/orm_run_disposition.rs +++ b/crates/graphql-orm-ai/src/orm_run_disposition.rs @@ -17,7 +17,7 @@ use time::Duration; use uuid::Uuid; use crate::orm_inbox::{PreparedAiInboxEvent, append_inbox_event}; -use crate::orm_runs::run_produced_assistant_output; +use crate::orm_runs::{run_produced_assistant_output, run_provider_dispatch_possible}; use crate::orm_sessions::{ content_context, map_orm, map_protection, map_transaction, principal_identity, record_scope, }; @@ -339,6 +339,8 @@ impl OrmAiRunDispositionService { tx, session_id, run_id, ) .await?, + provider_dispatch_possible: run_provider_dispatch_possible(tx, run_id) + .await?, }; if classify_run_retry(evidence, run.error_code.as_deref()) != AiRunRetryAdmission::Allowed diff --git a/crates/graphql-orm-ai/src/orm_runs.rs b/crates/graphql-orm-ai/src/orm_runs.rs index 7611315..fa1249d 100644 --- a/crates/graphql-orm-ai/src/orm_runs.rs +++ b/crates/graphql-orm-ai/src/orm_runs.rs @@ -5016,6 +5016,7 @@ pub(crate) async fn append_terminal_run_event( run.id, ) .await?, + provider_dispatch_possible: run_provider_dispatch_possible(tx, run.id).await?, }; Some(AiRunFailure::new( classify_run_retry(evidence, outcome_code), @@ -5363,6 +5364,79 @@ mod tests { .expect("inbox events should query") } + async fn provider_dispatch_possible(fixture: &Fixture, run_id: AiRunId) -> bool { + fixture + .database + .transaction(TransactionMode::Default, move |tx| { + Box::pin(async move { run_provider_dispatch_possible(tx, run_id.0).await }) + }) + .await + .expect("provider-dispatch evidence should query") + } + + #[tokio::test] + async fn retry_evidence_treats_only_absent_or_released_reservations_as_no_dispatch() { + let empty = fixture().await; + let empty_run = seed_queued(&empty).await; + assert!(!provider_dispatch_possible(&empty, empty_run).await); + + for (state, expected) in [ + ("released", false), + ("expired", false), + ("reserved", true), + ("committed", true), + ("uncertain", true), + ("future_state", true), + ] { + let fixture = fixture().await; + let run_id = seed_queued(&fixture).await; + let run = run_record(&fixture, run_id).await; + AiBudgetReservationRecord::insert( + &fixture.database, + CreateAiBudgetReservationRecordInput { + budget_counter_ids: serde_json::json!([]), + scope_kind: "tenant".to_owned(), + scope_id: "tenant-run".to_owned(), + tenant_id: Some("tenant-run".to_owned()), + principal_kind: "user".to_owned(), + principal_subject: "run-user".to_owned(), + session_id: run.session_id, + run_id: run.id, + attempt_id: Uuid::new_v4(), + lease_generation: 1, + provider_kind: "openai".to_owned(), + provider_model: "reviewed-model".to_owned(), + reasoning_effort: "unspecified".to_owned(), + pricing_policy_version: "pricing-v1".to_owned(), + reserved_input_tokens: 1, + reserved_output_tokens: 1, + reserved_tool_units: 0, + reserved_image_units: 0, + reserved_cost_microunits: 1, + reserved_runs: 1, + actual_input_tokens: None, + actual_cached_input_tokens: None, + actual_output_tokens: None, + actual_tool_units: None, + actual_image_units: None, + actual_cost_microunits: None, + actual_runs: None, + idempotency_key: format!("retry-evidence-{state}"), + state: state.to_owned(), + expires_at: (fixture.clock.now() + Duration::minutes(1)).unix_timestamp(), + reconciled_at: None, + }, + ) + .await + .expect("test reservation should insert"); + assert_eq!( + provider_dispatch_possible(&fixture, run_id).await, + expected, + "unexpected dispatch evidence for {state}", + ); + } + } + #[tokio::test] async fn concurrent_workers_cannot_claim_the_same_run() { let fixture = fixture().await; @@ -6070,3 +6144,32 @@ pub(crate) async fn run_produced_assistant_output( .map_err(OrmPublicError::from)?; Ok(!rows.is_empty()) } + +/// Returns whether committed budget rows leave any provider dispatch possible. +/// +/// Absence is proof-bearing only when no reservation exists or every exact-run +/// reservation is durably `released`/`expired`. Any other value, including a +/// malformed future state, remains fail-closed as possible dispatch. +pub(crate) async fn run_provider_dispatch_possible( + tx: &mut MutationContext<'_, DefaultWriteBackend>, + run_id: Uuid, +) -> Result { + let rows = tx + .query::() + .filter(AiBudgetReservationRecordWhereInput { + run_id: Some(UuidFilter { + eq: Some(run_id), + ..Default::default() + }), + state: Some(StringFilter { + not_in: Some(vec!["released".to_owned(), "expired".to_owned()]), + ..Default::default() + }), + ..Default::default() + }) + .limit(1) + .fetch_all() + .await + .map_err(OrmPublicError::from)?; + Ok(!rows.is_empty()) +} diff --git a/crates/graphql-orm-ai/src/orm_supervised_coordinator.rs b/crates/graphql-orm-ai/src/orm_supervised_coordinator.rs index d9562e5..5435f35 100644 --- a/crates/graphql-orm-ai/src/orm_supervised_coordinator.rs +++ b/crates/graphql-orm-ai/src/orm_supervised_coordinator.rs @@ -3053,6 +3053,7 @@ mod tests { crate::AiRunRetryEvidence { terminal: crate::AiRunTerminalEvent::Failed, produced_assistant_output: false, + provider_dispatch_possible: false, }, Some("provider_budget_denied"), ), @@ -3116,6 +3117,7 @@ mod tests { crate::AiRunRetryEvidence { terminal: crate::AiRunTerminalEvent::Failed, produced_assistant_output: false, + provider_dispatch_possible: false, }, Some("provider_pre_transport_failed"), ), diff --git a/crates/graphql-orm-ai/src/provider_calls.rs b/crates/graphql-orm-ai/src/provider_calls.rs index 463450e..89f42fd 100644 --- a/crates/graphql-orm-ai/src/provider_calls.rs +++ b/crates/graphql-orm-ai/src/provider_calls.rs @@ -2967,20 +2967,17 @@ impl AiProviderCallExecutor { let _ = session_service .require_cleanup(&claim, "provider_session_wait_plan_changed") .await; - return Err(AiError::Conflict); + return Err(AiError::ProviderSessionDeferred); } (claim, None) } - crate::AiProviderSessionRunDisposition::Unavailable( - crate::AiProviderSessionState::CleanupRequired - | crate::AiProviderSessionState::CleanupInProgress - | crate::AiProviderSessionState::CleanupBackoff - | crate::AiProviderSessionState::Deleted, - ) => { - return Err(AiError::ProviderSessionDeferred); - } crate::AiProviderSessionRunDisposition::Unavailable(_) => { - return Err(AiError::Conflict); + // No provider request has been dispatched. Durable stores use + // the disposition transaction to fence an incompatible active + // binding into cleanup; transient claimed/parked states also + // converge through retry/expiry rather than manufacturing an + // uncertain provider turn. + return Err(AiError::ProviderSessionDeferred); } }; let opened = match session_service.open_for_run(&lease, &claim).await { diff --git a/crates/graphql-orm-ai/src/provider_session.rs b/crates/graphql-orm-ai/src/provider_session.rs index 7769b05..b214404 100644 --- a/crates/graphql-orm-ai/src/provider_session.rs +++ b/crates/graphql-orm-ai/src/provider_session.rs @@ -1358,6 +1358,12 @@ pub trait AiProviderSessionService: Send + Sync { /// [`AiProviderSessionRebindAuthorization`] only after validating exact /// provider absence and the current run fence. /// + /// Durable implementations may atomically transition an incompatible + /// active binding into cleanup while classifying it. That transition must + /// validate the current run, owner, scope, descriptor/transcript plan, and + /// retained cursor generation; it grants no replacement authority until + /// exact provider absence is later recorded. + /// /// # Errors /// /// Returns an error when the run fence, current principal/session access, diff --git a/crates/graphql-orm-ai/src/run_state.rs b/crates/graphql-orm-ai/src/run_state.rs index 9d6d75e..bf6ed53 100644 --- a/crates/graphql-orm-ai/src/run_state.rs +++ b/crates/graphql-orm-ai/src/run_state.rs @@ -349,15 +349,24 @@ pub struct AiRunRetryEvidence { pub terminal: AiRunTerminalEvent, /// Whether a durable assistant message exists for this run. pub produced_assistant_output: bool, + /// Whether committed reservation rows leave any provider dispatch + /// possible for this run. + /// + /// `false` is a positive absence proof: no reservation exists, or every + /// reservation is durably `released`/`expired`. `true` is deliberately + /// conservative and includes reserved, committed, uncertain, or unknown + /// states. + pub provider_dispatch_possible: bool, } /// Classifies whether a terminal run may be retried as a new run. /// /// The rules are deliberately conservative: /// -/// - `RecoveryRequired` is never retryable. It exists precisely because an -/// external effect could not be proven safe, and re-execution is what the -/// safe-failure guardrail forbids. +/// - `RecoveryRequired` is retryable only when committed rows prove that no +/// assistant output exists and no provider dispatch was possible. This +/// repairs conservative pre-transport classifications without weakening the +/// fail-closed boundary for an actually possible external effect. /// - `Completed` is never retryable; the message already has its answer. /// - `Cancelled` is retryable only when the run produced no durable assistant /// output. Cancellation observed *after* a provider turn was persisted @@ -371,7 +380,15 @@ pub fn classify_run_retry( outcome_code: Option<&str>, ) -> AiRunRetryAdmission { match evidence.terminal { - AiRunTerminalEvent::RecoveryRequired => AiRunRetryAdmission::RefusedUncertain, + AiRunTerminalEvent::RecoveryRequired => { + if evidence.produced_assistant_output { + AiRunRetryAdmission::RefusedAlreadyAnswered + } else if evidence.provider_dispatch_possible { + AiRunRetryAdmission::RefusedUncertain + } else { + AiRunRetryAdmission::Allowed + } + } AiRunTerminalEvent::Completed => AiRunRetryAdmission::RefusedAlreadyAnswered, AiRunTerminalEvent::Cancelled => { if evidence.produced_assistant_output { diff --git a/crates/graphql-orm-ai/tests/orm_run_disposition.rs b/crates/graphql-orm-ai/tests/orm_run_disposition.rs index b651a84..c6c6b7c 100644 --- a/crates/graphql-orm-ai/tests/orm_run_disposition.rs +++ b/crates/graphql-orm-ai/tests/orm_run_disposition.rs @@ -146,12 +146,12 @@ async fn fixture_on(database: Database, migrate: bool) -> Fixture .expect("run limits should validate"), ); let dispositions = OrmAiRunDispositionService::new( - database, + database.clone(), access_policy, protection_policy, content_protector, principal_resolver, - clock, + clock.clone(), AiRunDispositionLimits::default(), ); Fixture { @@ -329,7 +329,7 @@ async fn retry_authors_a_new_run_over_the_same_message_and_is_idempotent() { } #[tokio::test] -async fn recovery_required_refuses_retry_but_still_admits_acknowledgement() { +async fn recovery_required_without_possible_provider_dispatch_admits_retry() { let fixture = fixture().await; let session = session(&fixture).await; let sent = failed_run( @@ -341,23 +341,56 @@ async fn recovery_required_refuses_retry_but_still_admits_acknowledgement() { ) .await; - assert!( - matches!( - fixture - .dispositions - .retry_run( - &fixture.owner, - RetryAiRunInput { - session_id: session.id, - run_id: sent.run_id, - client_request_id: Uuid::new_v4(), - }, - ) - .await, - Err(AiError::Conflict) + let page = fixture + .sessions + .session_event_page(&fixture.owner, AiSessionId(session.id), 0, 500) + .await + .expect("events should replay"); + let failure = failure_record(&page, "run_recovery_required"); + assert_eq!(failure["retryable"], serde_json::json!(true)); + assert_eq!(failure["admission"], serde_json::json!("allowed")); + + fixture + .dispositions + .retry_run( + &fixture.owner, + RetryAiRunInput { + session_id: session.id, + run_id: sent.run_id, + client_request_id: Uuid::new_v4(), + }, + ) + .await + .expect("committed absence of provider dispatch should admit retry"); +} + +#[test] +fn recovery_required_with_possible_provider_dispatch_remains_non_retryable() { + assert_eq!( + classify_run_retry( + AiRunRetryEvidence { + terminal: AiRunTerminalEvent::RecoveryRequired, + produced_assistant_output: false, + provider_dispatch_possible: true, + }, + Some("provider_turn_uncertain"), ), - "an unproven external effect must never be re-executed" + AiRunRetryAdmission::RefusedUncertain, ); +} + +#[tokio::test] +async fn recovery_required_still_admits_acknowledgement() { + let fixture = fixture().await; + let session = session(&fixture).await; + let sent = failed_run( + &fixture, + session.id, + AiRunState::RecoveryRequired, + "provider_turn_uncertain", + Some("provider_turn_uncertain"), + ) + .await; let acknowledged = fixture .dispositions diff --git a/crates/graphql-orm-ai/tests/provider_sessions.rs b/crates/graphql-orm-ai/tests/provider_sessions.rs index 7302c6e..e959244 100644 --- a/crates/graphql-orm-ai/tests/provider_sessions.rs +++ b/crates/graphql-orm-ai/tests/provider_sessions.rs @@ -622,6 +622,155 @@ async fn settled_interrupt_advances_the_durable_watermark_and_reuses_the_thread( ); } +#[tokio::test] +async fn changed_active_plan_is_cleaned_and_rebound_without_stranding_the_run() { + let fixture = provider_session_fixture().await; + let owner = fixture.owner.clone(); + let first = active_run(&fixture, &owner, "changed-plan-workspace").await; + let original_descriptor = AiProviderSessionDescriptor::new( + ProviderKind::LocalHarness, + "reviewed-local-profile", + "reviewed-model", + "a".repeat(64), + "retained-runtime/v2", + "b".repeat(64), + ) + .expect("original descriptor should validate"); + fixture + .provider_sessions + .bind_for_run( + &first, + AiProviderSessionBindRequest::new( + original_descriptor, + AiProviderSessionCursor::new("retained.thread", "original-thread") + .expect("original cursor should validate"), + "c".repeat(64), + None, + ) + .expect("original bind request should validate"), + ) + .await + .expect("original retained thread should bind"); + fixture + .cancellation + .request_cancellation( + &owner, + CancelAiRunInput { + session_id: first.session_id().0, + run_id: first.run_id().0, + client_request_id: Uuid::new_v4(), + }, + ) + .await + .expect("owner cancellation should become durable"); + fixture + .provider_sessions + .settle_interrupted_turn(&first, AiRunInterruptSettlement::Settled) + .await + .expect("settled empty turn should release the original thread"); + + let next = next_active_run(&fixture, &owner, first.session_id()).await; + let changed_descriptor = AiProviderSessionDescriptor::new( + ProviderKind::LocalHarness, + "reviewed-local-profile", + "reviewed-model", + "d".repeat(64), + "retained-runtime/v2", + "e".repeat(64), + ) + .expect("changed descriptor should validate"); + let changed_plan = AiProviderSessionTurnPlan::new(changed_descriptor.clone(), "f".repeat(64)) + .expect("changed plan should validate"); + assert!(matches!( + fixture + .provider_sessions + .disposition_for_run(&next, &changed_plan) + .await + .expect("changed active plan should converge into cleanup"), + AiProviderSessionRunDisposition::Unavailable(AiProviderSessionState::CleanupRequired) + )); + + let cleanup = fixture + .provider_sessions + .claim_cleanup("changed-plan-cleanup-worker") + .await + .expect("cleanup claim should succeed") + .expect("changed active plan should require deletion"); + let deletion = fixture + .provider_sessions + .open_for_cleanup(&cleanup, &protection_policy(cleanup.scope().clone())) + .await + .expect("old cursor should open only for deletion"); + assert_eq!( + deletion.cursor().expose_to_provider_adapter(), + "original-thread" + ); + fixture + .provider_sessions + .schedule_cleanup_retry( + &cleanup, + Duration::seconds(1), + "provider_delete_unavailable", + ) + .await + .expect("transient deletion failure should retain the cleanup origin"); + fixture.clock.advance_seconds(2); + let cleanup = fixture + .provider_sessions + .claim_cleanup("changed-plan-cleanup-worker") + .await + .expect("cleanup retry claim should succeed") + .expect("backoff expiry should expose the same deletion"); + let deletion = fixture + .provider_sessions + .open_for_cleanup(&cleanup, &protection_policy(cleanup.scope().clone())) + .await + .expect("retried cleanup cursor should open"); + fixture + .provider_sessions + .complete_cleanup( + &cleanup, + AiProviderSessionAbsenceProof::for_request(&deletion, fixture.clock.now()), + ) + .await + .expect("exact absence should tombstone the old cursor"); + + let authorization = match fixture + .provider_sessions + .disposition_for_run(&next, &changed_plan) + .await + .expect("the same pre-dispatch run should receive rebind authority") + { + AiProviderSessionRunDisposition::RebindAllowed(authorization) => *authorization, + other => panic!("expected same-run rebind authority, got {other:?}"), + }; + let rebound = fixture + .provider_sessions + .rebind_for_run( + &next, + authorization, + AiProviderSessionBindRequest::new( + changed_descriptor, + AiProviderSessionCursor::new("retained.thread", "replacement-thread") + .expect("replacement cursor should validate"), + "f".repeat(64), + None, + ) + .expect("replacement bind request should validate"), + ) + .await + .expect("the current plan should bind after old-provider absence"); + let opened = fixture + .provider_sessions + .open_for_run(&next, &rebound) + .await + .expect("replacement cursor should open under the same run fence"); + assert_eq!( + opened.cursor().expose_to_provider_adapter(), + "replacement-thread" + ); +} + #[tokio::test] async fn provider_session_resume_atomically_rotates_a_fresh_current_principal() { let fixture = provider_session_fixture().await; diff --git a/crates/graphql-orm/tests/fixtures/backend-coexistence/Cargo.lock b/crates/graphql-orm/tests/fixtures/backend-coexistence/Cargo.lock index 234b764..4712615 100644 --- a/crates/graphql-orm/tests/fixtures/backend-coexistence/Cargo.lock +++ b/crates/graphql-orm/tests/fixtures/backend-coexistence/Cargo.lock @@ -1329,7 +1329,7 @@ dependencies = [ [[package]] name = "graphql-orm-ai" -version = "0.95.14" +version = "0.96.0" dependencies = [ "agql-auth", "async-graphql", diff --git a/docs/plans/active/ai-production-readiness/README.md b/docs/plans/active/ai-production-readiness/README.md index 684f002..2e0ce3a 100644 --- a/docs/plans/active/ai-production-readiness/README.md +++ b/docs/plans/active/ai-production-readiness/README.md @@ -59,7 +59,7 @@ retention, or restore proofs remain closed. ## Current checkpoint -Package 0.95.14 and AI schema module 0.64.0 provide the protected runtime, +Package 0.96.0 and AI schema module 0.64.0 provide the protected runtime, provider adapters, exact completed-batch adoption, retention foundations, restore planning, and readiness observation contracts. Database-derived collection covers bounded conservative run classification, approval and diff --git a/docs/reference/workspace-packages.md b/docs/reference/workspace-packages.md index f5ab505..dca6b74 100644 --- a/docs/reference/workspace-packages.md +++ b/docs/reference/workspace-packages.md @@ -19,7 +19,7 @@ changes. | Package | Version | Path | Default features | Direct internal dependencies | | --- | --- | --- | --- | --- | | `graphql-orm` | `0.30.0` | `crates/graphql-orm` | `sqlite` | `graphql-orm-macros`, `graphql-orm-operation-catalog`, `graphql-orm-router-protocol` (dev-only) | -| `graphql-orm-ai` | `0.95.14` | `crates/graphql-orm-ai` | `sqlite` | `graphql-orm`, `graphql-orm-ai-tool-profiles`, `graphql-orm-storage` | +| `graphql-orm-ai` | `0.96.0` | `crates/graphql-orm-ai` | `sqlite` | `graphql-orm`, `graphql-orm-ai-tool-profiles`, `graphql-orm-storage` | | `graphql-orm-ai-tool-profiles` | `0.10.4` | `crates/graphql-orm-ai-tool-profiles` | none | `graphql-orm-operation-catalog`, `graphql-orm-router-protocol` (dev-only) | | `graphql-orm-backup` | `0.7.2` | `crates/graphql-orm-backup` | `local` | `graphql-orm` (optional), `graphql-orm-storage` | | `graphql-orm-macros` | `0.30.0` | `crates/graphql-orm-macros` | `sqlite` | none |