From 0480dc32109928765e91eb788ce016d31dc7ccb5 Mon Sep 17 00:00:00 2001 From: Kandev Agent Date: Mon, 12 Oct 2026 02:46:09 +0100 Subject: [PATCH 1/7] fix: reconcile interrupted shutdown delivery without replay --- .../lifecycle/native_resume_adoption.go | 2 +- .../lifecycle/native_resume_delivery.go | 45 +++- .../lifecycle/native_resume_delivery_test.go | 16 +- .../native_resume_dispatch_reconciliation.go | 2 +- .../lifecycle/native_resume_evidence.go | 8 + .../runtime/lifecycle/native_resume_proof.go | 105 ++++++++ .../native_resume_state_matrix_test.go | 82 ++++++ .../agentctl/journal/journal_submissions.go | 4 +- .../agentctl/journal/native_resume_test.go | 32 ++- .../internal/agentctl/journal/recovery.go | 2 +- .../agentctl/journal/submission_outcome.go | 15 ++ .../agentctl/server/process/manager.go | 17 +- .../server/process/submission_delivery.go | 23 +- .../process/submission_shutdown_test.go | 124 +++++++++ .../backendapp/adapters_native_resume.go | 5 + .../orchestrator/native_resume_proof_test.go | 50 ++++ .../restart_submission_reconciliation.go | 2 +- .../internal/orchestrator/session_launch.go | 9 +- .../session_native_delivery_recovery.go | 26 ++ .../task/models/native_resume_delivery.go | 23 ++ .../sqlite/native_resume_binding.go | 239 ++++++++++++++++++ .../sqlite/native_resume_binding_test.go | 148 +++++++++++ .../2026-10-12-total-session-recovery.md | 51 ++++ .../session-recovery-convergence/plan.md | 55 ++++ .../task-01-shutdown-settlement.md | 35 +++ .../task-02-native-convergence.md | 37 +++ .../task-03-recovery-contract.md | 37 +++ docs/public/operations.md | 2 +- .../durable-restart-reconciliation.md | 21 +- .../session-recovery-convergence.md | 94 +++++++ 30 files changed, 1275 insertions(+), 36 deletions(-) create mode 100644 apps/backend/internal/agent/runtime/lifecycle/native_resume_proof.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/native_resume_state_matrix_test.go create mode 100644 apps/backend/internal/agentctl/journal/submission_outcome.go create mode 100644 apps/backend/internal/agentctl/server/process/submission_shutdown_test.go create mode 100644 apps/backend/internal/orchestrator/native_resume_proof_test.go create mode 100644 apps/backend/internal/task/models/native_resume_delivery.go create mode 100644 apps/backend/internal/task/repository/sqlite/native_resume_binding.go create mode 100644 apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go create mode 100644 docs/decisions/2026-10-12-total-session-recovery.md create mode 100644 docs/plans/session-recovery-convergence/plan.md create mode 100644 docs/plans/session-recovery-convergence/task-01-shutdown-settlement.md create mode 100644 docs/plans/session-recovery-convergence/task-02-native-convergence.md create mode 100644 docs/plans/session-recovery-convergence/task-03-recovery-contract.md create mode 100644 docs/specs/platform/system-design/session-recovery-convergence.md diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_adoption.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_adoption.go index f1c7601b509..aae25dff577 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/native_resume_adoption.go +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_adoption.go @@ -64,7 +64,7 @@ func peerAcknowledgesUnknownHistory(execution *AgentExecution, status *agentctl. func peerMatchesUnknownHistory(execution *AgentExecution, status *agentctl.DeliveryStatus, peer journal.Submission, submission *models.AgentDeliverySubmission) bool { return submission.State == models.DeliverySubmissionInterruptedUnknown && - peer.ID == submission.ID && peer.State == journal.SubmissionInterruptedUnknown && + submission.TerminalSequence == 0 && peer.ID == submission.ID && journal.HasUnprovenOutcome(peer) && peer.SessionID == execution.SessionID && peer.SessionID == submission.SessionID && peer.IncarnationID == status.IncarnationID && peer.IncarnationID == submission.IncarnationID && peer.HarnessGeneration > 0 && peer.HarnessGeneration <= status.HarnessGeneration && int64(peer.HarnessGeneration) == submission.HarnessGeneration && diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery.go index 61bfbdc9e78..a3f74752101 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery.go +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery.go @@ -8,6 +8,7 @@ import ( agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/agentctl/journal" + "github.com/kandev/kandev/internal/task/models" ) // AcknowledgeNativeResumeDelivery seals uncertain prior prompts after an @@ -36,10 +37,13 @@ func (m *Manager) AcknowledgeNativeResumeDelivery(ctx context.Context, sessionID if err := m.reconcileNativeResumeSQLDispatch(ctx, execution, status, submissions); err != nil { return err } + if _, err := authenticatedDeliveryReplayCursor(ctx, execution, m.streamManager.deliveryRepository(), status); err != nil { + return err + } if err := m.validateNativeResumeSQLHistory(ctx, execution, status, submissions, false); err != nil { return err } - ids, err := nativeResumeInterruptedSubmissionIDs(execution, submissions) + ids, err := m.nativeResumeInterruptedSubmissionIDs(ctx, execution, status, submissions) if err != nil { return err } @@ -79,25 +83,44 @@ func validateNativeResumeDeliveryOwner(execution *AgentExecution, status *agentc return nil } -func nativeResumeInterruptedSubmissionIDs(execution *AgentExecution, submissions []journal.Submission) ([]string, error) { +func (m *Manager) nativeResumeInterruptedSubmissionIDs(ctx context.Context, execution *AgentExecution, status *agentctl.DeliveryStatus, submissions []journal.Submission) ([]string, error) { + rows, err := activeBackendSubmissions(ctx, m.streamManager.deliveryRepository(), execution.SessionID) + if err != nil { + return nil, err + } + return nativeResumeRetirementIDs(execution, status, submissions, rows) +} + +func nativeResumeRetirementIDs(execution *AgentExecution, status *agentctl.DeliveryStatus, submissions []journal.Submission, rows []*models.AgentDeliverySubmission) ([]string, error) { var ids []string + backend := make(map[string]*models.AgentDeliverySubmission, len(rows)) + for _, row := range rows { + backend[row.ID] = row + } for _, submission := range submissions { + if submission.Retired { + continue + } switch submission.State { case journal.SubmissionPrepared, journal.SubmissionAccepted, journal.SubmissionDispatching: return nil, fmt.Errorf("native resume cannot acknowledge live delivery work") - case journal.SubmissionCompleted: - if !submission.TerminalEventRetained { - return nil, fmt.Errorf("native resume cannot acknowledge incomplete terminal evidence") - } - case journal.SubmissionInterruptedUnknown: - if submission.Retired { + case journal.SubmissionInterruptedUnknown, journal.SubmissionCompleted, journal.SubmissionFailed, journal.SubmissionCancelled: + if !journal.HasUnprovenOutcome(submission) { continue } - if submission.SessionID != execution.SessionID || submission.IncarnationID != execution.DeliveryIncarnationID || - submission.HarnessGeneration == 0 || submission.HarnessGeneration > execution.DeliveryHarnessGeneration { - return nil, journal.ErrOwnerMismatch + sqlSubmission, found := backend[submission.ID] + if !found || sqlSubmission.State != models.DeliverySubmissionInterruptedUnknown { + if submission.State == journal.SubmissionFailed || submission.State == journal.SubmissionCancelled { + continue + } + return nil, fmt.Errorf("%w: native resume requires uncertain SQL history", ErrDurableAdoptionBlocked) + } + if !peerMatchesUnknownHistory(execution, status, submission, sqlSubmission) { + return nil, fmt.Errorf("%w: native resume submission identity changed", ErrDurableAdoptionBlocked) } ids = append(ids, submission.ID) + default: + return nil, fmt.Errorf("%w: unsupported native resume submission state", ErrDurableAdoptionBlocked) } } return ids, nil diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery_test.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery_test.go index 1c30c7f4002..463d011c8ba 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery_test.go @@ -63,7 +63,7 @@ func nativeResumeJournalPeer(t *testing.T, store *journal.Journal, rejectRetirem // @covers AC-PLATFORM-DURABLE-AGENT-DELIVERY-006.12 func TestNativeResumeAcknowledgesRealJournalBeforeNewWork(t *testing.T) { - for _, condition := range []string{"interrupted", "earlier generation", "live work", "retirement rejected", "uninitialized", "wrong incarnation"} { + for _, condition := range []string{"interrupted", "earlier generation", "legacy failure", "live work", "retirement rejected", "uninitialized", "wrong incarnation"} { t.Run(condition, func(t *testing.T) { ctx := context.Background() store, err := journal.Open(journal.Config{Path: filepath.Join(t.TempDir(), "delivery.bbolt")}) @@ -73,6 +73,9 @@ func TestNativeResumeAcknowledgesRealJournalBeforeNewWork(t *testing.T) { if condition == "earlier generation" { original.HarnessGeneration = 1 } + if condition == "legacy failure" { + original.State = journal.SubmissionFailed + } _, err = store.PutSubmission(ctx, original) require.NoError(t, err) if condition == "live work" { @@ -96,18 +99,25 @@ func TestNativeResumeAcknowledgesRealJournalBeforeNewWork(t *testing.T) { resumer, ok := any(mgr).(nativeDeliveryResumer) require.True(t, ok, "explicit native resume must reconcile the retained journal") err = resumer.AcknowledgeNativeResumeDelivery(ctx, execution.SessionID) - if condition == "interrupted" || condition == "earlier generation" { + if condition == "interrupted" || condition == "earlier generation" || condition == "legacy failure" { require.NoError(t, err) } else { require.Error(t, err) } stored, err := store.GetSubmission(ctx, original.ID) require.NoError(t, err) - require.Equal(t, condition == "interrupted" || condition == "earlier generation", stored.Retired) + require.Equal(t, condition == "interrupted" || condition == "earlier generation" || condition == "legacy failure", stored.Retired) require.Equal(t, original.State, stored.State) require.Equal(t, original.Payload, stored.Payload) require.Equal(t, "native-conversation", execution.ACPSessionID) if stored.Retired { + proof, err := mgr.GetNativeResumeDeliveryEvidence(ctx, execution.SessionID) + require.NoError(t, err) + require.Equal(t, execution.ACPSessionID, proof.NativeSessionID) + require.Equal(t, execution.DeliveryStreamID, proof.StreamID) + require.Len(t, proof.Submissions, 1) + require.Equal(t, original.Hash, proof.Submissions[0].PayloadHash) + require.NoError(t, mgr.AcknowledgeNativeResumeDelivery(ctx, execution.SessionID), "repeated recovery must use retained evidence") status, err := execution.agentctl.GetDeliveryStatus(ctx, execution.DeliveryStreamID) require.NoError(t, err) require.NoError(t, newAdoptionManager(repository).restoreRecoveredSubmission(ctx, execution, status, repository, execution.agentctl)) diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_dispatch_reconciliation.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_dispatch_reconciliation.go index eab63f72201..7f2a3f6513e 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/native_resume_dispatch_reconciliation.go +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_dispatch_reconciliation.go @@ -36,7 +36,7 @@ func (m *Manager) reconcileNativeResumeSQLDispatch(ctx context.Context, executio abandoned = append(abandoned, row) } evidence, found := retained[row.ID] - if !found || !peerMatchesUnknownHistory(execution, status, evidence, &candidate) { + if !found || (row.State == models.DeliverySubmissionDispatching && evidence.State != journal.SubmissionInterruptedUnknown) || !peerMatchesUnknownHistory(execution, status, evidence, &candidate) { return fmt.Errorf("%w: native resume peer and SQL submission evidence disagree", ErrDurableAdoptionBlocked) } } diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_evidence.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_evidence.go index 305c847af48..0d6b5e397a5 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/native_resume_evidence.go +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_evidence.go @@ -6,6 +6,7 @@ import ( agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/agentctl/journal" + "github.com/kandev/kandev/internal/task/models" ) // Validate the complete SQL history before retiring any peer work and require @@ -28,8 +29,15 @@ func (m *Manager) validateNativeResumeSQLHistory( if err != nil { return err } + return validateNativeResumeSQLRows(execution, status, peer, active, retired) +} + +func validateNativeResumeSQLRows(execution *AgentExecution, status *agentctl.DeliveryStatus, peer []journal.Submission, active []*models.AgentDeliverySubmission, retired bool) error { retained := make(map[string]journal.Submission, len(peer)) for _, submission := range peer { + if _, duplicate := retained[submission.ID]; duplicate { + return fmt.Errorf("%w: duplicate native resume evidence", ErrDurableAdoptionBlocked) + } retained[submission.ID] = submission } for _, submission := range active { diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof.go new file mode 100644 index 00000000000..3977c6fe106 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof.go @@ -0,0 +1,105 @@ +package lifecycle + +import ( + "context" + "database/sql" + "errors" + "fmt" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/task/models" +) + +// GetNativeResumeDeliveryEvidence revalidates retirement before SQL clears an +// admission fence. The snapshot cannot authorize dispatch or alter old outcomes. +func (m *Manager) GetNativeResumeDeliveryEvidence(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { + execution, exists := m.GetExecutionBySessionID(sessionID) + if !exists || execution == nil { + return nil, ErrDurableAdoptionEvidenceUnavailable + } + client, release := execution.AcquireAgentCtlClient() + defer release() + if client == nil { + return nil, ErrDurableAdoptionEvidenceUnavailable + } + status, err := m.nativeResumeDeliveryStatus(ctx, execution, client) + if err != nil || status == nil { + return nil, err + } + if m.streamManager == nil || m.streamManager.deliveryRepository() == nil { + return nil, ErrDurableAdoptionEvidenceUnavailable + } + repository := m.streamManager.deliveryRepository() + if _, err := authenticatedDeliveryReplayCursor(ctx, execution, repository, status); err != nil { + return nil, err + } + if status.Unresolved { + return nil, fmt.Errorf("%w: native resume journal work remains unresolved", ErrDurableAdoptionBlocked) + } + rows, streams, err := m.nativeResumeProofRows(ctx, execution, status, client, repository) + if err != nil { + return nil, err + } + proof := &models.NativeResumeDeliveryEvidence{SessionID: sessionID, IncarnationID: status.IncarnationID, + HarnessGeneration: int64(status.HarnessGeneration), StreamID: status.StreamID, + ExecutionID: execution.ID, NativeSessionID: execution.ACPSessionID} + proof.Cursor, err = nativeResumeProofCursor(ctx, repository, status) + if err != nil { + return nil, err + } + for _, row := range rows { + proof.Submissions = append(proof.Submissions, models.NativeResumeSubmissionEvidence{ID: row.ID, + HarnessGeneration: row.HarnessGeneration, PayloadHash: row.PayloadHash, StreamID: streams[row.ID], + TerminalSequence: row.TerminalSequence, TerminalStreamID: row.TerminalStreamID}) + } + after, err := client.GetDeliveryStatus(ctx, execution.DeliveryStreamID) + if err != nil { + return nil, err + } + if after == nil || !sameDeliveryRecordDescriptor(*status, *after) { + return nil, ErrDeliveryRecordEvidenceStale + } + return proof, nil +} + +func (m *Manager) nativeResumeProofRows(ctx context.Context, execution *AgentExecution, status *agentctl.DeliveryStatus, client *agentctl.Client, repository AgentDeliveryRepository) ([]*models.AgentDeliverySubmission, map[string]string, error) { + peer, err := client.ListDeliverySubmissions(ctx, execution.SessionID) + if err != nil { + return nil, nil, err + } + rows, err := activeBackendSubmissions(ctx, repository, execution.SessionID) + if err != nil { + return nil, nil, err + } + if err := validateNativeResumeSQLRows(execution, status, peer, rows, true); err != nil { + return nil, nil, err + } + if ids, err := nativeResumeRetirementIDs(execution, status, peer, rows); err != nil || len(ids) != 0 { + return nil, nil, fmt.Errorf("%w: native resume inventory has unsettled work: %v", ErrDurableAdoptionBlocked, err) + } + streams := make(map[string]string, len(peer)) + for _, submission := range peer { + streams[submission.ID] = submission.StreamID + } + return rows, streams, nil +} + +func nativeResumeProofCursor(ctx context.Context, repository AgentDeliveryRepository, status *agentctl.DeliveryStatus) (*models.AgentDeliveryCursor, error) { + cursor, err := repository.GetAgentDeliveryCursor(ctx, status.StreamID) + if errors.Is(err, sql.ErrNoRows) && (status.Stream == nil || status.Stream.HighWater == 0) { + return nil, nil + } + if err != nil { + return nil, err + } + if err := validateAuthenticatedSQLDeliveryCursor(cursor, status); err != nil { + return nil, err + } + if _, err := projectedCursorCoveredByPeer(cursor, status); err != nil { + return nil, err + } + if cursor.ReceivedSequence != cursor.ProjectedSequence { + return nil, fmt.Errorf("%w: native resume projection is incomplete", ErrDurableAdoptionBlocked) + } + return cursor, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_state_matrix_test.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_state_matrix_test.go new file mode 100644 index 00000000000..390a13dddf4 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_state_matrix_test.go @@ -0,0 +1,82 @@ +package lifecycle + +import ( + "context" + "path/filepath" + "testing" + + "github.com/kandev/kandev/internal/agentctl/journal" + "github.com/kandev/kandev/internal/task/models" + "github.com/stretchr/testify/require" +) + +// The table describes admission after native restoration with no retained +// terminal. R retires unknown history, K preserves settled history, B blocks. +// A dispatch label alone is never evidence that an external call did not occur. +func TestNativeResumeCrossStoreStateDisposition(t *testing.T) { + states := []string{"prepared", "accepted", "dispatching", "interrupted_unknown", "completed", "failed", "cancelled"} + model := []string{ + "BBBBBBB", // SQL prepared + "BBBBBBB", // SQL accepted + "BBBRBBB", // SQL dispatching: only journal unknown proves abandonment + "BBBRRRR", // SQL unknown: retain historical labels while retiring + "BBBBBKK", // SQL completed: no terminal journal evidence may be invented + "BBBBBKK", // SQL failed + "BBBBBKK", // SQL cancelled + } + for sqlIndex, sqlState := range states { + for peerIndex, peerState := range states { + t.Run(sqlState+"/"+peerState, func(t *testing.T) { + ctx := context.Background() + store, err := journal.Open(journal.Config{Path: filepath.Join(t.TempDir(), "delivery.bbolt")}) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + peer := journal.Submission{ID: "prompt:queue:interrupted", SessionID: "session", IncarnationID: "incarnation", + HarnessGeneration: 2, StreamID: "stream", Hash: "immutable-hash", Payload: []byte("old instruction"), State: journal.SubmissionState(peerState)} + _, err = store.PutSubmission(ctx, peer) + require.NoError(t, err) + repo, _ := runOwnerTestRepository(t) + require.NoError(t, repo.CreateTask(ctx, &models.Task{ID: "task", Title: "State model"})) + require.NoError(t, repo.CreateTaskSession(ctx, &models.TaskSession{ID: "session", TaskID: "task", QueueIncarnationID: "incarnation"})) + _, err = repo.PrepareAgentDeliverySubmission(ctx, &models.AgentDeliverySubmission{ID: peer.ID, SessionID: peer.SessionID, + IncarnationID: peer.IncarnationID, HarnessGeneration: 2, OwnerGeneration: 2, PayloadHash: peer.Hash, Payload: peer.Payload, State: models.DeliverySubmissionState(sqlState)}) + require.NoError(t, err) + manager := newTestManager(t) + manager.SetAgentDeliveryRepository(repo) + execution := &AgentExecution{ID: "replacement", SessionID: "session", ACPSessionID: "saved-native", DeliveryIncarnationID: "incarnation", + DeliveryHarnessGeneration: 2, DeliveryStreamID: "stream", agentctl: nativeResumeJournalPeer(t, store, false)} + execution.setSessionInitialized(true) + require.NoError(t, manager.executionStore.Add(execution)) + err = manager.AcknowledgeNativeResumeDelivery(ctx, "session") + disposition := model[sqlIndex][peerIndex] + if disposition == 'B' { + require.Error(t, err) + } else { + require.NoError(t, err) + } + stored, err := store.GetSubmission(ctx, peer.ID) + require.NoError(t, err) + require.Equal(t, disposition == 'R', stored.Retired) + require.Equal(t, peer.State, stored.State) + require.Equal(t, peer.Payload, stored.Payload) + require.Equal(t, peer.Hash, stored.Hash) + require.Zero(t, stored.TerminalSequence) + row, err := repo.GetAgentDeliverySubmission(ctx, peer.ID) + require.NoError(t, err) + if disposition == 'R' { + require.Equal(t, models.DeliverySubmissionInterruptedUnknown, row.State) + } else { + require.Equal(t, models.DeliverySubmissionState(sqlState), row.State) + } + require.Zero(t, row.TerminalSequence) + if disposition == 'R' { + require.NoError(t, manager.AcknowledgeNativeResumeDelivery(ctx, "session"), "repeated recovery must converge") + status, err := execution.agentctl.GetDeliveryStatus(ctx, "stream") + require.NoError(t, err) + require.NoError(t, newAdoptionManager(repo).restoreRecoveredSubmission(ctx, execution, status, repo, execution.agentctl)) + require.Empty(t, execution.deliverySubmissionIDSnapshot(), "old queue instruction cannot become current work") + } + }) + } + } +} diff --git a/apps/backend/internal/agentctl/journal/journal_submissions.go b/apps/backend/internal/agentctl/journal/journal_submissions.go index 30a727dca59..5dceca44b89 100644 --- a/apps/backend/internal/agentctl/journal/journal_submissions.go +++ b/apps/backend/internal/agentctl/journal/journal_submissions.go @@ -80,7 +80,7 @@ func (j *Journal) ListSubmissions(ctx context.Context, sessionID string) ([]Subm // RetireSubmission seals an explicitly recovered submission. Native resume // may acknowledge an interrupted unknown outcome in the same generation; -// retiring other work requires a newer generation. +// unproven terminal labels retain their state; live work requires a newer generation. func (j *Journal) RetireSubmission(ctx context.Context, id string, recoveryGeneration uint64) (Submission, error) { j.mu.RLock() defer j.mu.RUnlock() @@ -125,7 +125,7 @@ func (j *Journal) RetireSubmission(ctx context.Context, id string, recoveryGener func canRetireSubmissionAtGeneration(submission Submission, recoveryGeneration uint64) bool { return recoveryGeneration > 0 && (recoveryGeneration > submission.HarnessGeneration || - (recoveryGeneration == submission.HarnessGeneration && submission.State == SubmissionInterruptedUnknown)) + (recoveryGeneration == submission.HarnessGeneration && HasUnprovenOutcome(submission))) } func retireSubmissionTx(ctx context.Context, tx *bolt.Tx, id string, recoveryGeneration uint64, updatedAt time.Time) (Submission, error) { diff --git a/apps/backend/internal/agentctl/journal/native_resume_test.go b/apps/backend/internal/agentctl/journal/native_resume_test.go index 42a7a9089d7..69695aa553a 100644 --- a/apps/backend/internal/agentctl/journal/native_resume_test.go +++ b/apps/backend/internal/agentctl/journal/native_resume_test.go @@ -34,7 +34,7 @@ func TestNativeResumeRetiresUnknownWithoutInventingOutcome(t *testing.T) { } func TestNativeResumeCannotRetireLiveWork(t *testing.T) { - for _, state := range []SubmissionState{SubmissionPrepared, SubmissionAccepted, SubmissionDispatching, SubmissionCompleted} { + for _, state := range []SubmissionState{SubmissionPrepared, SubmissionAccepted, SubmissionDispatching} { t.Run(string(state), func(t *testing.T) { j, event := terminalSubmissionFixture(t, state, false) _, err := j.RetireSubmission(context.Background(), event.SubmissionID, event.HarnessGeneration) @@ -67,3 +67,33 @@ func TestRecoveredUnknownHistorySurvivesGenerationAndReopen(t *testing.T) { require.NoError(t, err) require.False(t, unresolved) } + +func TestNativeResumeRetiresUnprovenTerminalLabelsWithoutChangingThem(t *testing.T) { + for _, state := range []SubmissionState{SubmissionFailed, SubmissionCompleted, SubmissionCancelled} { + t.Run(string(state), func(t *testing.T) { + ctx := context.Background() + j, event := terminalSubmissionFixture(t, state, false) + before, err := j.GetSubmission(ctx, event.SubmissionID) + require.NoError(t, err) + retired, err := j.RetireSubmission(ctx, before.ID, before.HarnessGeneration) + require.NoError(t, err) + require.True(t, retired.Retired) + require.Equal(t, before.State, retired.State) + require.Equal(t, before.Payload, retired.Payload) + require.Equal(t, before.Hash, retired.Hash) + require.False(t, retired.TerminalEventRetained) + require.Zero(t, retired.TerminalSequence) + require.NoError(t, j.Close()) + reopened, err := Open(j.config) + require.NoError(t, err) + t.Cleanup(func() { _ = reopened.Close() }) + stored, err := reopened.GetSubmission(ctx, before.ID) + require.NoError(t, err) + require.True(t, stored.Retired) + require.Equal(t, state, stored.State) + unresolved, err := reopened.HasUnresolvedSubmissions(ctx) + require.NoError(t, err) + require.False(t, unresolved) + }) + } +} diff --git a/apps/backend/internal/agentctl/journal/recovery.go b/apps/backend/internal/agentctl/journal/recovery.go index 21b05f7436e..182115cb899 100644 --- a/apps/backend/internal/agentctl/journal/recovery.go +++ b/apps/backend/internal/agentctl/journal/recovery.go @@ -181,7 +181,7 @@ func submissionSummary(submission Submission) SubmissionSummary { } func submissionNeedsRecovery(submission Submission) bool { - if submission.Retired && submission.State == SubmissionInterruptedUnknown { + if submission.Retired && (submission.State == SubmissionInterruptedUnknown || HasUnprovenOutcome(submission)) { return false } switch submission.State { diff --git a/apps/backend/internal/agentctl/journal/submission_outcome.go b/apps/backend/internal/agentctl/journal/submission_outcome.go new file mode 100644 index 00000000000..4f18d0b1e36 --- /dev/null +++ b/apps/backend/internal/agentctl/journal/submission_outcome.go @@ -0,0 +1,15 @@ +package journal + +// HasUnprovenOutcome identifies history that native restoration can acknowledge +// without changing its recorded outcome. It is not permission to retire live work. +func HasUnprovenOutcome(submission Submission) bool { + if submission.TerminalEventRetained || submission.TerminalSequence != 0 { + return false + } + switch submission.State { + case SubmissionInterruptedUnknown, SubmissionFailed, SubmissionCompleted, SubmissionCancelled: + return true + default: + return false + } +} diff --git a/apps/backend/internal/agentctl/server/process/manager.go b/apps/backend/internal/agentctl/server/process/manager.go index 1355af398c1..42b771a29ae 100644 --- a/apps/backend/internal/agentctl/server/process/manager.go +++ b/apps/backend/internal/agentctl/server/process/manager.go @@ -712,7 +712,8 @@ func (m *Manager) DispatchDeliverySubmission( m.deliveryActiveID = "" m.deliveryActiveMu.Unlock() }() - return (&SubmissionDelivery{Journal: deliveryJournal}).Dispatch(ctx, id, call) + stopped, _ := m.stopChSnapshot.Load().(chan struct{}) + return (&SubmissionDelivery{Journal: deliveryJournal, Stopped: stopped}).Dispatch(ctx, id, call) } func (m *Manager) activeDeliverySubmissionID() string { @@ -785,7 +786,7 @@ func (m *Manager) RetireDeliverySubmission(ctx context.Context, id string, recov sessionID, incarnationID, generation := m.DeliverySubmissionIdentity() if submission.SessionID != sessionID || submission.IncarnationID != incarnationID || generation != recoveryGeneration || submission.HarnessGeneration > generation || - (submission.HarnessGeneration == generation && submission.State != journal.SubmissionInterruptedUnknown) { + (submission.HarnessGeneration == generation && !journal.HasUnprovenOutcome(submission)) { return journal.ErrOwnerMismatch } _, err = deliveryJournal.RetireSubmission(ctx, id, recoveryGeneration) @@ -2908,6 +2909,12 @@ func (m *Manager) stopShellAndProcesses(ctx context.Context) error { // It reports whether this call closed stopCh, i.e. whether goroutines were just // released and still need draining. func (m *Manager) closeAdapterAndStdin() bool { + // Publish this child's interruption before closing the adapter can release a prompt RPC. + closedStopCh := false + if m.stopCh != nil && m.stopChClosed.CompareAndSwap(false, true) { + close(m.stopCh) + closedStopCh = true + } m.deliveryJournalMu.RLock() writer := m.deliveryWriter m.deliveryJournalMu.RUnlock() @@ -2922,12 +2929,6 @@ func (m *Manager) closeAdapterAndStdin() bool { } m.logger.Debug("adapter closed") - m.logger.Debug("closing stop channel") - closedStopCh := false - if m.stopCh != nil && m.stopChClosed.CompareAndSwap(false, true) { - close(m.stopCh) - closedStopCh = true - } m.logger.Debug("stop channel closed", zap.Bool("closed_now", closedStopCh)) // Close stdin to signal EOF to agent diff --git a/apps/backend/internal/agentctl/server/process/submission_delivery.go b/apps/backend/internal/agentctl/server/process/submission_delivery.go index 6532e901aca..d84e5753f3b 100644 --- a/apps/backend/internal/agentctl/server/process/submission_delivery.go +++ b/apps/backend/internal/agentctl/server/process/submission_delivery.go @@ -31,6 +31,7 @@ func isKnownSubmissionFailure(err error) bool { type SubmissionDelivery struct { Journal *journal.Journal Now func() time.Time + Stopped <-chan struct{} } func (d *SubmissionDelivery) Admit(ctx context.Context, submission journal.Submission) (journal.Submission, error) { @@ -102,21 +103,35 @@ func (d *SubmissionDelivery) prepareDispatch(ctx context.Context, id string) (jo } func (d *SubmissionDelivery) handleDispatchError(ctx context.Context, id string, dispatchErr error) (journal.Submission, error) { - submission, err := d.Journal.GetSubmission(ctx, id) + interrupted := ctx.Err() != nil + select { + case <-d.Stopped: + interrupted = true + default: + } + // Settling an external call survives cancellation of the request that made it. + settleCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + submission, err := d.Journal.GetSubmission(settleCtx, id) if err != nil { return journal.Submission{}, errors.Join(ErrSubmissionUncertain, err) } if submission.State == journal.SubmissionCompleted && submission.TerminalEventRetained { return submission, nil } - if isKnownSubmissionFailure(dispatchErr) { - failed, transitionErr := d.Journal.TransitionSubmission(ctx, id, journal.SubmissionFailed, d.now()) + if submission.TerminalEventRetained { + return submission, dispatchErr + } + var providerFailure deterministicPromptFailure + knownProviderFailure := errors.As(dispatchErr, &providerFailure) && providerFailure.DeterministicPromptFailure() + if knownProviderFailure || (!interrupted && isKnownSubmissionFailure(dispatchErr)) { + failed, transitionErr := d.Journal.TransitionSubmission(settleCtx, id, journal.SubmissionFailed, d.now()) if transitionErr != nil { return journal.Submission{}, errors.Join(ErrSubmissionUncertain, transitionErr) } return failed, dispatchErr } - unknown, transitionErr := d.Journal.TransitionSubmission(ctx, id, journal.SubmissionInterruptedUnknown, d.now()) + unknown, transitionErr := d.Journal.TransitionSubmission(settleCtx, id, journal.SubmissionInterruptedUnknown, d.now()) journal.RecordUncertainSubmission("dispatch_error") if transitionErr != nil { return journal.Submission{}, errors.Join(ErrSubmissionUncertain, transitionErr) diff --git a/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go b/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go new file mode 100644 index 00000000000..9348ebae9ef --- /dev/null +++ b/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go @@ -0,0 +1,124 @@ +package process + +import ( + "context" + "errors" + "path/filepath" + "sync" + "testing" + "time" + + acp "github.com/coder/acp-go-sdk" + "github.com/kandev/kandev/internal/agentctl/journal" + "github.com/kandev/kandev/internal/agentctl/server/adapter" + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/pkg/agent" + "github.com/stretchr/testify/require" +) + +type shutdownRequestAdapter struct { + adapter.AgentAdapter + closing chan struct{} + dispatchDone chan struct{} + once sync.Once +} + +func (a *shutdownRequestAdapter) Close() error { + a.once.Do(func() { close(a.closing) }) + select { + case <-a.dispatchDone: + case <-time.After(5 * time.Second): + return errors.New("durable dispatch did not settle during adapter shutdown") + } + return a.AgentAdapter.Close() +} + +func TestSubmissionGracefulStopPreservesUnknownRequestOutcome(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + mgr := NewManager(&config.InstanceConfig{ + AgentArgs: fixtureArgs(), AgentEnv: fixtureEnvSlice("cat"), WorkDir: t.TempDir(), Protocol: agent.ProtocolACP, + SessionID: "session", DeliveryIncarnationID: "incarnation", DeliveryHarnessGeneration: 1, + DeliveryStreamID: "incarnation:g1", DurableJournalPath: filepath.Join(t.TempDir(), "delivery.bbolt"), + }, newTestLogger(t)) + require.NoError(t, mgr.Start(ctx)) + t.Cleanup(func() { _ = mgr.Stop(context.Background()); _ = mgr.closeDeliveryJournal() }) + wrapped := &shutdownRequestAdapter{AgentAdapter: mgr.adapter, closing: make(chan struct{}), dispatchDone: make(chan struct{})} + mgr.adapter = wrapped + original := journal.Submission{ID: "prompt:old", SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: 1, + StreamID: "incarnation:g1", Payload: []byte("do not replay"), Hash: journal.SubmissionHash([]byte("do not replay"))} + _, err := mgr.AdmitDeliverySubmission(ctx, original) + require.NoError(t, err) + began := make(chan struct{}) + result := make(chan error, 1) + go func() { + _, err := mgr.DispatchDeliverySubmission(context.Background(), original.ID, func(context.Context) error { + close(began) + <-wrapped.closing + return &acp.RequestError{Code: -32603, Message: "prompt aborted during shutdown"} + }) + result <- err + close(wrapped.dispatchDone) + }() + select { + case <-began: + case <-ctx.Done(): + t.Fatal("durable prompt did not start") + } + require.NoError(t, mgr.Stop(ctx)) + require.Error(t, <-result) + retained, err := mgr.DeliveryJournal() + require.NoError(t, err) + stored, err := retained.GetSubmission(context.Background(), original.ID) + require.NoError(t, err) + require.Equal(t, journal.SubmissionInterruptedUnknown, stored.State) + require.False(t, stored.TerminalEventRetained) + require.Zero(t, stored.TerminalSequence) + require.Equal(t, original.Hash, stored.Hash) + require.Equal(t, original.Payload, stored.Payload) + calls := 0 + _, err = mgr.DispatchDeliverySubmission(context.Background(), original.ID, func(context.Context) error { calls++; return nil }) + require.ErrorIs(t, err, ErrSubmissionUncertain) + require.Zero(t, calls) +} + +type rejectedPromptError struct{} + +func (rejectedPromptError) Error() string { return "provider rejected prompt" } +func (rejectedPromptError) DeterministicPromptFailure() bool { return true } + +func TestSubmissionCancellationSettlesJournalAndKeepsKnownFailures(t *testing.T) { + for _, condition := range []string{"transport", "request cancelled", "provider rejected"} { + t.Run(condition, func(t *testing.T) { + store, err := journal.Open(journal.Config{Path: filepath.Join(t.TempDir(), "delivery.bbolt")}) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + delivery := &SubmissionDelivery{Journal: store} + _, err = delivery.Admit(context.Background(), journal.Submission{ID: "prompt", Hash: "immutable", Payload: []byte("never replay")}) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stored, err := delivery.Dispatch(ctx, "prompt", func(context.Context) error { + cancel() + switch condition { + case "request cancelled": + return &acp.RequestError{Code: -32603, Message: "aborted"} + case "provider rejected": + return rejectedPromptError{} + default: + return context.Canceled + } + }) + require.Error(t, err) + want := journal.SubmissionInterruptedUnknown + if condition == "provider rejected" { + want = journal.SubmissionFailed + } + require.Equal(t, want, stored.State) + retained, err := store.GetSubmission(context.Background(), "prompt") + require.NoError(t, err) + require.Equal(t, want, retained.State) + require.False(t, retained.TerminalEventRetained) + }) + } +} diff --git a/apps/backend/internal/backendapp/adapters_native_resume.go b/apps/backend/internal/backendapp/adapters_native_resume.go index 5ac21e683c4..e75adf99e3f 100644 --- a/apps/backend/internal/backendapp/adapters_native_resume.go +++ b/apps/backend/internal/backendapp/adapters_native_resume.go @@ -4,6 +4,7 @@ import ( "context" runtimeapi "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/task/models" ) func (a *lifecycleAdapter) AcknowledgeNativeResumeDelivery(ctx context.Context, sessionID string) error { @@ -13,3 +14,7 @@ func (a *lifecycleAdapter) AcknowledgeNativeResumeDelivery(ctx context.Context, func (a *lifecycleAdapter) ReconcileAlreadyResumedDelivery(ctx context.Context, identity runtimeapi.AgentDeliveryRecoveryIdentity, nativeID string) (bool, error) { return a.mgr.ReconcileAlreadyResumedDelivery(ctx, identity, nativeID) } + +func (a *lifecycleAdapter) GetNativeResumeDeliveryEvidence(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { + return a.mgr.GetNativeResumeDeliveryEvidence(ctx, sessionID) +} diff --git a/apps/backend/internal/orchestrator/native_resume_proof_test.go b/apps/backend/internal/orchestrator/native_resume_proof_test.go new file mode 100644 index 00000000000..2b41db8f28a --- /dev/null +++ b/apps/backend/internal/orchestrator/native_resume_proof_test.go @@ -0,0 +1,50 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" + "github.com/stretchr/testify/require" +) + +type nativeProofTestManager struct { + *nativeResumeTestManager + proof func(context.Context, string) (*models.NativeResumeDeliveryEvidence, error) +} + +func (m *nativeProofTestManager) GetNativeResumeDeliveryEvidence(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { + return m.proof(ctx, sessionID) +} + +func TestNativeResumeRepairsAbsentNoticeAndStreamThroughAuthenticatedProof(t *testing.T) { + ctx := context.Background() + service, manager, block, submission, _, _ := nativeResumeFixture(t, "unknown_prompt_outcome", false, false) + repo := service.repo.(*sqliterepo.Repository) + require.NoError(t, repo.SetSessionMetadataKey(ctx, submission.SessionID, models.SessionMetaKeyAgentDeliveryRecovery, nil)) + _, err := repo.DB().ExecContext(ctx, `UPDATE session_recovery_blocks SET delivery_stream_id = '' WHERE id = ?`, block.ID) + require.NoError(t, err) + service.agentManager = &nativeProofTestManager{nativeResumeTestManager: manager, proof: func(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { + running, err := repo.GetExecutorRunningBySessionID(ctx, sessionID) + require.NoError(t, err) + return &models.NativeResumeDeliveryEvidence{SessionID: sessionID, IncarnationID: submission.IncarnationID, + HarnessGeneration: submission.HarnessGeneration, NativeSessionID: "native-conversation", ExecutionID: running.AgentExecutionID, + StreamID: block.DeliveryStreamID, Submissions: []models.NativeResumeSubmissionEvidence{{ID: submission.ID, + HarnessGeneration: submission.HarnessGeneration, PayloadHash: submission.PayloadHash}}}, nil + }} + response, err := service.RecoverSession(ctx, "task1", submission.SessionID, "resume") + require.NoError(t, err) + require.True(t, response.Success) + restored, err := repo.GetSessionRecoveryBlock(ctx, block.ID) + require.NoError(t, err) + require.Equal(t, models.RecoveryBlockResolved, restored.State) + require.Equal(t, block.DeliveryStreamID, restored.DeliveryStreamID) + require.NoError(t, service.checkSessionRecoveryBlock(ctx, submission.SessionID)) + require.Empty(t, manager.capturedPromptCalls) + retained, err := repo.GetAgentDeliverySubmission(ctx, submission.ID) + require.NoError(t, err) + require.Equal(t, models.DeliverySubmissionInterruptedUnknown, retained.State) + require.Equal(t, submission.Payload, retained.Payload) + require.Zero(t, retained.TerminalSequence) +} diff --git a/apps/backend/internal/orchestrator/restart_submission_reconciliation.go b/apps/backend/internal/orchestrator/restart_submission_reconciliation.go index df7ec707869..eb04598894a 100644 --- a/apps/backend/internal/orchestrator/restart_submission_reconciliation.go +++ b/apps/backend/internal/orchestrator/restart_submission_reconciliation.go @@ -148,7 +148,7 @@ func (s *Service) verifyRetiredRestartHistory(ctx context.Context, inputs *silen } func matchingRetiredRestartSubmission(inputs *silentRestoreInputs, row *models.AgentDeliverySubmission, peer *journal.Submission) bool { - return peer != nil && peer.Retired && peer.State == journal.SubmissionInterruptedUnknown && peer.ID == row.ID && + return peer != nil && peer.Retired && journal.HasUnprovenOutcome(*peer) && row.State == models.DeliverySubmissionInterruptedUnknown && row.TerminalSequence == 0 && peer.ID == row.ID && peer.StreamID != "" && peer.SessionID == inputs.session.ID && row.SessionID == peer.SessionID && peer.IncarnationID == inputs.recovery.IncarnationID && row.IncarnationID == peer.IncarnationID && row.HarnessGeneration > 0 && row.HarnessGeneration <= inputs.recovery.HarnessGeneration && diff --git a/apps/backend/internal/orchestrator/session_launch.go b/apps/backend/internal/orchestrator/session_launch.go index e610e05cfac..d6b3c6557ad 100644 --- a/apps/backend/internal/orchestrator/session_launch.go +++ b/apps/backend/internal/orchestrator/session_launch.go @@ -548,7 +548,14 @@ func (s *Service) resolveSessionRecoveryBlock(ctx context.Context, sessionID, ac if err != nil { return fmt.Errorf("load session recovery block for resolution: %w", err) } - resolved, err := store.ResolveSessionRecoveryBlock(ctx, block.ID, action, time.Now().UTC()) + completedAt := time.Now().UTC() + resolved, handled := false, false + if action == recoveryActionResume { + resolved, handled, err = s.resolveNativeDeliveryBlock(ctx, block, completedAt) + } + if !handled { + resolved, err = store.ResolveSessionRecoveryBlock(ctx, block.ID, action, completedAt) + } if err != nil { return fmt.Errorf("resolve session recovery block: %w", err) } diff --git a/apps/backend/internal/orchestrator/session_native_delivery_recovery.go b/apps/backend/internal/orchestrator/session_native_delivery_recovery.go index ed2ec62e7e9..cd8e27b7eaa 100644 --- a/apps/backend/internal/orchestrator/session_native_delivery_recovery.go +++ b/apps/backend/internal/orchestrator/session_native_delivery_recovery.go @@ -3,6 +3,7 @@ package orchestrator import ( "context" "fmt" + "time" "github.com/kandev/kandev/internal/task/models" ) @@ -76,3 +77,28 @@ func (s *Service) acknowledgeNativeResumeDelivery(ctx context.Context, sessionID func nativeResumeSubmissionMatchesBlock(submission *models.AgentDeliverySubmission, block *models.SessionRecoveryBlock) bool { return submission != nil && submission.IncarnationID == block.IncarnationID && submission.HarnessGeneration == block.ExpectedGeneration } + +type nativeResumeDeliveryEvidenceReader interface { + GetNativeResumeDeliveryEvidence(context.Context, string) (*models.NativeResumeDeliveryEvidence, error) +} + +type nativeResumeEvidenceBlockResolver interface { + ResolveSessionRecoveryBlockWithNativeEvidence(context.Context, string, *models.NativeResumeDeliveryEvidence, time.Time) (bool, error) +} + +func (s *Service) resolveNativeDeliveryBlock(ctx context.Context, block *models.SessionRecoveryBlock, completedAt time.Time) (bool, bool, error) { + reader, canRead := s.agentManager.(nativeResumeDeliveryEvidenceReader) + resolver, canResolve := s.repo.(nativeResumeEvidenceBlockResolver) + if !canRead || !canResolve || block.ConsumerReference != agentDeliveryConsumer { + return false, false, nil + } + proof, err := reader.GetNativeResumeDeliveryEvidence(ctx, block.SessionID) + if err != nil { + return false, true, err + } + if proof == nil { + return false, false, nil + } + changed, err := resolver.ResolveSessionRecoveryBlockWithNativeEvidence(ctx, block.ID, proof, completedAt) + return changed, true, err +} diff --git a/apps/backend/internal/task/models/native_resume_delivery.go b/apps/backend/internal/task/models/native_resume_delivery.go new file mode 100644 index 00000000000..4c7f30ac8f5 --- /dev/null +++ b/apps/backend/internal/task/models/native_resume_delivery.go @@ -0,0 +1,23 @@ +package models + +// NativeResumeDeliveryEvidence is a pinned authenticated snapshot after native +// restoration and journal retirement. SQL verifies it under admission locks. +type NativeResumeDeliveryEvidence struct { + SessionID string + IncarnationID string + HarnessGeneration int64 + ExecutionID string + NativeSessionID string + StreamID string + Cursor *AgentDeliveryCursor + Submissions []NativeResumeSubmissionEvidence +} + +type NativeResumeSubmissionEvidence struct { + StreamID string + ID string + HarnessGeneration int64 + PayloadHash string + TerminalStreamID string + TerminalSequence int64 +} diff --git a/apps/backend/internal/task/repository/sqlite/native_resume_binding.go b/apps/backend/internal/task/repository/sqlite/native_resume_binding.go new file mode 100644 index 00000000000..b2421bc0b04 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/native_resume_binding.go @@ -0,0 +1,239 @@ +package sqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "time" + + "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/task/models" +) + +// ResolveSessionRecoveryBlockWithNativeEvidence commits only after authenticated +// native retirement. All incident fences are verified before any becomes usable. +func (r *Repository) ResolveSessionRecoveryBlockWithNativeEvidence(ctx context.Context, id string, proof *models.NativeResumeDeliveryEvidence, completedAt time.Time) (bool, error) { + if proof == nil || proof.SessionID == "" || proof.IncarnationID == "" || proof.HarnessGeneration <= 0 || + proof.NativeSessionID == "" || proof.StreamID == "" || proof.ExecutionID == "" { + return false, nil + } + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + if err := lockSessionTurnWrites(ctx, tx, r.db.DriverName(), proof.SessionID); err != nil { + return false, err + } + if err := lockAgentDeliveryWrites(ctx, tx, r.db.DriverName(), []string{proof.SessionID}, []string{proof.StreamID}); err != nil { + return false, err + } + valid, err := r.verifyNativeResumeSnapshotTx(ctx, tx, proof) + if err != nil || !valid { + return false, err + } + blocks, err := r.nativeResumeIncidentBlocksTx(ctx, tx, id, proof) + if err != nil || len(blocks) == 0 { + return false, err + } + valid, err = r.restoreAuthenticatedNativeNoticeTx(ctx, tx, proof, completedAt) + if err != nil || !valid { + return false, err + } + if err := r.advanceRecoverySessionRevisionTx(ctx, tx, proof.SessionID); err != nil { + return false, err + } + for _, block := range blocks { + stream := block.DeliveryStreamID + if block.DeliverySubmissionID != "" && stream == "" { + stream = proof.StreamID + } + result, err := tx.ExecContext(ctx, r.db.Rebind(`UPDATE session_recovery_blocks SET delivery_stream_id = ?, state = ?, authorized_action = ?, updated_at = ?, resolved_at = ? WHERE id = ? AND state = ?`), + stream, models.RecoveryBlockResolved, nativeResumeRecoveryAction, completedAt, completedAt, block.ID, models.RecoveryBlockOpen) + if err != nil { + return false, err + } + count, err := result.RowsAffected() + if err != nil { + return false, err + } + if count != 1 { + return false, nil + } + } + return true, tx.Commit() +} + +func (r *Repository) verifyNativeResumeSnapshotTx(ctx context.Context, tx *sqlx.Tx, proof *models.NativeResumeDeliveryEvidence) (bool, error) { + var valid bool + err := tx.QueryRowxContext(ctx, r.db.Rebind(`SELECT EXISTS ( + SELECT 1 FROM task_sessions ts JOIN harness_session_generations g ON g.session_id = ts.id AND g.incarnation_id = ts.queue_incarnation_id + JOIN executors_running e ON e.session_id = ts.id + WHERE ts.id = ? AND ts.queue_incarnation_id = ? AND ts.state <> 'CANCELLED' + AND g.generation = ? AND g.native_session_id = ? AND e.agent_execution_id = ? + AND NOT EXISTS (SELECT 1 FROM harness_session_generations newer WHERE newer.session_id = ts.id AND newer.incarnation_id = g.incarnation_id AND newer.generation > g.generation) + AND NOT EXISTS (SELECT 1 FROM session_restore_attempts WHERE session_id = ts.id AND completed_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM agent_delivery_cursors WHERE session_id = ts.id AND incarnation_id = g.incarnation_id AND harness_generation = g.generation AND received_sequence <> projected_sequence) + )`), proof.SessionID, proof.IncarnationID, proof.HarnessGeneration, proof.NativeSessionID, proof.ExecutionID).Scan(&valid) + if err != nil || !valid { + return false, err + } + valid, err = r.verifyNativeResumeInventoryTx(ctx, tx, proof) + if err != nil || !valid { + return false, err + } + return r.verifyNativeResumeCursorTx(ctx, tx, proof) +} + +func (r *Repository) verifyNativeResumeInventoryTx(ctx context.Context, tx *sqlx.Tx, proof *models.NativeResumeDeliveryEvidence) (bool, error) { + retired := make(map[string]models.NativeResumeSubmissionEvidence, len(proof.Submissions)) + for _, row := range proof.Submissions { + if row.ID == "" || row.PayloadHash == "" || row.HarnessGeneration <= 0 || row.HarnessGeneration > proof.HarnessGeneration { + return false, nil + } + if _, duplicate := retired[row.ID]; duplicate { + return false, nil + } + retired[row.ID] = row + } + rows, err := tx.QueryxContext(ctx, r.db.Rebind(`SELECT id, incarnation_id, harness_generation, payload_hash, state, terminal_stream_id, terminal_sequence FROM agent_delivery_submissions WHERE session_id = ? AND state IN ('prepared', 'accepted', 'dispatching', 'interrupted_unknown')`), proof.SessionID) + if err != nil { + return false, err + } + defer func() { _ = rows.Close() }() + count := 0 + for rows.Next() { + var id, incarnation, hash, stream string + var generation, sequence int64 + var state models.DeliverySubmissionState + if err := rows.Scan(&id, &incarnation, &generation, &hash, &state, &stream, &sequence); err != nil { + return false, err + } + row, found := retired[id] + if !found || state != models.DeliverySubmissionInterruptedUnknown || incarnation != proof.IncarnationID || + generation != row.HarnessGeneration || hash != row.PayloadHash || sequence != row.TerminalSequence || stream != row.TerminalStreamID { + return false, nil + } + count++ + } + return count == len(retired), rows.Err() +} + +func (r *Repository) verifyNativeResumeCursorTx(ctx context.Context, tx *sqlx.Tx, proof *models.NativeResumeDeliveryEvidence) (bool, error) { + var cursor models.AgentDeliveryCursor + err := tx.QueryRowxContext(ctx, r.db.Rebind(`SELECT session_id, incarnation_id, harness_generation, received_sequence, projected_sequence, retired_sequence FROM agent_delivery_cursors WHERE stream_id = ?`), proof.StreamID).Scan( + &cursor.SessionID, &cursor.IncarnationID, &cursor.HarnessGeneration, &cursor.ReceivedSequence, &cursor.ProjectedSequence, &cursor.RetiredSequence) + if errors.Is(err, sql.ErrNoRows) { + return proof.Cursor == nil, nil + } + if err != nil { + return false, err + } + pinned := proof.Cursor + return pinned != nil && pinned.StreamID == proof.StreamID && cursor.SessionID == proof.SessionID && cursor.IncarnationID == proof.IncarnationID && + cursor.HarnessGeneration == proof.HarnessGeneration && cursor.SessionID == pinned.SessionID && cursor.IncarnationID == pinned.IncarnationID && + cursor.HarnessGeneration == pinned.HarnessGeneration && cursor.ReceivedSequence == pinned.ReceivedSequence && + cursor.ProjectedSequence == pinned.ProjectedSequence && cursor.RetiredSequence == pinned.RetiredSequence, nil +} + +func (r *Repository) nativeResumeIncidentBlocksTx(ctx context.Context, tx *sqlx.Tx, id string, proof *models.NativeResumeDeliveryEvidence) ([]models.SessionRecoveryBlock, error) { + rows, err := tx.QueryxContext(ctx, r.db.Rebind(`SELECT id, incarnation_id, expected_generation, reason, consumer_reference, delivery_submission_id, delivery_stream_id, delivery_sequence, delivery_turn_id, delivery_outcome FROM session_recovery_blocks WHERE session_id = ? AND state = 'open'`), proof.SessionID) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + found := false + var blocks []models.SessionRecoveryBlock + for rows.Next() { + var block models.SessionRecoveryBlock + if err := rows.Scan(&block.ID, &block.IncarnationID, &block.ExpectedGeneration, &block.Reason, &block.ConsumerReference, &block.DeliverySubmissionID, + &block.DeliveryStreamID, &block.DeliverySequence, &block.DeliveryTurnID, &block.DeliveryOutcome); err != nil { + return nil, err + } + if !nativeResumeIncidentBlockMatches(block, proof) { + return nil, nil + } + found = found || block.ID == id + blocks = append(blocks, block) + } + if !found { + return nil, rows.Err() + } + return blocks, rows.Err() +} + +func nativeResumeIncidentBlockMatches(block models.SessionRecoveryBlock, proof *models.NativeResumeDeliveryEvidence) bool { + if block.IncarnationID != proof.IncarnationID || block.ExpectedGeneration != proof.HarnessGeneration || block.ConsumerReference != reconstructionDeliveryConsumer || + (block.Reason != "unknown_prompt_outcome" && block.Reason != "unresolved_durable_work") || + block.DeliverySequence != 0 || block.DeliveryTurnID != "" || block.DeliveryOutcome != "" { + return false + } + if block.DeliverySubmissionID == "" { + return block.Reason == "unresolved_durable_work" && block.DeliveryStreamID == "" + } + if block.DeliveryStreamID != "" && block.DeliveryStreamID != proof.StreamID { + return false + } + for _, row := range proof.Submissions { + if row.ID == block.DeliverySubmissionID && row.HarnessGeneration == proof.HarnessGeneration { + return true + } + } + return false +} + +func (r *Repository) restoreAuthenticatedNativeNoticeTx(ctx context.Context, tx *sqlx.Tx, proof *models.NativeResumeDeliveryEvidence, completedAt time.Time) (bool, error) { + var taskID string + var state models.TaskSessionState + var metadataRaw sql.NullString + if err := tx.QueryRowxContext(ctx, r.db.Rebind(`SELECT task_id, state, metadata FROM task_sessions WHERE id = ?`), proof.SessionID).Scan(&taskID, &state, &metadataRaw); err != nil { + return false, err + } + metadata, err := decodeAgentDeliveryMetadata(metadataRaw) + if err != nil { + return false, err + } + var acp struct { + SessionID string `json:"session_id"` + } + if raw := metadata["acp"]; len(raw) != 0 && (json.Unmarshal(raw, &acp) != nil || acp.SessionID != proof.NativeSessionID) { + return false, nil + } + recovery, exists, err := agentDeliveryRecoveryFromMetadata(metadata) + if err != nil || !exists { + return !exists, err + } + if recovery.HistoryGap != nil || recovery.SessionID != proof.SessionID || recovery.IncarnationID != proof.IncarnationID { + return false, nil + } + if recovery.Phase == models.AgentDeliveryRecoveryRestored { + resolved, err := r.hasResolvedNativeResumeRecoveryTx(ctx, tx, recovery) + if err != nil || resolved { + return resolved, err + } + return nativeResumeProofCoversNotice(proof, recovery), nil + } + if recovery.HarnessGeneration != proof.HarnessGeneration || (recovery.StreamID != "" && recovery.StreamID != proof.StreamID) || + (recovery.Phase != models.AgentDeliveryRecoveryUncertain && recovery.Phase != models.AgentDeliveryRecoveryReconnecting) { + return false, nil + } + found := false + for _, row := range proof.Submissions { + found = found || row.ID == recovery.SubmissionID && row.HarnessGeneration == recovery.HarnessGeneration + } + if !found { + return false, nil + } + recovery.StreamID = proof.StreamID + return writeSilentRestoreRecoveryTx(ctx, tx, r.db.Rebind, &silentRestoreSessionRow{taskID: taskID, state: state, metadata: metadata, current: recovery}, metadataRaw, completedAt, "") +} + +func nativeResumeProofCoversNotice(proof *models.NativeResumeDeliveryEvidence, recovery models.AgentDeliveryRecovery) bool { + for _, row := range proof.Submissions { + if row.ID == recovery.SubmissionID && row.HarnessGeneration == recovery.HarnessGeneration && row.StreamID != "" && row.StreamID == recovery.StreamID { + return true + } + } + return false +} diff --git a/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go b/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go new file mode 100644 index 00000000000..4254a276786 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go @@ -0,0 +1,148 @@ +package sqlite + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/testutil" + "github.com/stretchr/testify/require" +) + +type nativeEvidenceResolver interface { + ResolveSessionRecoveryBlockWithNativeEvidence(context.Context, string, *models.NativeResumeDeliveryEvidence, time.Time) (bool, error) +} + +func TestNativeResumeAuthenticatedMissingBinding(t *testing.T) { + testNativeResumeAuthenticatedMissingBinding(t, newRepoForSessionTests) +} + +func TestPostgresNativeResumeAuthenticatedMissingBinding(t *testing.T) { + testNativeResumeAuthenticatedMissingBinding(t, func(t *testing.T) *Repository { + database := openIsolatedPostgresMultiConn(t, testutil.PostgresDSNFromEnv(t), 2) + repo, err := NewWithDB(database, database, nil) + require.NoError(t, err) + return repo + }) +} + +func testNativeResumeAuthenticatedMissingBinding(t *testing.T, open func(*testing.T) *Repository) { + for _, scenario := range []string{"absent notice", "partial notice", "resolved earlier notice", "matching fences", "independent fence", "foreign native", "hash changed", "active work", "unmatched unknown", "foreign stream", "rollback"} { + t.Run(scenario, func(t *testing.T) { + repo := open(t) + ctx := context.Background() + fixture := newSilentRestoreFixture(t, repo, "bound-native", models.TaskSessionStateWaitingForInput) + submission, err := repo.GetAgentDeliverySubmission(ctx, fixture.recovery.SubmissionID) + require.NoError(t, err) + metadata := map[string]any{"acp": map[string]any{"session_id": fixture.generation.NativeSessionID}} + if scenario == "resolved earlier notice" { + changed, err := repo.ResolveSessionRecoveryBlock(ctx, fixture.block.ID, "resume", time.Now().UTC()) + require.NoError(t, err) + require.True(t, changed) + previousSession, err := repo.GetTaskSession(ctx, fixture.session.ID) + require.NoError(t, err) + previous, exists := models.LoadAgentDeliveryRecovery(previousSession.Metadata) + require.True(t, exists) + metadata[models.SessionMetaKeyAgentDeliveryRecovery] = previous + older := *submission + older.ID = "new-current-submission" + _, err = repo.PrepareAgentDeliverySubmission(ctx, &older) + require.NoError(t, err) + fixture.block.ID = "current-block" + fixture.block.State = models.RecoveryBlockOpen + fixture.block.ResolvedAt = nil + fixture.block.AuthorizedAction = "" + fixture.block.DeliverySubmissionID = older.ID + require.NoError(t, repo.UpsertSessionRecoveryBlock(ctx, &fixture.block)) + } + if scenario == "matching fences" || scenario == "independent fence" { + companion := fixture.block + companion.ID = "companion" + companion.Reason = "unresolved_durable_work" + companion.DeliverySubmissionID, companion.DeliveryStreamID = "", "" + if scenario == "independent fence" { + companion.ConsumerReference = "office" + } + require.NoError(t, repo.UpsertSessionRecoveryBlock(ctx, &companion)) + } + + if scenario == "partial notice" { + partial := fixture.recovery + partial.StreamID = "" + metadata[models.SessionMetaKeyAgentDeliveryRecovery] = partial + } + raw, err := json.Marshal(metadata) + require.NoError(t, err) + _, err = repo.DB().ExecContext(ctx, repo.db.Rebind(`UPDATE task_sessions SET metadata = ? WHERE id = ?`), string(raw), fixture.session.ID) + require.NoError(t, err) + _, err = repo.DB().ExecContext(ctx, repo.db.Rebind(`UPDATE session_recovery_blocks SET delivery_stream_id = '' WHERE id = ?`), fixture.block.ID) + require.NoError(t, err) + proof := &models.NativeResumeDeliveryEvidence{SessionID: fixture.session.ID, IncarnationID: fixture.recovery.IncarnationID, + HarnessGeneration: 1, NativeSessionID: fixture.generation.NativeSessionID, StreamID: fixture.recovery.StreamID, + ExecutionID: fixture.recovery.AgentExecutionID, + Submissions: []models.NativeResumeSubmissionEvidence{{ID: submission.ID, HarnessGeneration: 1, PayloadHash: submission.PayloadHash, StreamID: fixture.recovery.StreamID}}} + if scenario == "resolved earlier notice" { + current, err := repo.GetAgentDeliverySubmission(ctx, fixture.block.DeliverySubmissionID) + require.NoError(t, err) + proof.Submissions = append(proof.Submissions, models.NativeResumeSubmissionEvidence{ID: current.ID, HarnessGeneration: 1, PayloadHash: current.PayloadHash}) + } + switch scenario { + case "foreign native": + proof.NativeSessionID = "foreign" + case "hash changed": + proof.Submissions[0].PayloadHash = "changed" + case "foreign stream": + _, err = repo.DB().ExecContext(ctx, repo.db.Rebind(`UPDATE session_recovery_blocks SET delivery_stream_id = 'foreign-stream' WHERE id = ?`), fixture.block.ID) + require.NoError(t, err) + case "active work", "unmatched unknown": + state := models.DeliverySubmissionAccepted + if scenario == "unmatched unknown" { + state = models.DeliverySubmissionInterruptedUnknown + } + _, err = repo.PrepareAgentDeliverySubmission(ctx, &models.AgentDeliverySubmission{ID: "independent", SessionID: proof.SessionID, IncarnationID: proof.IncarnationID, + HarnessGeneration: 1, OwnerGeneration: 1, State: state, PayloadHash: "independent", Payload: []byte("independent")}) + require.NoError(t, err) + case "rollback": + if repo.db.DriverName() == "sqlite3" { + _, err = repo.DB().ExecContext(ctx, `CREATE TRIGGER reject_bound_resume BEFORE UPDATE ON session_recovery_blocks WHEN NEW.state = 'resolved' BEGIN SELECT RAISE(ABORT, 'reject resolution'); END`) + require.NoError(t, err) + } else { + t.Skip("SQLite write-boundary injection; PostgreSQL transactional controls have separate coverage") + } + } + resolver, ok := any(repo).(nativeEvidenceResolver) + require.True(t, ok, "authenticated retirement must support guarded missing binding recovery") + changed, err := resolver.ResolveSessionRecoveryBlockWithNativeEvidence(ctx, fixture.block.ID, proof, time.Now().UTC()) + if scenario == "rollback" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + want := scenario == "absent notice" || scenario == "partial notice" || scenario == "resolved earlier notice" || scenario == "matching fences" + require.Equal(t, want, changed) + block, err := repo.GetSessionRecoveryBlock(ctx, fixture.block.ID) + require.NoError(t, err) + if want { + require.Equal(t, models.RecoveryBlockResolved, block.State) + require.Equal(t, proof.StreamID, block.DeliveryStreamID) + if scenario == "matching fences" { + companion, err := repo.GetSessionRecoveryBlock(ctx, "companion") + require.NoError(t, err) + require.Equal(t, models.RecoveryBlockResolved, companion.State) + } + } else { + require.Equal(t, models.RecoveryBlockOpen, block.State) + if scenario != "foreign stream" { + require.Empty(t, block.DeliveryStreamID) + } + } + stored, err := repo.GetAgentDeliverySubmission(ctx, submission.ID) + require.NoError(t, err) + require.Equal(t, submission.State, stored.State) + require.Equal(t, submission.PayloadHash, stored.PayloadHash) + require.Zero(t, stored.TerminalSequence) + }) + } +} diff --git a/docs/decisions/2026-10-12-total-session-recovery.md b/docs/decisions/2026-10-12-total-session-recovery.md new file mode 100644 index 00000000000..d6b57e6cdfb --- /dev/null +++ b/docs/decisions/2026-10-12-total-session-recovery.md @@ -0,0 +1,51 @@ +# ADR-2026-10-12-total-session-recovery: Preserve outcomes while converging admission + +**Status:** accepted +**Date:** 2026-10-12 +**Area:** protocol + +## Context + +Graceful shutdown can release a background ACP call before its stop channel is +closed. A typed RPC error then becomes a journal failure while SQL records +uncertainty. Native restoration succeeds, but label equality strands admission. +Legacy recovery fences can also lack their stream binding or carry a restored +notice for a different earlier incident. Independent incident exceptions do not +establish a recovery contract. + +## Decision + +Separate historical outcome from admission. Keep journal labels, immutable +payloads, SQL uncertainty, terminal references and receipt cursors. A loaded +native conversation and authenticated complete inventory permit retirement of +matching uncertain history, including an unproven historical terminal label, +without asserting completion or replaying the payload. Actual terminal evidence, +known SQL failures and independently active work retain their authority. + +Use one lifecycle submission assessment for native retirement, repeated adoption +and automatic already-loaded recovery. Commit missing fence associations only +with authenticated current stream and exact inventory proof, under existing +SQL admission locks. Publish original-child stop intent before adapter teardown; +bounded settlement survives cancellation. Keep existing exact dead-owner proof, +restore checkpoints, gap archives and independent admission rules. + +## Consequences + +Recovery is restartable across journal retirement and SQL fence resolution. +Legacy labels remain readable and are not interpreted as terminal proof alone. +Incomplete or conflicting evidence yields a precise refusal; no storage surgery +or fabricated receipt is a supported recovery method. New state additions must +extend the finite disposition and persistence interruption tests. + +## Alternatives considered + +- Normalize every failure to unknown: erases genuine deterministic failure. +- Clear every recovery fence: overrides independent incidents and live owners. +- Replay the old instruction: duplicates external effects. +- Make the stores atomic: unavailable across backend and agentctl failure domains. + +## Related contracts + +- [Requirements](../specs/platform/requirements/durable-restart-reconciliation.md). +- [Convergence design](../specs/platform/system-design/session-recovery-convergence.md). +- [Honest delivery loss](2026-10-11-honest-delivery-loss-recovery.md). diff --git a/docs/plans/session-recovery-convergence/plan.md b/docs/plans/session-recovery-convergence/plan.md new file mode 100644 index 00000000000..10bfbc0076c --- /dev/null +++ b/docs/plans/session-recovery-convergence/plan.md @@ -0,0 +1,55 @@ +--- +created: 2026-10-12 +status: in_progress +requirements: + - REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-001 + - REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-002 +system_design: + - ../../specs/platform/system-design/session-recovery-convergence.md +legacy_specs: [] +--- + +# Implementation plan: Session recovery convergence + +## Outcome and ownership + +This original task/session solely owns permanent recovery through merge. No +parallel tasks, delegation, new sessions, local E2E or production mutation. +PR #4425 merged as 179fc69cc6838878c6db96f8743a42f9bb0041fc; its receipt/gap +correction remains the base. This follow-up closes producer shutdown ordering +and completes evidence-based native retirement and admission convergence. + +## Confirmed reproduction + +TestSubmissionGracefulStopPreservesUnknownRequestOutcome starts a real child, +admits an immutable prompt, invokes real Stop while RPC is active, and releases +a typed RequestError from adapter Close. Before the fix it expected unknown but +observed failed, without retained terminal. Existing process-manager code closed +stopCh after Close; dispatch used Background and classified every RequestError +as known. Private incident recovery also required filling an empty bound stream. +No private payloads or live native conversations belong in fixtures. + +## Sequence + +1. [Producer settlement](task-01-shutdown-settlement.md). +2. [Retirement and fence convergence](task-02-native-convergence.md). +3. [State/boundary contract and delivery](task-03-recovery-contract.md). + +Each work order is executed here sequentially. Native restoration and retirement +are never prompt authority. Historical labels, hashes, cursors and independent +fences remain evidence. Unknown ownership and true missing dependencies may +remain blocked with a specific supported action; they are not fabricated away. + +## Validation and delivery + +Focused Go tests use GOMEMLIMIT=768MiB GOMAXPROCS=2, -p 1, -race, -trimpath, +-tags fts5; only one heavy local operation runs at once. Required PostgreSQL and +all desktop/phone browser suites run in CI. Push and open a follow-up PR promptly, +address every actionable CI/review finding, and merge the actual passing head. +No completion claim follows merely from recovering the four observed records. + +## Results + +In progress. Real shutdown regression was red; initial focused producer race +suite is green. Comprehensive assessment, binding convergence and delivery +remain incomplete. No local E2E has run. diff --git a/docs/plans/session-recovery-convergence/task-01-shutdown-settlement.md b/docs/plans/session-recovery-convergence/task-01-shutdown-settlement.md new file mode 100644 index 00000000000..74102bf1775 --- /dev/null +++ b/docs/plans/session-recovery-convergence/task-01-shutdown-settlement.md @@ -0,0 +1,35 @@ +--- +status: in_progress +--- + +# Task 01: Shutdown settlement + +## Outcome and scope + +Publish original-child interruption before adapter Close, persist bounded +settlement after cancellation, preserve genuine deterministic failures and +retained terminals. Own process/submission_delivery.go, manager.go and sibling +tests. No protocol replacement or user prompt replay. + +## Contract + +REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-002, criteria .1 and .2. +[Design](../../specs/platform/system-design/session-recovery-convergence.md). + +## Acceptance + +- Real Stop while a background prompt is active records uncertainty without terminal invention. +- Genuine provider failure and retained complete/failed/cancelled evidence survive teardown. +- Cancellation and child replacement cannot leave replayable or cross-child settlement. + +## Check + +From apps/backend: +`GOMEMLIMIT=768MiB GOMAXPROCS=2 go test -p 1 -race -trimpath -tags fts5 ./internal/agentctl/server/process -run 'TestSubmission|TestStop' -count=1` + +## Results + +Real graceful-stop regression confirmed red before production correction. +Focused TestSubmission race suite passed after initial correction; controls pending. + +[Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/plans/session-recovery-convergence/task-02-native-convergence.md b/docs/plans/session-recovery-convergence/task-02-native-convergence.md new file mode 100644 index 00000000000..e5761fe65ed --- /dev/null +++ b/docs/plans/session-recovery-convergence/task-02-native-convergence.md @@ -0,0 +1,37 @@ +--- +status: in_progress +--- + +# Task 02: Native convergence + +## Outcome and scope + +Shared evidence assessment, guarded legacy retirement, authenticated missing +stream binding and atomic notice/fence resolution. Own lifecycle/native_resume*, +journal retirement, process native retirement, repository/sqlite/native_resume* +and affected orchestrator seams. Depends on task 01. Preserve old terminal +labels, unknown SQL, native conversation and all event cursors. + +## Contract + +REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-002, criteria .2 through .4. +[Design](../../specs/platform/system-design/session-recovery-convergence.md). + +## Acceptance + +- All active SQL records match authenticated peer evidence before any retirement. +- Legacy failed/no-terminal versus SQL unknown retires without normalization or replay. +- Empty bound stream with absent/stale resolved metadata converges under authenticated exact proof; rollback and conflicts preserve the fence. + +## Check + +From apps/backend, with GOMEMLIMIT=768MiB GOMAXPROCS=2, run sequentially: +`go test -p 1 -race -trimpath -tags fts5 ./internal/agentctl/journal ./internal/agentctl/server/process ./internal/agent/runtime/lifecycle -run 'NativeResume|Acknowledged|Retirement|Retire' -count=1` +`go test -p 1 -race -trimpath -tags fts5 ./internal/task/repository/sqlite ./internal/orchestrator -run 'NativeResume|GenericNative' -count=1` +PostgreSQL variants run in CI against real PostgreSQL. + +## Results + +Pending. + +[Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/plans/session-recovery-convergence/task-03-recovery-contract.md b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md new file mode 100644 index 00000000000..b6038502c76 --- /dev/null +++ b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md @@ -0,0 +1,37 @@ +--- +status: pending +--- + +# Task 03: Recovery contract and delivery + +## Outcome and scope + +Independent reachable-state and persistence-boundary regressions spanning +shutdown, repeated native recovery/adoption, SQL commit and new prompt admission. +Reconcile docs and public recovery guidance. Commit, push, open PR, fix CI/review +and merge in this session. Depends on tasks 01 and 02. + +## Contract + +REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-002, criteria .3 through .5. +[Design](../../specs/platform/system-design/session-recovery-convergence.md). + +## Acceptance + +- Finite supported state/identity/terminal matrix has an explicit safe disposition. +- Failures between durable writes converge on retry without old prompt replay, false receipt, independent owner takeover or resolved-history resurrection. +- Normal launch/Resume/direct/queue admission agree; actual final-head required CI and review gates pass. + +## Check + +Focused Go race checks from tasks 01/02 and affected production entrypoint tests. +`python3 scripts/list-docs.py validate` +`python3 scripts/lint-spec-files.py --all` +`git diff --check` +Desktop and phone E2E are CI-only; never run locally. + +## Results + +Pending. PR #4425 is merged; overall recovery remains in progress. + +[Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/public/operations.md b/docs/public/operations.md index 24c48013d1b..2edb20b0d08 100644 --- a/docs/public/operations.md +++ b/docs/public/operations.md @@ -31,7 +31,7 @@ See [Desktop app](desktop-app.md), [CLI](cli.md), [Run as a service](run-as-a-se Kandev automatically checks interrupted sessions at startup and after replacement of a crashed local agent runtime. When retained delivery records agree with the session's saved identity and the original process is proven terminated, Kandev restores the same native conversation without resending its interrupted prompt. Recovery checks continue while Kandev runs, so a temporarily unavailable runtime can recover later. These checks are enabled by default; the Agent Survival feature toggle separately controls whether local agents survive a graceful backend shutdown. -The interrupted prompt's historical outcome remains unknown unless a retained terminal event proves it. Missing native state, conflicting records, or an owner whose termination cannot be verified keep the session blocked. **Retry connection** checks retained evidence without sending another prompt. A successful native **Resume** clears the matching interrupted notice, including after a page reload, while preserving the conversation history. +The interrupted prompt's historical outcome remains unknown unless a retained terminal event proves it. Recovery also reconciles a shutdown failure label that lacks terminal evidence with the backend's unknown outcome. It preserves both historical records and repairs missing recovery associations only after verifying the saved conversation and retired work. Missing native state, conflicting records, or an owner whose termination cannot be verified keep the session blocked. **Retry connection** checks retained evidence without sending another prompt. A successful native **Resume** clears the matching interrupted notice, including after a page reload, while preserving the conversation history. If an older crash left journal acknowledgments ahead of saved output, Kandev can restore a verified conversation on a new delivery generation while keeping diff --git a/docs/specs/platform/requirements/durable-restart-reconciliation.md b/docs/specs/platform/requirements/durable-restart-reconciliation.md index 566f7f990ab..e33607617db 100644 --- a/docs/specs/platform/requirements/durable-restart-reconciliation.md +++ b/docs/specs/platform/requirements/durable-restart-reconciliation.md @@ -2,7 +2,7 @@ status: active system: platform created: 2026-10-11 -updated: 2026-10-11 +updated: 2026-10-12 owners: - kandev --- @@ -34,6 +34,22 @@ inconsistent delivery records do not strand their saved conversations. - **AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-001.4:** Successful automatic restoration or explicit native Resume shall persist and publish a newer restored recovery snapshot with resolution of the matching delivery block. Desktop and phone shall show the ordinary available composer after recovery and reload, without an obsolete interruption banner. A distinct later instruction shall use normal admission; the old interrupted submission remains non-replayable. - **AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-001.5:** Repeated reconciliation and crashes between journal and SQL writes shall converge from retained progress without duplicate restore, fabricated completion, or clearing independent recovery causes. Restored sessions shall not repeatedly consult retired runtime handles or rediscover resolved historical recovery as new uncertainty. A newer notice for an identical source may refresh an unallocated checkpoint; allocating a candidate pins its source. Later partial failure notices shall preserve the same submission's known stream binding; existing partial fences require authenticated owner and payload evidence before repair. +### REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-002: Convergent interrupted-session recovery + +**Intent:** All recovery entry paths apply the same honest disposition after hard +crash or graceful shutdown, without repeated incident-specific database repair. + +**User story:** As a user, I want my saved conversation to become usable when its +owner is safely recoverable, even when recovery bookkeeping is inconsistent. + +#### Acceptance criteria + +- **AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.1:** Graceful shutdown during an active durable prompt shall preserve uncertainty when teardown releases the call without terminal proof. Deterministic provider rejection and retained terminal effects remain authoritative; request cancellation cannot prevent durable settlement. +- **AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.2:** Native recovery shall preserve differing historical journal labels and SQL uncertainty while acknowledging exact unproven history. It shall neither invent completion/cancellation nor redispatch the old instruction, including queue-origin instructions. +- **AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.3:** Restart, runtime replacement, ordinary resume launch, explicit Resume, and new prompt/queue admission shall consume one consistent recovery disposition. Complete authenticated inventory and source-owner proof are required; live or unknown independent ownership remains protected. +- **AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.4:** Missing stream associations, absent notices and stale already-restored earlier notices shall converge through guarded exact proof and atomic resolution. Repeated restart and interruption during recovery shall reuse retained progress and admit a distinct new instruction when all independent gates pass. +- **AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.5:** Every supported reachable cross-store state shall have a tested safe disposition at dispatch, terminal, acknowledgement, teardown, restoration and publication boundaries. Incomplete/conflicting evidence shall surface the actual cause and supported recovery action without fabricating certainty. + ## Technical design and delivery records - [System design](../system-design/durable-restart-reconciliation.md). @@ -41,3 +57,6 @@ inconsistent delivery records do not strand their saved conversations. - [Proposed pruned-history recovery](pruned-delivery-recovery.md) defines an additional restoration contract when acknowledged events are absent from both stores. Current reconciliation does not establish that missing receipt. + +- [Convergence design](../system-design/session-recovery-convergence.md). +- [Convergence implementation](../../../plans/session-recovery-convergence/plan.md). diff --git a/docs/specs/platform/system-design/session-recovery-convergence.md b/docs/specs/platform/system-design/session-recovery-convergence.md new file mode 100644 index 00000000000..942779ba0ec --- /dev/null +++ b/docs/specs/platform/system-design/session-recovery-convergence.md @@ -0,0 +1,94 @@ +--- +status: current +system: platform +requirements: + - REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-001 + - REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-002 +--- + +# Session recovery convergence + +## Boundaries + +Platform owns the contract. Lifecycle authenticates the runtime and assesses the +complete SQL/journal inventory; orchestrator retains admission authority; SQL and +journal retain their own transactions. This extends [restart reconciliation](durable-restart-reconciliation.md) +and [pruned recovery](pruned-delivery-recovery.md), not their dead-owner or gap +guards. The [decision](../../../decisions/2026-10-12-total-session-recovery.md) +separates historical outcome from current admission. + +## Disposition precedence + +| Evidence | Disposition | +| --- | --- | +| Archived/cancelled, Office/workflow/automation or recipient restriction | Preserve independent policy | +| Exact original owner alive | Reattach; no native replacement or old dispatch | +| Owner unknown, foreign identity/hash, future generation or incomplete inventory | Refuse with missing/conflicting evidence | +| Retained terminal with real SQL receipt | Ordered projection and terminal settlement | +| SQL dispatching, exact abandoned journal unknown | Guarded SQL unknown transition, then restore-only | +| SQL unknown, exact journal unknown | Restore-only and retire exact history | +| SQL unknown, legacy terminal label without any retained terminal reference | Preserve both labels; retire as historical uncertainty after native load and idle owner proof | +| SQL known terminal, prepared/accepted or independent peer live work | Preserve known result or refuse takeover | +| SQL inbox covers replay expiry | Project actual inbox; no fabricated receipt | +| Actually missing history with proven eligible dead source | Existing pinned successor and immutable gap archive | +| Unsupported peer/storage/native evidence | Specific dependency refusal, supported retry | + +Unknown enum values refuse. Evaluate every unresolved record before mutation. +A failed label alone does not authorize SQL dispatch-to-unknown conversion. +No terminal label is downgraded. Retired uncertain history is excluded only +with exact authenticated journal evidence; repeat adoption uses the same rule. + +## Producer ordering + +Capture the admitted child's stop channel for dispatch. Publish interruption +before closing its writer or adapter, so a teardown-generated RequestError is +uncertain. Do not consult mutable replacement status or acquire the teardown +mutex inside settlement. A deterministic provider rejection remains a failure; +a retained terminal wins over a late transport error. Settle with a bounded +cancellation-independent context. Journal close or failed writes leave retained +progress for constructor reconciliation, never authorize redispatch. + +## Native acknowledgement and SQL convergence + +Require native initialized, exact native ID and current session/incarnation/ +generation/stream, authenticated full inventory, and no independent live work. +Validate receipt/projection coverage before retiring history. Retire only exact +eligible SQL unknown records and journal unknown records; retain payload/hash, +state and terminal absence. Re-read owner and inventory after retirement. + +Carry authenticated stream and exact submission identities into missing-binding +resolution. Pin session/native generation, block identity and current SQL rows +under existing turn/delivery locks; no derivation from a guessed canonical name. +Absent metadata and an earlier already-resolved notice are presentation states, +not new outcome authority. Restore the current notice and resolve only matching +incident fences atomically. A changed source, independent fence, pending restore +or unprojected receipt refuses resolution. A transaction rollback changes none +of the binding, notice or fence. A retry reuses retired journal proof. + +## Entry points + +Startup/runtime replacement and explicit Resume reuse existing restore ownership +and source-death checks. Normal launch/open and direct/queued prompt admission +consume the same committed disposition; read-only inspection cannot grant +admission. An already-loaded idle native successor is reused. Recovery supplies +no prompt and does not revive a dispatched queue entry. A distinct new prompt +requires a new immutable submission and normal workflow/queue authority. + +## Verification + +Generate reachable states using admission, dispatch, terminal, shutdown, +retirement and restore transitions. Exercise all supported SQL/journal state +pairs with complete and absent terminal references and exact/foreign identities. +Independently assert no old dispatch, no invented completion/receipt, no live +owner takeover, and eventual new-prompt admission when proofs are complete. + +Interrupt at dispatch-to-call, terminal-to-response, journal retirement-to-SQL +resolution, SQL resolution-to-publication and ACK-to-prune boundaries. Include +failed writes and cancellation, then repeat restart and recovery. Use real bbolt, +SQLite, PostgreSQL conformance and a real isolated process-manager Stop. Existing +hard-crash and gap tests remain relevant. Desktop and phone E2E run only in CI; +no host shutdown, production mutation or local browser testing is authorized. + +## Delivery + +[Sequential work package](../../../plans/session-recovery-convergence/plan.md). From 24748932855b7e13858cef11d7124f4d1102f737 Mon Sep 17 00:00:00 2001 From: Kandev Agent Date: Mon, 12 Oct 2026 03:10:07 +0100 Subject: [PATCH 2/7] fix: converge native recovery and preserve shutdown events --- .../already_resumed_recovery_linux_test.go | 30 +++- .../lifecycle/delivery_record_evidence.go | 7 + .../lifecycle/loaded_native_reconciliation.go | 95 ++++++++++++ .../lifecycle/native_resume_adoption.go | 5 +- .../runtime/lifecycle/native_resume_proof.go | 22 ++- .../native_resume_proof_status_test.go | 34 +++++ .../agentctl/journal/native_resume_test.go | 14 ++ .../agentctl/journal/submission_outcome.go | 5 +- .../agentctl/server/process/manager.go | 57 +++++-- .../server/process/submission_delivery.go | 21 +-- .../process/submission_shutdown_test.go | 56 +++++++ .../backendapp/adapters_native_resume.go | 4 + .../already_resumed_restart_recovery.go | 5 +- .../orchestrator/legacy_native_restart.go | 142 ++++++++++++++++++ .../legacy_native_restart_test.go | 110 ++++++++++++++ .../orchestrator/native_resume_proof_test.go | 39 ++++- .../restart_recovery_presentation.go | 5 + .../internal/orchestrator/session_launch.go | 3 + .../session_native_delivery_recovery.go | 29 ++++ .../silent_restart_recovery_inputs.go | 7 + .../sqlite/native_resume_binding.go | 115 +++++++++----- .../sqlite/native_resume_binding_test.go | 17 ++- .../session-recovery-convergence/plan.md | 8 +- .../task-01-shutdown-settlement.md | 16 +- .../task-02-native-convergence.md | 19 ++- .../task-03-recovery-contract.md | 22 ++- .../session-recovery-convergence.md | 12 +- 27 files changed, 805 insertions(+), 94 deletions(-) create mode 100644 apps/backend/internal/agent/runtime/lifecycle/loaded_native_reconciliation.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/native_resume_proof_status_test.go create mode 100644 apps/backend/internal/orchestrator/legacy_native_restart.go create mode 100644 apps/backend/internal/orchestrator/legacy_native_restart_test.go diff --git a/apps/backend/internal/agent/runtime/lifecycle/already_resumed_recovery_linux_test.go b/apps/backend/internal/agent/runtime/lifecycle/already_resumed_recovery_linux_test.go index bc352b41b1c..7e7eafb9690 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/already_resumed_recovery_linux_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/already_resumed_recovery_linux_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "os/exec" "path/filepath" + "strings" "syscall" "testing" ) @@ -20,7 +21,7 @@ type loadedNativeDeliveryReconciler interface { } func TestRestartRecoveryRetiresOnlyProvenDeadLoadedConversation(t *testing.T) { - for _, condition := range []string{"dead owner", "live owner", "missing proof", "different native", "active replacement"} { + for _, condition := range []string{"dead owner", "live owner", "missing proof", "different native", "active replacement", "legacy dead owner", "legacy live owner", "legacy missing proof"} { t.Run(condition, func(t *testing.T) { ctx := context.Background() child := exec.Command("/bin/sleep", "60") @@ -29,7 +30,7 @@ func TestRestartRecoveryRetiresOnlyProvenDeadLoadedConversation(t *testing.T) { t.Cleanup(func() { _ = child.Process.Kill(); _ = child.Wait() }) birth, err := processidentity.Capture(child.Process.Pid) require.NoError(t, err) - if condition != "live owner" { + if !strings.HasSuffix(condition, "live owner") { require.NoError(t, child.Process.Kill()) require.Error(t, child.Wait()) } @@ -38,6 +39,9 @@ func TestRestartRecoveryRetiresOnlyProvenDeadLoadedConversation(t *testing.T) { t.Cleanup(func() { _ = store.Close() }) peer := journal.Submission{ID: "interrupted", SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: 2, StreamID: "stream", Hash: "hash", State: journal.SubmissionInterruptedUnknown} + if strings.HasPrefix(condition, "legacy") { + peer.State = journal.SubmissionFailed + } _, err = store.PutSubmission(ctx, peer) require.NoError(t, err) manager := newTestManager(t) @@ -58,13 +62,25 @@ func TestRestartRecoveryRetiresOnlyProvenDeadLoadedConversation(t *testing.T) { identity := AgentDeliveryRecoveryIdentity{TaskID: "task", SessionID: "session", ExecutionID: "old", IncarnationID: "incarnation", HarnessGeneration: 2, StreamID: "stream", SubmissionID: peer.ID, PromptGeneration: 3, OriginalRuntime: birth} - if condition == "missing proof" { + if strings.HasSuffix(condition, "missing proof") { identity.OriginalRuntime = processidentity.Identity{} } reconciler, supported := any(manager).(loadedNativeDeliveryReconciler) require.True(t, supported, "restart recovery must finish an already loaded native successor") - restored, recoverErr := reconciler.ReconcileAlreadyResumedDelivery(ctx, identity, "native") - if condition == "dead owner" { + restored, recoverErr := false, error(nil) + if strings.HasPrefix(condition, "legacy") { + manager.runningWriter = &captureExecutorRunningWriter{prior: &models.ExecutorRunning{TaskID: "task", SessionID: "session", AgentExecutionID: "replacement", Metadata: map[string]any{deliveryPromptTurnBindingMetadataKey: deliveryPromptTurnBinding{ + TaskID: "task", SessionID: "session", ExecutionID: "old", IncarnationID: "incarnation", HarnessGeneration: 2, StreamID: "stream", SubmissionID: peer.ID, TurnID: "turn", PromptGeneration: 3, OriginalRuntime: identity.OriginalRuntime}}}} + automatic, ok := any(manager).(interface { + ReconcileLoadedNativeDelivery(context.Context, string) (*models.NativeResumeDeliveryEvidence, error) + }) + require.True(t, ok, "loaded replacement must reconcile without recovery presentation metadata") + proof, err := automatic.ReconcileLoadedNativeDelivery(ctx, "session") + restored, recoverErr = proof != nil, err + } else { + restored, recoverErr = reconciler.ReconcileAlreadyResumedDelivery(ctx, identity, "native") + } + if strings.HasSuffix(condition, "dead owner") { require.NoError(t, recoverErr) require.True(t, restored) } else { @@ -73,8 +89,8 @@ func TestRestartRecoveryRetiresOnlyProvenDeadLoadedConversation(t *testing.T) { } saved, err := store.GetSubmission(ctx, peer.ID) require.NoError(t, err) - require.Equal(t, condition == "dead owner", saved.Retired) - require.Equal(t, journal.SubmissionInterruptedUnknown, saved.State) + require.Equal(t, strings.HasSuffix(condition, "dead owner"), saved.Retired) + require.Equal(t, peer.State, saved.State) current, exists := manager.GetExecutionBySessionID("session") require.True(t, exists) require.Same(t, replacement, current) diff --git a/apps/backend/internal/agent/runtime/lifecycle/delivery_record_evidence.go b/apps/backend/internal/agent/runtime/lifecycle/delivery_record_evidence.go index 2502275de1d..1116c4610ce 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/delivery_record_evidence.go +++ b/apps/backend/internal/agent/runtime/lifecycle/delivery_record_evidence.go @@ -93,6 +93,13 @@ func (m *Manager) InspectDeliveryRecordEvidence( return result, errors.Join(ErrDeliveryRecordEvidenceTruncated, ErrDeliveryRecoveryBlocked) } result.ControlIdentity = m.deliveryRecordControlIdentity(ctx, request, result.Descriptor) + if result.ControlIdentity == nil && result.Descriptor.SubmissionCount == 0 && result.RetiredSubmission != nil { + selected := result.RetiredSubmission + if selected.ID == request.RetiredSubmissionID && selected.SessionID == request.SessionID && selected.IncarnationID == request.IncarnationID && selected.HarnessGeneration == request.HarnessGeneration { + result.ControlIdentity = m.persistedDeliveryRecordControlIdentity(ctx, request, journal.SubmissionSummary{ + ID: selected.ID, SessionID: selected.SessionID, IncarnationID: selected.IncarnationID, HarnessGeneration: selected.HarnessGeneration, StreamID: selected.StreamID}) + } + } if identity := result.ControlIdentity; identity != nil { result.ProcessTerminated = historicalProcessTermination(DeliveryRecordEvidenceRequest{ ExecutionID: identity.ExecutionID, OriginalRuntime: &identity.OriginalRuntime, diff --git a/apps/backend/internal/agent/runtime/lifecycle/loaded_native_reconciliation.go b/apps/backend/internal/agent/runtime/lifecycle/loaded_native_reconciliation.go new file mode 100644 index 00000000000..85182fb4302 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/loaded_native_reconciliation.go @@ -0,0 +1,95 @@ +package lifecycle + +import ( + "context" + "fmt" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/journal" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// ReconcileLoadedNativeDelivery completes retirement for an idle native successor. +// The persisted prompt binding proves the original owner, independently of the +// replacement's current process and of missing UI recovery metadata. +func (m *Manager) ReconcileLoadedNativeDelivery(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { + execution, exists := m.GetExecutionBySessionID(sessionID) + if !exists || execution == nil { + return nil, ErrDurableAdoptionEvidenceUnavailable + } + if !execution.promptMu.TryLock() { + return nil, ErrDeliveryRecoveryBlocked + } + defer execution.promptMu.Unlock() + if execution.Status != v1.AgentStatusReady || execution.dispatchedPromptPending.Load() { + return nil, ErrDeliveryRecoveryBlocked + } + if proof, err := m.GetNativeResumeDeliveryEvidence(ctx, sessionID); err == nil && proof != nil { + return proof, nil + } + historical, err := m.readLoadedNativeHistoricalOwner(ctx, execution) + if err != nil { + return nil, err + } + if historical == nil || historical.ExecutionID == execution.ID || !loadedRecoverySuccessorMatches(execution, *historical) { + return nil, ErrDeliveryRecoveryBlocked + } + if err := proveHistoricalDeliveryOwnerDead(*historical); err != nil { + return nil, err + } + if err := m.AcknowledgeNativeResumeDelivery(ctx, sessionID); err != nil { + return nil, err + } + return m.GetNativeResumeDeliveryEvidence(ctx, sessionID) +} + +func (m *Manager) readLoadedNativeHistoricalOwner(ctx context.Context, execution *AgentExecution) (*AgentDeliveryRecoveryIdentity, error) { + client, release := execution.AcquireAgentCtlClient() + defer release() + if client == nil { + return nil, ErrDurableAdoptionEvidenceUnavailable + } + status, err := m.nativeResumeDeliveryStatus(ctx, execution, client) + if err != nil || status == nil { + return nil, err + } + if m.streamManager == nil || m.streamManager.deliveryRepository() == nil { + return nil, ErrDurableAdoptionEvidenceUnavailable + } + peer, err := client.ListDeliverySubmissions(ctx, execution.SessionID) + if err != nil { + return nil, err + } + rows, err := activeBackendSubmissions(ctx, m.streamManager.deliveryRepository(), execution.SessionID) + if err != nil { + return nil, err + } + return m.loadedNativeHistoricalOwner(ctx, execution, status, peer, rows) +} +func (m *Manager) loadedNativeHistoricalOwner(ctx context.Context, execution *AgentExecution, status *agentctl.DeliveryStatus, peer []journal.Submission, rows []*models.AgentDeliverySubmission) (*AgentDeliveryRecoveryIdentity, error) { + var historical *AgentDeliveryRecoveryIdentity + for _, row := range rows { + for _, submission := range peer { + if submission.ID != row.ID || submission.Retired { + continue + } + if historical != nil { + return nil, fmt.Errorf("%w: independent unretired history", ErrDeliveryRecoveryBlocked) + } + candidate := *row + if candidate.State == models.DeliverySubmissionDispatching && submission.State == journal.SubmissionInterruptedUnknown { + candidate.State = models.DeliverySubmissionInterruptedUnknown + } + if !peerMatchesUnknownHistory(execution, status, submission, &candidate) { + return nil, ErrDeliveryOwnerMismatch + } + historical = m.persistedDeliveryRecordControlIdentity(ctx, DeliveryRecordEvidenceRequest{TaskID: execution.TaskID, SessionID: execution.SessionID}, journal.SubmissionSummary{ + ID: submission.ID, SessionID: submission.SessionID, IncarnationID: submission.IncarnationID, HarnessGeneration: submission.HarnessGeneration, StreamID: submission.StreamID}) + if historical == nil { + return nil, ErrDeliveryRecoveryBlocked + } + } + } + return historical, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_adoption.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_adoption.go index aae25dff577..eca5969433b 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/native_resume_adoption.go +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_adoption.go @@ -64,9 +64,10 @@ func peerAcknowledgesUnknownHistory(execution *AgentExecution, status *agentctl. func peerMatchesUnknownHistory(execution *AgentExecution, status *agentctl.DeliveryStatus, peer journal.Submission, submission *models.AgentDeliverySubmission) bool { return submission.State == models.DeliverySubmissionInterruptedUnknown && - submission.TerminalSequence == 0 && peer.ID == submission.ID && journal.HasUnprovenOutcome(peer) && + (peer.State == journal.SubmissionInterruptedUnknown || submission.TerminalSequence == 0) && peer.ID == submission.ID && journal.HasUnprovenOutcome(peer) && peer.SessionID == execution.SessionID && peer.SessionID == submission.SessionID && peer.IncarnationID == status.IncarnationID && peer.IncarnationID == submission.IncarnationID && peer.HarnessGeneration > 0 && peer.HarnessGeneration <= status.HarnessGeneration && int64(peer.HarnessGeneration) == submission.HarnessGeneration && - peer.Hash != "" && peer.Hash == submission.PayloadHash + peer.Hash != "" && peer.Hash == submission.PayloadHash && + (peer.HarnessGeneration != status.HarnessGeneration || peer.StreamID == status.StreamID) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof.go index 3977c6fe106..0ebca39b3f8 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof.go +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof.go @@ -14,7 +14,7 @@ import ( // admission fence. The snapshot cannot authorize dispatch or alter old outcomes. func (m *Manager) GetNativeResumeDeliveryEvidence(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { execution, exists := m.GetExecutionBySessionID(sessionID) - if !exists || execution == nil { + if !exists { return nil, ErrDurableAdoptionEvidenceUnavailable } client, release := execution.AcquireAgentCtlClient() @@ -56,7 +56,7 @@ func (m *Manager) GetNativeResumeDeliveryEvidence(ctx context.Context, sessionID if err != nil { return nil, err } - if after == nil || !sameDeliveryRecordDescriptor(*status, *after) { + if !sameNativeResumeProofStatus(status, after) { return nil, ErrDeliveryRecordEvidenceStale } return proof, nil @@ -74,8 +74,12 @@ func (m *Manager) nativeResumeProofRows(ctx context.Context, execution *AgentExe if err := validateNativeResumeSQLRows(execution, status, peer, rows, true); err != nil { return nil, nil, err } - if ids, err := nativeResumeRetirementIDs(execution, status, peer, rows); err != nil || len(ids) != 0 { - return nil, nil, fmt.Errorf("%w: native resume inventory has unsettled work: %v", ErrDurableAdoptionBlocked, err) + ids, err := nativeResumeRetirementIDs(execution, status, peer, rows) + if err != nil { + return nil, nil, err + } + if len(ids) != 0 { + return nil, nil, fmt.Errorf("%w: native resume inventory has %d unsettled submissions", ErrDurableAdoptionBlocked, len(ids)) } streams := make(map[string]string, len(peer)) for _, submission := range peer { @@ -103,3 +107,13 @@ func nativeResumeProofCursor(ctx context.Context, repository AgentDeliveryReposi } return cursor, nil } + +func sameNativeResumeProofStatus(before, after *agentctl.DeliveryStatus) bool { + if before == nil || after == nil || !sameDeliveryRecordDescriptor(*before, *after) || before.StreamID != after.StreamID || before.Unresolved != after.Unresolved { + return false + } + if before.Stream == nil || after.Stream == nil { + return before.Stream == nil && after.Stream == nil + } + return *before.Stream == *after.Stream +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof_status_test.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof_status_test.go new file mode 100644 index 00000000000..d7836afe25c --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_proof_status_test.go @@ -0,0 +1,34 @@ +package lifecycle + +import ( + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/journal" + "github.com/stretchr/testify/require" + "testing" +) + +func TestNativeResumeProofRejectsConcurrentStreamProgress(t *testing.T) { + before := &agentctl.DeliveryStatus{StreamID: "stream", Stream: &journal.Stream{StreamID: "stream", HighWater: 5, Acknowledged: 5, FirstRetained: 6}} + for _, condition := range []string{"unchanged", "late terminal", "acknowledged", "pruned", "unresolved", "owner", "missing stream"} { + t.Run(condition, func(t *testing.T) { + after := *before + stream := *before.Stream + after.Stream = &stream + switch condition { + case "late terminal": + after.Stream.HighWater++ + case "acknowledged": + after.Stream.Acknowledged++ + case "pruned": + after.Stream.FirstRetained++ + case "unresolved": + after.Unresolved = true + case "owner": + after.StreamID = "foreign" + case "missing stream": + after.Stream = nil + } + require.Equal(t, condition == "unchanged", sameNativeResumeProofStatus(before, &after)) + }) + } +} diff --git a/apps/backend/internal/agentctl/journal/native_resume_test.go b/apps/backend/internal/agentctl/journal/native_resume_test.go index 69695aa553a..94063e4705f 100644 --- a/apps/backend/internal/agentctl/journal/native_resume_test.go +++ b/apps/backend/internal/agentctl/journal/native_resume_test.go @@ -97,3 +97,17 @@ func TestNativeResumeRetiresUnprovenTerminalLabelsWithoutChangingThem(t *testing }) } } + +func TestNativeResumeRetiresUnknownWithRetainedTerminal(t *testing.T) { + j, event := terminalSubmissionFixture(t, SubmissionInterruptedUnknown, false) + _, appendErr := j.Append(context.Background(), event) + require.NoError(t, appendErr) + before, err := j.GetSubmission(context.Background(), event.SubmissionID) + require.NoError(t, err) + require.True(t, before.TerminalEventRetained) + after, err := j.RetireSubmission(context.Background(), event.SubmissionID, event.HarnessGeneration) + require.NoError(t, err) + require.True(t, after.Retired) + require.Equal(t, before.State, after.State) + require.Equal(t, before.TerminalSequence, after.TerminalSequence) +} diff --git a/apps/backend/internal/agentctl/journal/submission_outcome.go b/apps/backend/internal/agentctl/journal/submission_outcome.go index 4f18d0b1e36..a00427f8600 100644 --- a/apps/backend/internal/agentctl/journal/submission_outcome.go +++ b/apps/backend/internal/agentctl/journal/submission_outcome.go @@ -3,11 +3,14 @@ package journal // HasUnprovenOutcome identifies history that native restoration can acknowledge // without changing its recorded outcome. It is not permission to retire live work. func HasUnprovenOutcome(submission Submission) bool { + if submission.State == SubmissionInterruptedUnknown { + return true + } if submission.TerminalEventRetained || submission.TerminalSequence != 0 { return false } switch submission.State { - case SubmissionInterruptedUnknown, SubmissionFailed, SubmissionCompleted, SubmissionCancelled: + case SubmissionFailed, SubmissionCompleted, SubmissionCancelled: return true default: return false diff --git a/apps/backend/internal/agentctl/server/process/manager.go b/apps/backend/internal/agentctl/server/process/manager.go index 42b771a29ae..b3c7500af68 100644 --- a/apps/backend/internal/agentctl/server/process/manager.go +++ b/apps/backend/internal/agentctl/server/process/manager.go @@ -297,6 +297,7 @@ type Manager struct { // startOneShot store the freshly created channel here in the same place // they assign stopCh itself. stopChSnapshot atomic.Value // chan struct{} + shutdownIntent atomic.Pointer[atomic.Bool] doneCh chan struct{} // attachedCount is the live count of backend event-stream connections // (see attachment.go). Zero value correctly starts an instance detached. @@ -712,8 +713,8 @@ func (m *Manager) DispatchDeliverySubmission( m.deliveryActiveID = "" m.deliveryActiveMu.Unlock() }() - stopped, _ := m.stopChSnapshot.Load().(chan struct{}) - return (&SubmissionDelivery{Journal: deliveryJournal, Stopped: stopped}).Dispatch(ctx, id, call) + stopping := m.shutdownIntent.Load() + return (&SubmissionDelivery{Journal: deliveryJournal, Stopping: stopping}).Dispatch(ctx, id, call) } func (m *Manager) activeDeliverySubmissionID() string { @@ -1833,6 +1834,7 @@ func (m *Manager) startManagedProcess() (uint64, error) { m.stopChSnapshot.Store(m.stopCh) m.doneCh = make(chan struct{}) m.stopChClosed.Store(false) + m.shutdownIntent.Store(&atomic.Bool{}) // Connect adapter to the process stdin/stdout pipes if err := m.adapter.Connect(m.stdin, m.stdout); err != nil { @@ -1889,6 +1891,7 @@ func (m *Manager) startOneShot() error { m.stopChSnapshot.Store(m.stopCh) m.doneCh = make(chan struct{}) m.stopChClosed.Store(false) + m.shutdownIntent.Store(&atomic.Bool{}) // Forward adapter updates to our channel m.wg.Add(1) @@ -2474,6 +2477,7 @@ func (m *Manager) createAdapter() error { // forwardUpdates forwards updates from the adapter to the manager's channel func (m *Manager) forwardUpdates(agentAdapter adapter.AgentAdapter, stopCh <-chan struct{}) { defer m.wg.Done() + defer m.closeDeliveryEventWriter() updatesCh := agentAdapter.Updates() for { @@ -2505,11 +2509,42 @@ func (m *Manager) forwardUpdates(agentAdapter adapter.AgentAdapter, stopCh <-cha return } case <-stopCh: + m.drainDeliveryUpdates(updatesCh) return } } } +func (m *Manager) drainDeliveryUpdates(updates <-chan adapter.AgentEvent) { + for { + select { + case update, ok := <-updates: + if !ok { + return + } + update.DeliverySubmissionID = m.deliverySubmissionIDForEvent(update) + persisted, err := m.persistDeliveryEvent(update) + if err != nil { + m.recordDeliveryStorageFailure(err) + return + } + m.recordTerminalOutcome(&persisted) + m.signalDeliveryWakeup() + default: + return + } + } +} + +func (m *Manager) closeDeliveryEventWriter() { + m.deliveryJournalMu.RLock() + writer := m.deliveryWriter + m.deliveryJournalMu.RUnlock() + if writer != nil { + writer.close() + } +} + // persistDeliveryEvent establishes the journal commit-before-publish barrier. // The normalized event is kept as an opaque payload here; backend projection // remains the owner of canonical task messages and workflow effects. @@ -2909,17 +2944,8 @@ func (m *Manager) stopShellAndProcesses(ctx context.Context) error { // It reports whether this call closed stopCh, i.e. whether goroutines were just // released and still need draining. func (m *Manager) closeAdapterAndStdin() bool { - // Publish this child's interruption before closing the adapter can release a prompt RPC. - closedStopCh := false - if m.stopCh != nil && m.stopChClosed.CompareAndSwap(false, true) { - close(m.stopCh) - closedStopCh = true - } - m.deliveryJournalMu.RLock() - writer := m.deliveryWriter - m.deliveryJournalMu.RUnlock() - if writer != nil { - writer.close() + if stopping := m.shutdownIntent.Load(); stopping != nil { + stopping.Store(true) } m.logger.Debug("closing adapter") if m.adapter != nil { @@ -2929,6 +2955,11 @@ func (m *Manager) closeAdapterAndStdin() bool { } m.logger.Debug("adapter closed") + closedStopCh := false + if m.stopCh != nil && m.stopChClosed.CompareAndSwap(false, true) { + close(m.stopCh) + closedStopCh = true + } m.logger.Debug("stop channel closed", zap.Bool("closed_now", closedStopCh)) // Close stdin to signal EOF to agent diff --git a/apps/backend/internal/agentctl/server/process/submission_delivery.go b/apps/backend/internal/agentctl/server/process/submission_delivery.go index d84e5753f3b..04a3642ce96 100644 --- a/apps/backend/internal/agentctl/server/process/submission_delivery.go +++ b/apps/backend/internal/agentctl/server/process/submission_delivery.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync/atomic" "time" acp "github.com/coder/acp-go-sdk" @@ -21,6 +22,10 @@ func isKnownSubmissionFailure(err error) bool { if errors.As(err, &requestErr) { return true } + return isDeterministicProviderFailure(err) +} + +func isDeterministicProviderFailure(err error) bool { var providerFailure deterministicPromptFailure return errors.As(err, &providerFailure) && providerFailure.DeterministicPromptFailure() } @@ -29,9 +34,9 @@ func isKnownSubmissionFailure(err error) bool { // record. It is deliberately independent of the ACP prompt call so a caller // can reconcile durable state before deciding whether a harness call is safe. type SubmissionDelivery struct { - Journal *journal.Journal - Now func() time.Time - Stopped <-chan struct{} + Journal *journal.Journal + Now func() time.Time + Stopping *atomic.Bool } func (d *SubmissionDelivery) Admit(ctx context.Context, submission journal.Submission) (journal.Submission, error) { @@ -104,10 +109,8 @@ func (d *SubmissionDelivery) prepareDispatch(ctx context.Context, id string) (jo func (d *SubmissionDelivery) handleDispatchError(ctx context.Context, id string, dispatchErr error) (journal.Submission, error) { interrupted := ctx.Err() != nil - select { - case <-d.Stopped: + if d.Stopping != nil && d.Stopping.Load() { interrupted = true - default: } // Settling an external call survives cancellation of the request that made it. settleCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) @@ -119,12 +122,10 @@ func (d *SubmissionDelivery) handleDispatchError(ctx context.Context, id string, if submission.State == journal.SubmissionCompleted && submission.TerminalEventRetained { return submission, nil } - if submission.TerminalEventRetained { + if submission.TerminalEventRetained && (submission.State == journal.SubmissionFailed || submission.State == journal.SubmissionCancelled) { return submission, dispatchErr } - var providerFailure deterministicPromptFailure - knownProviderFailure := errors.As(dispatchErr, &providerFailure) && providerFailure.DeterministicPromptFailure() - if knownProviderFailure || (!interrupted && isKnownSubmissionFailure(dispatchErr)) { + if isDeterministicProviderFailure(dispatchErr) || (!interrupted && isKnownSubmissionFailure(dispatchErr)) { failed, transitionErr := d.Journal.TransitionSubmission(settleCtx, id, journal.SubmissionFailed, d.now()) if transitionErr != nil { return journal.Submission{}, errors.Join(ErrSubmissionUncertain, transitionErr) diff --git a/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go b/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go index 9348ebae9ef..561b192df29 100644 --- a/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go +++ b/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go @@ -122,3 +122,59 @@ func TestSubmissionCancellationSettlesJournalAndKeepsKnownFailures(t *testing.T) }) } } + +func TestSubmissionRetainedErrorSettlesKnownFailure(t *testing.T) { + ctx := context.Background() + store, err := journal.Open(journal.Config{Path: filepath.Join(t.TempDir(), "delivery.bbolt")}) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + delivery := &SubmissionDelivery{Journal: store} + _, err = delivery.Admit(ctx, journal.Submission{ID: "prompt", SessionID: "session", IncarnationID: "incarnation", + HarnessGeneration: 1, StreamID: "stream", Hash: "immutable", Payload: []byte("old instruction")}) + require.NoError(t, err) + stored, err := delivery.Dispatch(ctx, "prompt", func(ctx context.Context) error { + _, appendErr := store.Append(ctx, journal.Event{SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: 1, + StreamID: "stream", SubmissionID: "prompt", Type: "error", Terminal: true, Payload: []byte("provider rejected prompt")}) + require.NoError(t, appendErr) + return rejectedPromptError{} + }) + require.Error(t, err) + require.Equal(t, journal.SubmissionFailed, stored.State) + require.True(t, stored.TerminalEventRetained) + require.EqualValues(t, 1, stored.TerminalSequence) +} + +type terminalOnCloseAdapter struct { + adapter.AgentAdapter + updates chan adapter.AgentEvent +} + +func (a *terminalOnCloseAdapter) Updates() <-chan adapter.AgentEvent { return a.updates } +func (a *terminalOnCloseAdapter) Close() error { + a.updates <- adapter.AgentEvent{Type: adapter.EventTypeComplete, SessionID: "session", DeliverySubmissionID: "prompt"} + close(a.updates) + return nil +} + +func TestStopDrainsTerminalBeforeClosingJournalWriter(t *testing.T) { + mgr := NewManager(&config.InstanceConfig{SessionID: "session", DeliveryIncarnationID: "incarnation", + DeliveryHarnessGeneration: 1, DeliveryStreamID: "stream", DurableJournalPath: filepath.Join(t.TempDir(), "delivery.bbolt")}, newTestLogger(t)) + t.Cleanup(func() { _ = mgr.closeDeliveryJournal() }) + _, err := mgr.AdmitDeliverySubmission(context.Background(), journal.Submission{ID: "prompt", SessionID: "session", + IncarnationID: "incarnation", HarnessGeneration: 1, StreamID: "stream", Hash: "hash", Payload: []byte("old")}) + require.NoError(t, err) + mgr.TrackDeliverySubmission("prompt", 1) + a := &terminalOnCloseAdapter{updates: make(chan adapter.AgentEvent, 1)} + mgr.adapter = a + mgr.stopCh = make(chan struct{}) + mgr.wg.Add(1) + // Closing first deterministically makes both the event and stop ready. + mgr.closeAdapterAndStdin() + mgr.forwardUpdates(a, mgr.stopCh) + store, err := mgr.DeliveryJournal() + require.NoError(t, err) + stored, err := store.GetSubmission(context.Background(), "prompt") + require.NoError(t, err) + require.True(t, stored.TerminalEventRetained) + require.EqualValues(t, 1, stored.TerminalSequence) +} diff --git a/apps/backend/internal/backendapp/adapters_native_resume.go b/apps/backend/internal/backendapp/adapters_native_resume.go index e75adf99e3f..5dcb3a51e4d 100644 --- a/apps/backend/internal/backendapp/adapters_native_resume.go +++ b/apps/backend/internal/backendapp/adapters_native_resume.go @@ -18,3 +18,7 @@ func (a *lifecycleAdapter) ReconcileAlreadyResumedDelivery(ctx context.Context, func (a *lifecycleAdapter) GetNativeResumeDeliveryEvidence(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { return a.mgr.GetNativeResumeDeliveryEvidence(ctx, sessionID) } + +func (a *lifecycleAdapter) ReconcileLoadedNativeDelivery(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { + return a.mgr.ReconcileLoadedNativeDelivery(ctx, sessionID) +} diff --git a/apps/backend/internal/orchestrator/already_resumed_restart_recovery.go b/apps/backend/internal/orchestrator/already_resumed_restart_recovery.go index 18130069ecb..10dc4c07fe8 100644 --- a/apps/backend/internal/orchestrator/already_resumed_restart_recovery.go +++ b/apps/backend/internal/orchestrator/already_resumed_restart_recovery.go @@ -51,7 +51,10 @@ func (s *Service) finishAlreadyResumedRestartRecovery(ctx context.Context, input if !ok { return false, errors.New("restart recovery block store is unavailable") } - changed, err := blocks.ResolveSessionRecoveryBlock(ownerCtx, inputs.block.ID, "resume", time.Now().UTC()) + changed, handled, err := s.resolveNativeDeliveryBlock(ownerCtx, inputs.block, time.Now().UTC()) + if !handled { + changed, err = blocks.ResolveSessionRecoveryBlock(ownerCtx, inputs.block.ID, "resume", time.Now().UTC()) + } if err == nil && !changed { err = errors.New("restart recovery block changed before resolution") } diff --git a/apps/backend/internal/orchestrator/legacy_native_restart.go b/apps/backend/internal/orchestrator/legacy_native_restart.go new file mode 100644 index 00000000000..a4c94f8f519 --- /dev/null +++ b/apps/backend/internal/orchestrator/legacy_native_restart.go @@ -0,0 +1,142 @@ +package orchestrator + +import ( + "context" + "database/sql" + "errors" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/agentctl/journal" + "github.com/kandev/kandev/internal/task/models" + taskrepo "github.com/kandev/kandev/internal/task/repository" +) + +// Legacy shutdown labels may have no unresolved descriptor or recovery notice. +// Native resume itself is repeatable: the persisted block remains the admission +// fence until authenticated retirement and SQL proof commit together. +func (s *Service) resumeLegacyNativeRestart(ctx context.Context, session *models.TaskSession) (bool, error) { + if session == nil || session.State != models.TaskSessionStateWaitingForInput || !silentRestoreSessionRouteEligible(session) { + return false, nil + } + source, err := s.legacyNativeRestartSource(ctx, session) + if err != nil || source == nil { + return false, err + } + client, request, evidence, selected, block := source.client, source.request, source.evidence, source.selected, source.block + latest, err := client.InspectDeliveryRecordEvidence(ctx, request) + if err != nil { + return false, err + } + if !legacyNativeRestartEvidenceMatches(latest, selected, block) || !sameRestartDescriptor(evidence.Descriptor, latest.Descriptor) || *evidence.ControlIdentity != *latest.ControlIdentity { + return false, errors.New("legacy native restart evidence changed") + } + response, err := s.RecoverSession(ctx, session.TaskID, session.ID, "resume") + return err == nil && response != nil && response.Success, err +} + +func legacyNativeRestartEvidenceMatches(evidence *agentruntime.DeliveryRecordEvidence, row *models.AgentDeliverySubmission, block *models.SessionRecoveryBlock) bool { + if evidence == nil || evidence.ControlIdentity == nil || evidence.ProcessTerminated == nil || !*evidence.ProcessTerminated { + return false + } + d, peer, identity := evidence.Descriptor, evidence.RetiredSubmission, evidence.ControlIdentity + if !legacyNativeDescriptorIdle(d) { + return false + } + if peer == nil || peer.State == journal.SubmissionInterruptedUnknown || !journal.HasUnprovenOutcome(*peer) || peer.Retired { + return false + } + return legacyNativeRecordMatches(peer, row, d) && legacyNativeOwnerMatches(identity, peer, block) + +} + +func legacyRestartCursorCovered(ctx context.Context, repository taskrepo.AgentDeliveryRepository, session *models.TaskSession, generation *models.HarnessSessionGeneration, d journal.RecoveryDescriptor) bool { + cursor, err := repository.GetAgentDeliveryCursor(ctx, d.StreamID) + if errors.Is(err, sql.ErrNoRows) { + return d.Stream == nil || d.Stream.HighWater == 0 + } + if err != nil || cursor == nil || d.Stream == nil { + return false + } + return cursor.SessionID == session.ID && cursor.IncarnationID == session.QueueIncarnationID && cursor.HarnessGeneration == generation.Generation && cursor.ReceivedSequence == cursor.ProjectedSequence && uint64(cursor.ProjectedSequence) == d.Stream.HighWater +} + +type legacyRestartSource struct { + client deliveryRecordEvidenceClient + request agentruntime.DeliveryRecordEvidenceRequest + evidence *agentruntime.DeliveryRecordEvidence + selected *models.AgentDeliverySubmission + block *models.SessionRecoveryBlock +} + +func (s *Service) legacyNativeRestartSource(ctx context.Context, session *models.TaskSession) (*legacyRestartSource, error) { + client, ok := s.agentManager.(deliveryRecordEvidenceClient) + rows, canList := s.repo.(nativeResumeDeliverySubmissionReader) + repository, canRead := s.repo.(taskrepo.AgentDeliveryRepository) + if !ok || !canList || !canRead { + return nil, nil + } + generation, err := s.restartRecoveryNativeGeneration(ctx, session) + if err != nil || generation == nil { + return nil, err + } + block, err := s.GetOpenSessionRecoveryBlock(ctx, session.ID) + if err != nil || block == nil || block.ConsumerReference != agentDeliveryConsumer { + return nil, err + } + submissions, err := rows.ListAgentDeliverySubmissions(ctx, session.ID) + if err != nil { + return nil, err + } + source, err := readLegacyRestartInventory(ctx, client, session, generation, block, submissions) + if err != nil || source == nil { + return nil, err + } + evidence := source.evidence + if !legacyRestartCursorCovered(ctx, repository, session, generation, evidence.Descriptor) { + return nil, nil + } + return source, nil +} + +func readLegacyRestartInventory(ctx context.Context, client deliveryRecordEvidenceClient, session *models.TaskSession, generation *models.HarnessSessionGeneration, block *models.SessionRecoveryBlock, rows []*models.AgentDeliverySubmission) (*legacyRestartSource, error) { + var selected *legacyRestartSource + var descriptor *journal.RecoveryDescriptor + inputs := &silentRestoreInputs{session: session, recovery: models.AgentDeliveryRecovery{IncarnationID: session.QueueIncarnationID, HarnessGeneration: generation.Generation}} + for _, row := range rows { + if terminalDeliverySubmission(row.State) { + continue + } + if row.State != models.DeliverySubmissionInterruptedUnknown { + return nil, nil + } + request := agentruntime.DeliveryRecordEvidenceRequest{TaskID: session.TaskID, SessionID: session.ID, IncarnationID: session.QueueIncarnationID, HarnessGeneration: uint64(generation.Generation), RetiredSubmissionID: row.ID} + evidence, err := client.InspectDeliveryRecordEvidence(ctx, request) + if err != nil || evidence == nil { + return nil, err + } + if descriptor != nil && !sameRestartDescriptor(*descriptor, evidence.Descriptor) { + return nil, errors.New("legacy inventory changed") + } + descriptor = &evidence.Descriptor + if matchingRetiredRestartSubmission(inputs, row, evidence.RetiredSubmission) { + continue + } + if selected != nil || row.TerminalSequence != 0 || !legacyNativeRestartEvidenceMatches(evidence, row, block) { + return nil, nil + } + selected = &legacyRestartSource{client: client, request: request, evidence: evidence, selected: row, block: block} + } + return selected, nil +} + +func legacyNativeRecordMatches(peer *journal.Submission, row *models.AgentDeliverySubmission, d journal.RecoveryDescriptor) bool { + return peer.ID == row.ID && peer.Hash != "" && peer.Hash == row.PayloadHash && peer.SessionID == row.SessionID && peer.IncarnationID == row.IncarnationID && peer.HarnessGeneration == uint64(row.HarnessGeneration) && + peer.StreamID != "" && peer.StreamID == d.StreamID && d.SessionID == row.SessionID && d.IncarnationID == row.IncarnationID && d.HarnessGeneration == peer.HarnessGeneration +} +func legacyNativeOwnerMatches(identity *agentruntime.AgentDeliveryRecoveryIdentity, peer *journal.Submission, block *models.SessionRecoveryBlock) bool { + return (block.DeliverySubmissionID == "" || block.DeliverySubmissionID == peer.ID) && (block.DeliveryStreamID == "" || block.DeliveryStreamID == peer.StreamID) && identity.SubmissionID == peer.ID && identity.SessionID == peer.SessionID && identity.IncarnationID == peer.IncarnationID && identity.HarnessGeneration == peer.HarnessGeneration && identity.StreamID == peer.StreamID && identity.OriginalRuntime.Validate() == nil && identity.ExecutionID != "" && identity.PromptGeneration > 0 +} + +func legacyNativeDescriptorIdle(d journal.RecoveryDescriptor) bool { + return d.Durable && d.Version == journal.CurrentVersion && !d.SubmissionsTruncated && d.SubmissionCount == 0 && len(d.Submissions) == 0 && !d.Unresolved +} diff --git a/apps/backend/internal/orchestrator/legacy_native_restart_test.go b/apps/backend/internal/orchestrator/legacy_native_restart_test.go new file mode 100644 index 00000000000..5fb0068eead --- /dev/null +++ b/apps/backend/internal/orchestrator/legacy_native_restart_test.go @@ -0,0 +1,110 @@ +package orchestrator + +import ( + "context" + "errors" + "path/filepath" + "testing" + + agentruntime "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/agentctl/journal" + "github.com/kandev/kandev/internal/common/processidentity" + "github.com/kandev/kandev/internal/task/models" + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" + "github.com/stretchr/testify/require" +) + +type legacyNativeTestManager struct { + *nativeProofTestManager + inspect func(context.Context, agentruntime.DeliveryRecordEvidenceRequest) (*agentruntime.DeliveryRecordEvidence, error) +} + +func (m *legacyNativeTestManager) InspectDeliveryRecordEvidence(ctx context.Context, request agentruntime.DeliveryRecordEvidenceRequest) (*agentruntime.DeliveryRecordEvidence, error) { + return m.inspect(ctx, request) +} +func (m *legacyNativeTestManager) ReadDeliveryRecordSubmission(context.Context, agentruntime.DeliveryRecordSubmissionRequest) (*agentruntime.DeliveryRecordSubmission, error) { + return nil, errors.New("old prompt must never be read for dispatch") +} + +func TestLegacyNativeRestartWithoutRecoveryNotice(t *testing.T) { + for _, condition := range []string{"dead owner", "live owner", "hash mismatch", "retained terminal", "changed owner", "independent work"} { + t.Run(condition, func(t *testing.T) { + ctx := context.Background() + service, manager, block, row, launched, _ := nativeResumeFixture(t, "unknown_prompt_outcome", false, false) + repo := service.repo.(*sqliterepo.Repository) + require.NoError(t, repo.UpdateTaskSessionState(ctx, row.SessionID, models.TaskSessionStateWaitingForInput, "")) + require.NoError(t, repo.SetSessionMetadataKey(ctx, row.SessionID, models.SessionMetaKeyAgentDeliveryRecovery, nil)) + _, err := repo.DB().ExecContext(ctx, `UPDATE session_recovery_blocks SET delivery_stream_id = '' WHERE id = ?`, block.ID) + require.NoError(t, err) + store, err := journal.Open(journal.Config{Path: filepath.Join(t.TempDir(), "journal.bbolt")}) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + peer := journal.Submission{ID: row.ID, SessionID: row.SessionID, IncarnationID: row.IncarnationID, HarnessGeneration: 1, StreamID: block.DeliveryStreamID, Hash: row.PayloadHash, Payload: row.Payload, State: journal.SubmissionFailed} + _, err = store.PutSubmission(ctx, peer) + require.NoError(t, err) + calls := 0 + identity := agentruntime.AgentDeliveryRecoveryIdentity{TaskID: "task1", SessionID: row.SessionID, ExecutionID: "old-execution", IncarnationID: row.IncarnationID, HarnessGeneration: 1, StreamID: peer.StreamID, SubmissionID: peer.ID, PromptGeneration: 1, OriginalRuntime: processidentity.Identity{PID: 42, GroupID: 42, SessionID: 42, BirthToken: "linux:boot:1"}} + wrapper := &legacyNativeTestManager{nativeProofTestManager: &nativeProofTestManager{nativeResumeTestManager: manager}} + wrapper.inspect = func(ctx context.Context, _ agentruntime.DeliveryRecordEvidenceRequest) (*agentruntime.DeliveryRecordEvidence, error) { + calls++ + d, err := store.RecoveryDescriptor(ctx, row.SessionID, row.IncarnationID, 1, peer.StreamID) + require.NoError(t, err) + p, err := store.GetSubmission(ctx, peer.ID) + require.NoError(t, err) + dead := condition != "live owner" + binding := identity + switch condition { + case "hash mismatch": + p.Hash = "foreign" + case "retained terminal": + p.TerminalEventRetained = true + p.TerminalSequence = 1 + case "changed owner": + if calls > 1 { + binding.ExecutionID = "foreign" + } + case "independent work": + d.SubmissionCount = 1 + } + return &agentruntime.DeliveryRecordEvidence{Descriptor: d, RetiredSubmission: &p, ControlIdentity: &binding, ProcessTerminated: &dead}, nil + } + originalAck := manager.acknowledge + manager.acknowledge = func(ctx context.Context, sessionID string) error { + require.NoError(t, originalAck(ctx, sessionID)) + _, err := store.RetireSubmission(ctx, peer.ID, 1) + return err + } + wrapper.proof = func(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { + running, err := repo.GetExecutorRunningBySessionID(ctx, sessionID) + require.NoError(t, err) + retired, err := store.GetSubmission(ctx, peer.ID) + require.NoError(t, err) + require.True(t, retired.Retired) + return &models.NativeResumeDeliveryEvidence{SessionID: sessionID, IncarnationID: row.IncarnationID, HarnessGeneration: 1, NativeSessionID: "native-conversation", ExecutionID: running.AgentExecutionID, StreamID: peer.StreamID, Submissions: []models.NativeResumeSubmissionEvidence{{ID: peer.ID, HarnessGeneration: 1, PayloadHash: peer.Hash, StreamID: peer.StreamID}}}, nil + } + service.agentManager = wrapper + session, err := repo.GetTaskSession(ctx, row.SessionID) + require.NoError(t, err) + restored, err := service.resumeLegacyNativeRestart(ctx, session) + if condition == "changed owner" { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, condition == "dead owner", restored) + if restored { + require.NotNil(t, *launched) + saved, err := store.GetSubmission(ctx, peer.ID) + require.NoError(t, err) + require.True(t, saved.Retired) + require.Equal(t, journal.SubmissionFailed, saved.State) + again, err := service.resumeLegacyNativeRestart(ctx, session) + require.NoError(t, err) + require.False(t, again) + } else { + require.Nil(t, *launched) + } + require.Empty(t, manager.capturedPromptCalls) + }) + } +} diff --git a/apps/backend/internal/orchestrator/native_resume_proof_test.go b/apps/backend/internal/orchestrator/native_resume_proof_test.go index 2b41db8f28a..fcff5f6dfc4 100644 --- a/apps/backend/internal/orchestrator/native_resume_proof_test.go +++ b/apps/backend/internal/orchestrator/native_resume_proof_test.go @@ -31,7 +31,7 @@ func TestNativeResumeRepairsAbsentNoticeAndStreamThroughAuthenticatedProof(t *te return &models.NativeResumeDeliveryEvidence{SessionID: sessionID, IncarnationID: submission.IncarnationID, HarnessGeneration: submission.HarnessGeneration, NativeSessionID: "native-conversation", ExecutionID: running.AgentExecutionID, StreamID: block.DeliveryStreamID, Submissions: []models.NativeResumeSubmissionEvidence{{ID: submission.ID, - HarnessGeneration: submission.HarnessGeneration, PayloadHash: submission.PayloadHash}}}, nil + HarnessGeneration: submission.HarnessGeneration, PayloadHash: submission.PayloadHash, StreamID: block.DeliveryStreamID}}}, nil }} response, err := service.RecoverSession(ctx, "task1", submission.SessionID, "resume") require.NoError(t, err) @@ -48,3 +48,40 @@ func TestNativeResumeRepairsAbsentNoticeAndStreamThroughAuthenticatedProof(t *te require.Equal(t, submission.Payload, retained.Payload) require.Zero(t, retained.TerminalSequence) } + +type loadedNativeProofTestManager struct{ *nativeProofTestManager } + +func (m *loadedNativeProofTestManager) ReconcileLoadedNativeDelivery(ctx context.Context, sessionID string) (*models.NativeResumeDeliveryEvidence, error) { + return m.proof(ctx, sessionID) +} + +func TestLoadedNativeRecoveryClearsAdmissionWithoutReplay(t *testing.T) { + for _, entry := range []string{"admission", "restart"} { + t.Run(entry, func(t *testing.T) { + ctx := context.Background() + service, manager, block, submission, _, _ := nativeResumeFixture(t, "unknown_prompt_outcome", false, false) + repo := service.repo.(*sqliterepo.Repository) + require.NoError(t, repo.SetSessionMetadataKey(ctx, submission.SessionID, models.SessionMetaKeyAgentDeliveryRecovery, nil)) + _, err := repo.DB().ExecContext(ctx, `UPDATE session_recovery_blocks SET delivery_stream_id = '' WHERE id = ?`, block.ID) + require.NoError(t, err) + running, err := repo.GetExecutorRunningBySessionID(ctx, submission.SessionID) + require.NoError(t, err) + service.agentManager = &loadedNativeProofTestManager{&nativeProofTestManager{nativeResumeTestManager: manager, proof: func(context.Context, string) (*models.NativeResumeDeliveryEvidence, error) { + return &models.NativeResumeDeliveryEvidence{SessionID: submission.SessionID, IncarnationID: submission.IncarnationID, HarnessGeneration: submission.HarnessGeneration, NativeSessionID: "native-conversation", ExecutionID: running.AgentExecutionID, StreamID: block.DeliveryStreamID, Submissions: []models.NativeResumeSubmissionEvidence{{ID: submission.ID, HarnessGeneration: submission.HarnessGeneration, PayloadHash: submission.PayloadHash, StreamID: block.DeliveryStreamID}}}, nil + }}} + if entry == "restart" { + session, err := repo.GetTaskSession(ctx, submission.SessionID) + require.NoError(t, err) + _, err = service.repairRestartRecoveryPresentation(ctx, session) + require.NoError(t, err) + } else { + require.NoError(t, service.checkSessionRecoveryBlock(ctx, submission.SessionID)) + } + restored, err := repo.GetSessionRecoveryBlock(ctx, block.ID) + require.NoError(t, err) + require.Equal(t, models.RecoveryBlockResolved, restored.State) + require.NoError(t, service.checkSessionRecoveryBlock(ctx, submission.SessionID)) + require.Empty(t, manager.capturedPromptCalls) + }) + } +} diff --git a/apps/backend/internal/orchestrator/restart_recovery_presentation.go b/apps/backend/internal/orchestrator/restart_recovery_presentation.go index 2bc4aecbad2..22c77aa39b5 100644 --- a/apps/backend/internal/orchestrator/restart_recovery_presentation.go +++ b/apps/backend/internal/orchestrator/restart_recovery_presentation.go @@ -27,6 +27,11 @@ func (s *Service) repairRestartRecoveryPresentation(ctx context.Context, session if !ok { return session, nil } + if open, readErr := blocks.GetOpenSessionRecoveryBlock(ctx, session.ID, session.QueueIncarnationID, generation.Generation); readErr == nil && open != nil { + if resolved, _ := s.finishLoadedNativeDeliveryBlock(ctx, open); resolved { + return s.repo.GetTaskSession(ctx, session.ID) + } + } block, reason, err := s.deliveryReconstructionBlock(ctx, session.ID, session.QueueIncarnationID, generation.Generation, nil) if err != nil { return nil, err diff --git a/apps/backend/internal/orchestrator/session_launch.go b/apps/backend/internal/orchestrator/session_launch.go index d6b3c6557ad..32918bfde11 100644 --- a/apps/backend/internal/orchestrator/session_launch.go +++ b/apps/backend/internal/orchestrator/session_launch.go @@ -481,6 +481,9 @@ func (s *Service) checkSessionRecoveryBlock(ctx context.Context, sessionID strin if err != nil { return fmt.Errorf("check session recovery block: %w", err) } + if resolved, _ := s.finishLoadedNativeDeliveryBlock(ctx, block); resolved { + return nil + } return &sessionRecoveryRequiredError{Block: block} } diff --git a/apps/backend/internal/orchestrator/session_native_delivery_recovery.go b/apps/backend/internal/orchestrator/session_native_delivery_recovery.go index cd8e27b7eaa..2099361a06f 100644 --- a/apps/backend/internal/orchestrator/session_native_delivery_recovery.go +++ b/apps/backend/internal/orchestrator/session_native_delivery_recovery.go @@ -102,3 +102,32 @@ func (s *Service) resolveNativeDeliveryBlock(ctx context.Context, block *models. changed, err := resolver.ResolveSessionRecoveryBlockWithNativeEvidence(ctx, block.ID, proof, completedAt) return changed, true, err } + +type loadedNativeDeliveryReconciler interface { + ReconcileLoadedNativeDelivery(context.Context, string) (*models.NativeResumeDeliveryEvidence, error) +} + +func (s *Service) finishLoadedNativeDeliveryBlock(ctx context.Context, block *models.SessionRecoveryBlock) (bool, error) { + manager, supported := s.agentManager.(loadedNativeDeliveryReconciler) + resolver, canResolve := s.repo.(nativeResumeEvidenceBlockResolver) + if !supported || !canResolve || block == nil || block.ConsumerReference != agentDeliveryConsumer { + return false, nil + } + session, err := s.repo.GetTaskSession(ctx, block.SessionID) + if err != nil || session == nil { + return false, err + } + office, err := s.lookupOfficeTask(ctx, session.TaskID) + if err != nil || office { + return false, err + } + proof, err := manager.ReconcileLoadedNativeDelivery(ctx, block.SessionID) + if err != nil || proof == nil { + return false, err + } + changed, err := resolver.ResolveSessionRecoveryBlockWithNativeEvidence(ctx, block.ID, proof, time.Now().UTC()) + if err == nil && changed { + s.publishAgentDeliveryRecoveryState(ctx, block.SessionID) + } + return changed, err +} diff --git a/apps/backend/internal/orchestrator/silent_restart_recovery_inputs.go b/apps/backend/internal/orchestrator/silent_restart_recovery_inputs.go index 44b19417702..27d892ca01d 100644 --- a/apps/backend/internal/orchestrator/silent_restart_recovery_inputs.go +++ b/apps/backend/internal/orchestrator/silent_restart_recovery_inputs.go @@ -38,6 +38,13 @@ func (s *Service) loadSilentRestoreInputs( if err != nil || !eligible { return nil, err } + current, err := s.repo.GetTaskSession(owner.ctx, candidate.SessionID) + if err != nil { + return nil, err + } + if restored, err := s.resumeLegacyNativeRestart(owner.ctx, current); err != nil || restored { + return nil, err + } session, recovery, eligible, err := s.loadSilentRestoreSession( owner.ctx, candidate, owner.task.ID, ) diff --git a/apps/backend/internal/task/repository/sqlite/native_resume_binding.go b/apps/backend/internal/task/repository/sqlite/native_resume_binding.go index b2421bc0b04..6de03e9bf1a 100644 --- a/apps/backend/internal/task/repository/sqlite/native_resume_binding.go +++ b/apps/backend/internal/task/repository/sqlite/native_resume_binding.go @@ -14,8 +14,7 @@ import ( // ResolveSessionRecoveryBlockWithNativeEvidence commits only after authenticated // native retirement. All incident fences are verified before any becomes usable. func (r *Repository) ResolveSessionRecoveryBlockWithNativeEvidence(ctx context.Context, id string, proof *models.NativeResumeDeliveryEvidence, completedAt time.Time) (bool, error) { - if proof == nil || proof.SessionID == "" || proof.IncarnationID == "" || proof.HarnessGeneration <= 0 || - proof.NativeSessionID == "" || proof.StreamID == "" || proof.ExecutionID == "" { + if !validNativeResumeProof(proof) { return false, nil } tx, err := r.db.BeginTxx(ctx, nil) @@ -44,23 +43,9 @@ func (r *Repository) ResolveSessionRecoveryBlockWithNativeEvidence(ctx context.C if err := r.advanceRecoverySessionRevisionTx(ctx, tx, proof.SessionID); err != nil { return false, err } - for _, block := range blocks { - stream := block.DeliveryStreamID - if block.DeliverySubmissionID != "" && stream == "" { - stream = proof.StreamID - } - result, err := tx.ExecContext(ctx, r.db.Rebind(`UPDATE session_recovery_blocks SET delivery_stream_id = ?, state = ?, authorized_action = ?, updated_at = ?, resolved_at = ? WHERE id = ? AND state = ?`), - stream, models.RecoveryBlockResolved, nativeResumeRecoveryAction, completedAt, completedAt, block.ID, models.RecoveryBlockOpen) - if err != nil { - return false, err - } - count, err := result.RowsAffected() - if err != nil { - return false, err - } - if count != 1 { - return false, nil - } + changed, err := r.resolveNativeIncidentBlocksTx(ctx, tx, blocks, proof, completedAt) + if err != nil || !changed { + return false, err } return true, tx.Commit() } @@ -87,15 +72,9 @@ func (r *Repository) verifyNativeResumeSnapshotTx(ctx context.Context, tx *sqlx. } func (r *Repository) verifyNativeResumeInventoryTx(ctx context.Context, tx *sqlx.Tx, proof *models.NativeResumeDeliveryEvidence) (bool, error) { - retired := make(map[string]models.NativeResumeSubmissionEvidence, len(proof.Submissions)) - for _, row := range proof.Submissions { - if row.ID == "" || row.PayloadHash == "" || row.HarnessGeneration <= 0 || row.HarnessGeneration > proof.HarnessGeneration { - return false, nil - } - if _, duplicate := retired[row.ID]; duplicate { - return false, nil - } - retired[row.ID] = row + retired, valid := nativeResumeSubmissionMap(proof) + if !valid { + return false, nil } rows, err := tx.QueryxContext(ctx, r.db.Rebind(`SELECT id, incarnation_id, harness_generation, payload_hash, state, terminal_stream_id, terminal_sequence FROM agent_delivery_submissions WHERE session_id = ? AND state IN ('prepared', 'accepted', 'dispatching', 'interrupted_unknown')`), proof.SessionID) if err != nil { @@ -164,9 +143,7 @@ func (r *Repository) nativeResumeIncidentBlocksTx(ctx context.Context, tx *sqlx. } func nativeResumeIncidentBlockMatches(block models.SessionRecoveryBlock, proof *models.NativeResumeDeliveryEvidence) bool { - if block.IncarnationID != proof.IncarnationID || block.ExpectedGeneration != proof.HarnessGeneration || block.ConsumerReference != reconstructionDeliveryConsumer || - (block.Reason != "unknown_prompt_outcome" && block.Reason != "unresolved_durable_work") || - block.DeliverySequence != 0 || block.DeliveryTurnID != "" || block.DeliveryOutcome != "" { + if !nativeResumeIncidentOwnerMatches(block, proof) { return false } if block.DeliverySubmissionID == "" { @@ -176,7 +153,7 @@ func nativeResumeIncidentBlockMatches(block models.SessionRecoveryBlock, proof * return false } for _, row := range proof.Submissions { - if row.ID == block.DeliverySubmissionID && row.HarnessGeneration == proof.HarnessGeneration { + if row.ID == block.DeliverySubmissionID && row.HarnessGeneration == proof.HarnessGeneration && row.StreamID == proof.StreamID { return true } } @@ -214,15 +191,7 @@ func (r *Repository) restoreAuthenticatedNativeNoticeTx(ctx context.Context, tx } return nativeResumeProofCoversNotice(proof, recovery), nil } - if recovery.HarnessGeneration != proof.HarnessGeneration || (recovery.StreamID != "" && recovery.StreamID != proof.StreamID) || - (recovery.Phase != models.AgentDeliveryRecoveryUncertain && recovery.Phase != models.AgentDeliveryRecoveryReconnecting) { - return false, nil - } - found := false - for _, row := range proof.Submissions { - found = found || row.ID == recovery.SubmissionID && row.HarnessGeneration == recovery.HarnessGeneration - } - if !found { + if !nativeResumeNoticeMatches(proof, recovery) { return false, nil } recovery.StreamID = proof.StreamID @@ -237,3 +206,67 @@ func nativeResumeProofCoversNotice(proof *models.NativeResumeDeliveryEvidence, r } return false } + +func validNativeResumeProof(proof *models.NativeResumeDeliveryEvidence) bool { + return proof != nil && proof.SessionID != "" && proof.IncarnationID != "" && proof.HarnessGeneration > 0 && proof.NativeSessionID != "" && proof.StreamID != "" && proof.ExecutionID != "" +} +func (r *Repository) resolveNativeIncidentBlocksTx(ctx context.Context, tx *sqlx.Tx, blocks []models.SessionRecoveryBlock, proof *models.NativeResumeDeliveryEvidence, completedAt time.Time) (bool, error) { + for _, block := range blocks { + stream := block.DeliveryStreamID + if block.DeliverySubmissionID != "" && stream == "" { + stream = proof.StreamID + } + result, err := tx.ExecContext(ctx, r.db.Rebind(`UPDATE session_recovery_blocks SET delivery_stream_id = ?, state = ?, authorized_action = ?, updated_at = ?, resolved_at = ? WHERE id = ? AND state = ?`), + stream, models.RecoveryBlockResolved, nativeResumeRecoveryAction, completedAt, completedAt, block.ID, models.RecoveryBlockOpen) + if err != nil { + return false, err + } + count, err := result.RowsAffected() + if err != nil { + return false, err + } + if count != 1 { + return false, nil + } + } + return true, nil +} + +func nativeResumeSubmissionMap(proof *models.NativeResumeDeliveryEvidence) (map[string]models.NativeResumeSubmissionEvidence, bool) { + retired := make(map[string]models.NativeResumeSubmissionEvidence, len(proof.Submissions)) + for _, row := range proof.Submissions { + if row.ID == "" || row.PayloadHash == "" || row.StreamID == "" || row.HarnessGeneration <= 0 || row.HarnessGeneration > proof.HarnessGeneration || + (row.HarnessGeneration == proof.HarnessGeneration && row.StreamID != proof.StreamID) { + return nil, false + } + if _, duplicate := retired[row.ID]; duplicate { + return nil, false + } + retired[row.ID] = row + } + return retired, true +} + +func nativeResumeIncidentOwnerMatches(block models.SessionRecoveryBlock, proof *models.NativeResumeDeliveryEvidence) bool { + if block.IncarnationID != proof.IncarnationID || block.ExpectedGeneration != proof.HarnessGeneration || block.ConsumerReference != reconstructionDeliveryConsumer || + (block.Reason != "unknown_prompt_outcome" && block.Reason != "unresolved_durable_work") || + block.DeliverySequence != 0 || block.DeliveryTurnID != "" || (block.DeliveryOutcome != "" && block.DeliveryOutcome != "unknown") { + return false + } + return true +} + +func nativeResumeNoticeMatches(proof *models.NativeResumeDeliveryEvidence, recovery models.AgentDeliveryRecovery) bool { + if recovery.HarnessGeneration != proof.HarnessGeneration || (recovery.StreamID != "" && recovery.StreamID != proof.StreamID) || + (recovery.Phase != models.AgentDeliveryRecoveryUncertain && recovery.Phase != models.AgentDeliveryRecoveryReconnecting) { + return false + } + found := false + for _, row := range proof.Submissions { + found = found || row.ID == recovery.SubmissionID && row.HarnessGeneration == recovery.HarnessGeneration + } + if !found { + return false + } + return true +} diff --git a/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go b/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go index 4254a276786..d5e11ae74e6 100644 --- a/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go +++ b/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go @@ -29,7 +29,7 @@ func TestPostgresNativeResumeAuthenticatedMissingBinding(t *testing.T) { } func testNativeResumeAuthenticatedMissingBinding(t *testing.T, open func(*testing.T) *Repository) { - for _, scenario := range []string{"absent notice", "partial notice", "resolved earlier notice", "matching fences", "independent fence", "foreign native", "hash changed", "active work", "unmatched unknown", "foreign stream", "rollback"} { + for _, scenario := range []string{"absent notice", "partial notice", "resolved earlier notice", "matching fences", "independent fence", "foreign native", "hash changed", "active work", "unmatched unknown", "foreign stream", "foreign submission stream", "missing submission stream", "reconstructed unknown", "known outcome", "rollback"} { t.Run(scenario, func(t *testing.T) { repo := open(t) ctx := context.Background() @@ -86,9 +86,20 @@ func testNativeResumeAuthenticatedMissingBinding(t *testing.T, open func(*testin if scenario == "resolved earlier notice" { current, err := repo.GetAgentDeliverySubmission(ctx, fixture.block.DeliverySubmissionID) require.NoError(t, err) - proof.Submissions = append(proof.Submissions, models.NativeResumeSubmissionEvidence{ID: current.ID, HarnessGeneration: 1, PayloadHash: current.PayloadHash}) + proof.Submissions = append(proof.Submissions, models.NativeResumeSubmissionEvidence{ID: current.ID, HarnessGeneration: 1, PayloadHash: current.PayloadHash, StreamID: proof.StreamID}) } switch scenario { + case "reconstructed unknown", "known outcome": + outcome := "unknown" + if scenario == "known outcome" { + outcome = "completed" + } + _, err = repo.DB().ExecContext(ctx, repo.db.Rebind(`UPDATE session_recovery_blocks SET delivery_outcome = ? WHERE id = ?`), outcome, fixture.block.ID) + require.NoError(t, err) + case "foreign submission stream": + proof.Submissions[0].StreamID = "foreign" + case "missing submission stream": + proof.Submissions[0].StreamID = "" case "foreign native": proof.NativeSessionID = "foreign" case "hash changed": @@ -120,7 +131,7 @@ func testNativeResumeAuthenticatedMissingBinding(t *testing.T, open func(*testin } else { require.NoError(t, err) } - want := scenario == "absent notice" || scenario == "partial notice" || scenario == "resolved earlier notice" || scenario == "matching fences" + want := scenario == "reconstructed unknown" || scenario == "absent notice" || scenario == "partial notice" || scenario == "resolved earlier notice" || scenario == "matching fences" require.Equal(t, want, changed) block, err := repo.GetSessionRecoveryBlock(ctx, fixture.block.ID) require.NoError(t, err) diff --git a/docs/plans/session-recovery-convergence/plan.md b/docs/plans/session-recovery-convergence/plan.md index 10bfbc0076c..a51f45c032a 100644 --- a/docs/plans/session-recovery-convergence/plan.md +++ b/docs/plans/session-recovery-convergence/plan.md @@ -50,6 +50,8 @@ No completion claim follows merely from recovering the four observed records. ## Results -In progress. Real shutdown regression was red; initial focused producer race -suite is green. Comprehensive assessment, binding convergence and delivery -remain incomplete. No local E2E has run. +PR #4432 is open. Focused producer, SQL and lifecycle race checks pass, including +the 49-state-pair matrix. CI identified documentation metadata and function +complexity failures; corrections and review regressions are in progress. +Automatic restart/admission closure and final CI remain pending. No local E2E +has run. diff --git a/docs/plans/session-recovery-convergence/task-01-shutdown-settlement.md b/docs/plans/session-recovery-convergence/task-01-shutdown-settlement.md index 74102bf1775..75514a24663 100644 --- a/docs/plans/session-recovery-convergence/task-01-shutdown-settlement.md +++ b/docs/plans/session-recovery-convergence/task-01-shutdown-settlement.md @@ -1,5 +1,17 @@ --- +id: "01-shutdown-settlement" +title: "Shutdown settlement" status: in_progress +wave: 1 +depends_on: [] +plan: plan.md +requirements: + - REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-002 +acceptance_criteria: + - AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.1 + - AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.2 +system_design: + - ../../specs/platform/system-design/session-recovery-convergence.md --- # Task 01: Shutdown settlement @@ -30,6 +42,8 @@ From apps/backend: ## Results Real graceful-stop regression confirmed red before production correction. -Focused TestSubmission race suite passed after initial correction; controls pending. +Focused producer race checks pass, including deterministic rejection, cancelled +RequestError, retained error settlement and queued terminal draining. Shutdown +intent is independent of update-forwarder termination. CI remains authoritative. [Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/plans/session-recovery-convergence/task-02-native-convergence.md b/docs/plans/session-recovery-convergence/task-02-native-convergence.md index e5761fe65ed..0b2cbf49701 100644 --- a/docs/plans/session-recovery-convergence/task-02-native-convergence.md +++ b/docs/plans/session-recovery-convergence/task-02-native-convergence.md @@ -1,5 +1,19 @@ --- +id: "02-native-convergence" +title: "Native convergence" status: in_progress +wave: 2 +depends_on: + - 01-shutdown-settlement +plan: plan.md +requirements: + - REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-002 +acceptance_criteria: + - AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.2 + - AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.3 + - AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.4 +system_design: + - ../../specs/platform/system-design/session-recovery-convergence.md --- # Task 02: Native convergence @@ -32,6 +46,9 @@ PostgreSQL variants run in CI against real PostgreSQL. ## Results -Pending. +The 49-pair lifecycle matrix and SQLite authenticated-binding suite pass. +Foreign/missing submission streams and reconstructed unknown outcomes were +reproduced before correction. Automatic entrypoint checks and final CI are +in progress. [Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/plans/session-recovery-convergence/task-03-recovery-contract.md b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md index b6038502c76..e20ee5eff61 100644 --- a/docs/plans/session-recovery-convergence/task-03-recovery-contract.md +++ b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md @@ -1,5 +1,19 @@ --- -status: pending +id: "03-recovery-contract" +title: "Recovery contract and delivery" +status: in_progress +wave: 3 +depends_on: + - 02-native-convergence +plan: plan.md +requirements: + - REQ-PLATFORM-DURABLE-RESTART-RECONCILIATION-002 +acceptance_criteria: + - AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.3 + - AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.4 + - AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-002.5 +system_design: + - ../../specs/platform/system-design/session-recovery-convergence.md --- # Task 03: Recovery contract and delivery @@ -32,6 +46,10 @@ Desktop and phone E2E are CI-only; never run locally. ## Results -Pending. PR #4425 is merged; overall recovery remains in progress. +PR #4432 is open and running CI. Documentation coverage passes locally after +correcting work-order metadata. Review remediation covers terminal drain ordering, +complete stream snapshot validation, exact submission stream binding and unknown +reconstruction outcomes. Final automatic restart checks, CI and review remain +in progress. No local E2E has run. [Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/specs/platform/system-design/session-recovery-convergence.md b/docs/specs/platform/system-design/session-recovery-convergence.md index 942779ba0ec..fdd4ead4009 100644 --- a/docs/specs/platform/system-design/session-recovery-convergence.md +++ b/docs/specs/platform/system-design/session-recovery-convergence.md @@ -40,9 +40,10 @@ with exact authenticated journal evidence; repeat adoption uses the same rule. ## Producer ordering -Capture the admitted child's stop channel for dispatch. Publish interruption -before closing its writer or adapter, so a teardown-generated RequestError is -uncertain. Do not consult mutable replacement status or acquire the teardown +Capture the admitted child's shutdown-intent flag for dispatch. Publish interruption +before closing its adapter, so a teardown-generated RequestError is uncertain. +Keep the update forwarder and journal writer open through adapter closure and +drain already-queued events before closing the writer. Do not consult mutable replacement status or acquire the teardown mutex inside settlement. A deterministic provider rejection remains a failure; a retained terminal wins over a late transport error. Settle with a bounded cancellation-independent context. Journal close or failed writes leave retained @@ -70,7 +71,10 @@ of the binding, notice or fence. A retry reuses retired journal proof. Startup/runtime replacement and explicit Resume reuse existing restore ownership and source-death checks. Normal launch/open and direct/queued prompt admission consume the same committed disposition; read-only inspection cannot grant -admission. An already-loaded idle native successor is reused. Recovery supplies +admission. An already-loaded idle native successor is reused after verifying the persisted +original prompt binding and original process death. Legacy failed history with +no recovery notice uses the same native Resume transaction only after retained +source evidence, exact SQL history and complete stream projection are rechecked. Recovery supplies no prompt and does not revive a dispatched queue entry. A distinct new prompt requires a new immutable submission and normal workflow/queue authority. From 0e514a3315df64e6824f90b47132f9c69271c2eb Mon Sep 17 00:00:00 2001 From: Kandev Agent Date: Mon, 12 Oct 2026 03:35:02 +0100 Subject: [PATCH 3/7] fix: settle recovery checkpoints and inspect retained shutdown history --- .../lifecycle/delivery_record_evidence.go | 21 ++++-- .../lifecycle/delivery_record_transport.go | 64 ++++++++++++---- .../lifecycle/native_resume_delivery.go | 17 ++--- .../lifecycle/native_resume_delivery_test.go | 2 +- .../internal/agentctl/journal/retained.go | 25 ++++++- .../journal/stored_recovery_evidence_test.go | 62 ++++++++++++++++ .../agentctl/server/process/manager.go | 3 + .../process/submission_shutdown_test.go | 9 ++- .../orchestrator/legacy_native_restart.go | 6 +- .../legacy_native_restart_test.go | 49 ++++++++++-- .../restart_submission_reconciliation.go | 2 +- .../session_delivery_reconstruction_test.go | 4 +- .../sqlite/native_resume_attempt.go | 60 +++++++++++++++ .../sqlite/native_resume_binding.go | 36 ++++++--- .../sqlite/native_resume_binding_test.go | 30 +++++++- .../repository/sqlite/native_resume_error.go | 74 +++++++++++++++++++ .../task-02-native-convergence.md | 8 +- .../session-recovery-convergence.md | 8 +- 18 files changed, 413 insertions(+), 67 deletions(-) create mode 100644 apps/backend/internal/agentctl/journal/stored_recovery_evidence_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/native_resume_attempt.go create mode 100644 apps/backend/internal/task/repository/sqlite/native_resume_error.go diff --git a/apps/backend/internal/agent/runtime/lifecycle/delivery_record_evidence.go b/apps/backend/internal/agent/runtime/lifecycle/delivery_record_evidence.go index 1116c4610ce..30a7cb88e24 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/delivery_record_evidence.go +++ b/apps/backend/internal/agent/runtime/lifecycle/delivery_record_evidence.go @@ -27,6 +27,7 @@ var ( // historical execution whose process proof may be available. It contains no // storage path and is not a browser-facing request. type DeliveryRecordEvidenceRequest struct { + StoredSubmissionID string RetiredSubmissionID string TaskID string SessionID string @@ -39,6 +40,7 @@ type DeliveryRecordEvidenceRequest struct { // DeliveryRecordEvidence separates the live session instance from proof about // the historical process that owned a candidate prompt. type DeliveryRecordEvidence struct { + StoredSubmission *journal.Submission RetiredSubmission *journal.Submission Descriptor journal.RecoveryDescriptor ProcessTerminated *bool @@ -93,12 +95,8 @@ func (m *Manager) InspectDeliveryRecordEvidence( return result, errors.Join(ErrDeliveryRecordEvidenceTruncated, ErrDeliveryRecoveryBlocked) } result.ControlIdentity = m.deliveryRecordControlIdentity(ctx, request, result.Descriptor) - if result.ControlIdentity == nil && result.Descriptor.SubmissionCount == 0 && result.RetiredSubmission != nil { - selected := result.RetiredSubmission - if selected.ID == request.RetiredSubmissionID && selected.SessionID == request.SessionID && selected.IncarnationID == request.IncarnationID && selected.HarnessGeneration == request.HarnessGeneration { - result.ControlIdentity = m.persistedDeliveryRecordControlIdentity(ctx, request, journal.SubmissionSummary{ - ID: selected.ID, SessionID: selected.SessionID, IncarnationID: selected.IncarnationID, HarnessGeneration: selected.HarnessGeneration, StreamID: selected.StreamID}) - } + if result.ControlIdentity == nil { + result.ControlIdentity = m.targetedDeliveryRecordControlIdentity(ctx, request, result) } if identity := result.ControlIdentity; identity != nil { result.ProcessTerminated = historicalProcessTermination(DeliveryRecordEvidenceRequest{ @@ -278,3 +276,14 @@ func historicalProcessTermination(request DeliveryRecordEvidenceRequest, liveExe } return &terminated } + +func (m *Manager) targetedDeliveryRecordControlIdentity(ctx context.Context, request DeliveryRecordEvidenceRequest, result *DeliveryRecordEvidence) *AgentDeliveryRecoveryIdentity { + if result.Descriptor.SubmissionCount == 0 && result.StoredSubmission != nil { + selected := result.StoredSubmission + if selected.ID == request.StoredSubmissionID && selected.SessionID == request.SessionID && selected.IncarnationID == request.IncarnationID && selected.HarnessGeneration == request.HarnessGeneration { + return m.persistedDeliveryRecordControlIdentity(ctx, request, journal.SubmissionSummary{ + ID: selected.ID, SessionID: selected.SessionID, IncarnationID: selected.IncarnationID, HarnessGeneration: selected.HarnessGeneration, StreamID: selected.StreamID}) + } + } + return nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/delivery_record_transport.go b/apps/backend/internal/agent/runtime/lifecycle/delivery_record_transport.go index 0a0c32085b9..0939a2b5133 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/delivery_record_transport.go +++ b/apps/backend/internal/agent/runtime/lifecycle/delivery_record_transport.go @@ -16,13 +16,13 @@ func (m *Manager) inspectDeliveryRecordTransport(ctx context.Context, lease *age func (m *Manager) inspectRetainedDeliveryRecord(ctx context.Context, lease *agentctl.RuntimeLease, control *agentctl.ControlClient, request DeliveryRecordEvidenceRequest) (*DeliveryRecordEvidence, error) { retained, readErr := control.ReadRetainedReconstructionEvidence(ctx, journal.RetainedReconstructionEvidenceRequest{ Root: m.dataDir, SessionID: request.SessionID, IncarnationID: request.IncarnationID, - HarnessGeneration: request.HarnessGeneration, OriginalRuntime: request.OriginalRuntime, RetiredSubmissionID: request.RetiredSubmissionID, + HarnessGeneration: request.HarnessGeneration, OriginalRuntime: request.OriginalRuntime, RetiredSubmissionID: request.RetiredSubmissionID, StoredSubmissionID: request.StoredSubmissionID, }) if readErr != nil { return nil, deliveryEvidenceReadError(lease, readErr) } return &DeliveryRecordEvidence{ - Descriptor: retained.Descriptor, ProcessTerminated: retained.ProcessTerminated, RetiredSubmission: retained.RetiredSubmission, + Descriptor: retained.Descriptor, ProcessTerminated: retained.ProcessTerminated, RetiredSubmission: retained.RetiredSubmission, StoredSubmission: retained.StoredSubmission, }, nil } @@ -44,20 +44,8 @@ func (m *Manager) inspectLiveDeliveryRecord(ctx context.Context, lease *agentctl Descriptor: *descriptor, LiveExecutionID: instance.ID, ProcessTerminated: historicalProcessTermination(request, instance.ID), } - if request.RetiredSubmissionID != "" { - retired, err := client.GetDeliverySubmission(ctx, request.RetiredSubmissionID) - if err != nil { - return nil, deliveryEvidenceReadError(lease, err) - } - after, err := client.GetDeliveryStatus(ctx, "") - if err != nil { - return nil, deliveryEvidenceReadError(lease, err) - } - if !sameDeliveryRecordDescriptor(*descriptor, *after) { - return nil, ErrDeliveryRecordEvidenceStale - } - retired.Payload = nil - result.RetiredSubmission = retired + if err := m.readLiveStoredEvidence(ctx, lease, client, request, result); err != nil { + return nil, err } return result, nil } @@ -147,3 +135,47 @@ func (m *Manager) readLiveDeliveryRecord(ctx context.Context, lease *agentctl.Ru after = *second return &deliveryRecordRead{before: before, after: after, submission: submission}, nil } + +func (m *Manager) readLiveStoredEvidence(ctx context.Context, lease *agentctl.RuntimeLease, client *agentctl.Client, request DeliveryRecordEvidenceRequest, result *DeliveryRecordEvidence) error { + for _, id := range []string{request.RetiredSubmissionID, request.StoredSubmissionID} { + if id == "" { + continue + } + record, err := readLiveSubmissionMetadata(ctx, client, request, id) + if err != nil { + return deliveryEvidenceReadError(lease, err) + } + if id == request.RetiredSubmissionID { + if !record.Retired || !journal.HasUnprovenOutcome(*record) { + return ErrDeliveryOwnerMismatch + } + result.RetiredSubmission = record + } + if id == request.StoredSubmissionID { + result.StoredSubmission = record + } + } + if request.RetiredSubmissionID == "" && request.StoredSubmissionID == "" { + return nil + } + after, err := client.GetDeliveryStatus(ctx, "") + if err != nil { + return deliveryEvidenceReadError(lease, err) + } + if !sameNativeResumeProofStatus(&result.Descriptor, after) { + return ErrDeliveryRecordEvidenceStale + } + return nil +} + +func readLiveSubmissionMetadata(ctx context.Context, client *agentctl.Client, request DeliveryRecordEvidenceRequest, id string) (*journal.Submission, error) { + record, err := client.GetDeliverySubmission(ctx, id) + if err != nil { + return nil, err + } + if record == nil || record.SessionID != request.SessionID || record.IncarnationID != request.IncarnationID || record.HarnessGeneration == 0 || record.HarnessGeneration > request.HarnessGeneration { + return nil, ErrDeliveryOwnerMismatch + } + record.Payload = nil + return record, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery.go index a3f74752101..fbc8c80bd06 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery.go +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery.go @@ -40,10 +40,15 @@ func (m *Manager) AcknowledgeNativeResumeDelivery(ctx context.Context, sessionID if _, err := authenticatedDeliveryReplayCursor(ctx, execution, m.streamManager.deliveryRepository(), status); err != nil { return err } - if err := m.validateNativeResumeSQLHistory(ctx, execution, status, submissions, false); err != nil { + // Dispatch reconciliation can change SQL state; pin one fresh inventory for validation and retirement selection. + rows, err := activeBackendSubmissions(ctx, m.streamManager.deliveryRepository(), sessionID) + if err != nil { + return err + } + if err := validateNativeResumeSQLRows(execution, status, submissions, rows, false); err != nil { return err } - ids, err := m.nativeResumeInterruptedSubmissionIDs(ctx, execution, status, submissions) + ids, err := nativeResumeRetirementIDs(execution, status, submissions, rows) if err != nil { return err } @@ -83,14 +88,6 @@ func validateNativeResumeDeliveryOwner(execution *AgentExecution, status *agentc return nil } -func (m *Manager) nativeResumeInterruptedSubmissionIDs(ctx context.Context, execution *AgentExecution, status *agentctl.DeliveryStatus, submissions []journal.Submission) ([]string, error) { - rows, err := activeBackendSubmissions(ctx, m.streamManager.deliveryRepository(), execution.SessionID) - if err != nil { - return nil, err - } - return nativeResumeRetirementIDs(execution, status, submissions, rows) -} - func nativeResumeRetirementIDs(execution *AgentExecution, status *agentctl.DeliveryStatus, submissions []journal.Submission, rows []*models.AgentDeliverySubmission) ([]string, error) { var ids []string backend := make(map[string]*models.AgentDeliverySubmission, len(rows)) diff --git a/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery_test.go b/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery_test.go index 463d011c8ba..58a5570c72f 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/native_resume_delivery_test.go @@ -138,7 +138,7 @@ func TestAcknowledgedUnknownHistorySurvivesDurableAdoption(t *testing.T) { if id == "earlier" { generation = 1 } - _, err = store.PutSubmission(ctx, journal.Submission{ID: id, SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: generation, Hash: "hash", Payload: []byte("prompt"), State: journal.SubmissionInterruptedUnknown, Retired: true}) + _, err = store.PutSubmission(ctx, journal.Submission{ID: id, SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: generation, StreamID: "stream", Hash: "hash", Payload: []byte("prompt"), State: journal.SubmissionInterruptedUnknown, Retired: true}) require.NoError(t, err) repository.submissions[id] = &models.AgentDeliverySubmission{ID: id, SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: int64(generation), PayloadHash: "hash", State: models.DeliverySubmissionInterruptedUnknown} } diff --git a/apps/backend/internal/agentctl/journal/retained.go b/apps/backend/internal/agentctl/journal/retained.go index 85b2b12799e..3bdbbb97548 100644 --- a/apps/backend/internal/agentctl/journal/retained.go +++ b/apps/backend/internal/agentctl/journal/retained.go @@ -44,6 +44,7 @@ type RetainedRecoveryResponse struct { // set for one retained session. Process identity is optional because older // records do not always contain enough information to prove termination. type RetainedReconstructionEvidenceRequest struct { + StoredSubmissionID string `json:"stored_submission_id,omitempty"` RetiredSubmissionID string `json:"retired_submission_id,omitempty"` Root string `json:"root"` SessionID string `json:"session_id"` @@ -55,6 +56,7 @@ type RetainedReconstructionEvidenceRequest struct { // RetainedReconstructionEvidence keeps process proof tri-state: nil means // that termination could not be established from retained evidence. type RetainedReconstructionEvidence struct { + StoredSubmission *Submission `json:"stored_submission,omitempty"` RetiredSubmission *Submission `json:"retired_submission,omitempty"` Descriptor RecoveryDescriptor `json:"descriptor"` ProcessTerminated *bool `json:"process_terminated,omitempty"` @@ -98,13 +100,30 @@ func ReadRetainedReconstructionEvidence( if err != nil { return nil, err } - if !retired.Retired || retired.State != SubmissionInterruptedUnknown || retired.SessionID != request.SessionID || + if !retired.Retired || !HasUnprovenOutcome(retired) || retired.SessionID != request.SessionID || retired.IncarnationID != request.IncarnationID || retired.HarnessGeneration == 0 || retired.HarnessGeneration > request.HarnessGeneration { return nil, ErrOwnerMismatch } retired.Payload = nil result.RetiredSubmission = &retired } + if request.StoredSubmissionID != "" { + stored, err := journal.GetSubmission(ctx, request.StoredSubmissionID) + if err != nil { + return nil, err + } + if !storedSubmissionOwnerMatches(stored, request) { + return nil, ErrOwnerMismatch + } + stored.Payload = nil + result.StoredSubmission = &stored + if result.Descriptor.StreamID == "" && stored.HarnessGeneration == request.HarnessGeneration { + result.Descriptor, err = journal.RecoveryDescriptor(ctx, request.SessionID, request.IncarnationID, request.HarnessGeneration, stored.StreamID) + if err != nil { + return nil, err + } + } + } if request.OriginalRuntime != nil { if err := request.OriginalRuntime.Validate(); err == nil { if terminated, err := processidentity.OwnedSessionTerminated(*request.OriginalRuntime); err == nil { @@ -326,3 +345,7 @@ func openRetainedJournal(request RetainedRecoveryRequest, readOnly bool) (*Journ func retainedSubmissionMatches(submission Submission, request RetainedRecoveryRequest) bool { return submission.SessionID == request.SessionID && submission.IncarnationID == request.IncarnationID && submission.HarnessGeneration == request.HarnessGeneration && submission.StreamID == request.StreamID } + +func storedSubmissionOwnerMatches(stored Submission, request RetainedReconstructionEvidenceRequest) bool { + return stored.SessionID == request.SessionID && stored.IncarnationID == request.IncarnationID && stored.HarnessGeneration > 0 && stored.HarnessGeneration <= request.HarnessGeneration +} diff --git a/apps/backend/internal/agentctl/journal/stored_recovery_evidence_test.go b/apps/backend/internal/agentctl/journal/stored_recovery_evidence_test.go new file mode 100644 index 00000000000..ace31adfd07 --- /dev/null +++ b/apps/backend/internal/agentctl/journal/stored_recovery_evidence_test.go @@ -0,0 +1,62 @@ +package journal + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStoredRecoveryEvidencePreservesHistoricalLabels(t *testing.T) { + for _, state := range []SubmissionState{SubmissionFailed, SubmissionCompleted, SubmissionCancelled, SubmissionInterruptedUnknown} { + for _, retired := range []bool{false, true} { + name := string(state) + if retired { + name += " retired" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + path := CheckStorage(root, "session").Path + store, err := Open(Config{Path: path}) + require.NoError(t, err) + original := Submission{ID: "history", SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: 1, StreamID: "stream", Hash: "hash", Payload: []byte("must not replay"), State: state} + _, err = store.PutSubmission(ctx, original) + require.NoError(t, err) + if retired { + _, err = store.RetireSubmission(ctx, original.ID, 1) + require.NoError(t, err) + } + require.NoError(t, store.Close()) + before, err := os.ReadFile(path) + require.NoError(t, err) + request := RetainedReconstructionEvidenceRequest{Root: root, SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: 1, StoredSubmissionID: original.ID} + evidence, err := ReadRetainedReconstructionEvidence(ctx, request) + require.NoError(t, err) + require.NotNil(t, evidence.StoredSubmission) + require.Equal(t, state, evidence.StoredSubmission.State) + require.Equal(t, retired, evidence.StoredSubmission.Retired) + require.Empty(t, evidence.StoredSubmission.Payload) + require.Equal(t, original.StreamID, evidence.Descriptor.StreamID) + request.StoredSubmissionID = "" + request.RetiredSubmissionID = original.ID + proof, err := ReadRetainedReconstructionEvidence(ctx, request) + if retired { + require.NoError(t, err) + require.True(t, proof.RetiredSubmission.Retired) + } else { + require.ErrorIs(t, err, ErrOwnerMismatch) + } + request.RetiredSubmissionID = "" + request.StoredSubmissionID = original.ID + request.IncarnationID = "foreign" + _, err = ReadRetainedReconstructionEvidence(ctx, request) + require.ErrorIs(t, err, ErrOwnerMismatch) + after, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, before, after) + }) + } + } +} diff --git a/apps/backend/internal/agentctl/server/process/manager.go b/apps/backend/internal/agentctl/server/process/manager.go index b3c7500af68..3e1313c2578 100644 --- a/apps/backend/internal/agentctl/server/process/manager.go +++ b/apps/backend/internal/agentctl/server/process/manager.go @@ -2855,6 +2855,9 @@ func (m *Manager) stop(ctx context.Context) error { zap.Int("pid", m.agentPID()), zap.String("protocol", m.agentProtocol())) m.status.Store(StatusStopping) + if stopping := m.shutdownIntent.Load(); stopping != nil { + stopping.Store(true) + } auxiliaryStopErr := errors.Join(comparisonStopErr, m.stopShellAndProcesses(ctx)) m.closeAdapterAndStdin() diff --git a/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go b/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go index 561b192df29..6cad245c9bd 100644 --- a/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go +++ b/apps/backend/internal/agentctl/server/process/submission_shutdown_test.go @@ -2,7 +2,7 @@ package process import ( "context" - "errors" + "fmt" "path/filepath" "sync" "testing" @@ -18,6 +18,7 @@ import ( type shutdownRequestAdapter struct { adapter.AgentAdapter + ctx context.Context closing chan struct{} dispatchDone chan struct{} once sync.Once @@ -27,8 +28,8 @@ func (a *shutdownRequestAdapter) Close() error { a.once.Do(func() { close(a.closing) }) select { case <-a.dispatchDone: - case <-time.After(5 * time.Second): - return errors.New("durable dispatch did not settle during adapter shutdown") + case <-a.ctx.Done(): + return fmt.Errorf("durable dispatch did not settle during adapter shutdown: %w", a.ctx.Err()) } return a.AgentAdapter.Close() } @@ -43,7 +44,7 @@ func TestSubmissionGracefulStopPreservesUnknownRequestOutcome(t *testing.T) { }, newTestLogger(t)) require.NoError(t, mgr.Start(ctx)) t.Cleanup(func() { _ = mgr.Stop(context.Background()); _ = mgr.closeDeliveryJournal() }) - wrapped := &shutdownRequestAdapter{AgentAdapter: mgr.adapter, closing: make(chan struct{}), dispatchDone: make(chan struct{})} + wrapped := &shutdownRequestAdapter{ctx: ctx, AgentAdapter: mgr.adapter, closing: make(chan struct{}), dispatchDone: make(chan struct{})} mgr.adapter = wrapped original := journal.Submission{ID: "prompt:old", SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: 1, StreamID: "incarnation:g1", Payload: []byte("do not replay"), Hash: journal.SubmissionHash([]byte("do not replay"))} diff --git a/apps/backend/internal/orchestrator/legacy_native_restart.go b/apps/backend/internal/orchestrator/legacy_native_restart.go index a4c94f8f519..f86977943e5 100644 --- a/apps/backend/internal/orchestrator/legacy_native_restart.go +++ b/apps/backend/internal/orchestrator/legacy_native_restart.go @@ -38,7 +38,7 @@ func legacyNativeRestartEvidenceMatches(evidence *agentruntime.DeliveryRecordEvi if evidence == nil || evidence.ControlIdentity == nil || evidence.ProcessTerminated == nil || !*evidence.ProcessTerminated { return false } - d, peer, identity := evidence.Descriptor, evidence.RetiredSubmission, evidence.ControlIdentity + d, peer, identity := evidence.Descriptor, evidence.StoredSubmission, evidence.ControlIdentity if !legacyNativeDescriptorIdle(d) { return false } @@ -109,7 +109,7 @@ func readLegacyRestartInventory(ctx context.Context, client deliveryRecordEviden if row.State != models.DeliverySubmissionInterruptedUnknown { return nil, nil } - request := agentruntime.DeliveryRecordEvidenceRequest{TaskID: session.TaskID, SessionID: session.ID, IncarnationID: session.QueueIncarnationID, HarnessGeneration: uint64(generation.Generation), RetiredSubmissionID: row.ID} + request := agentruntime.DeliveryRecordEvidenceRequest{TaskID: session.TaskID, SessionID: session.ID, IncarnationID: session.QueueIncarnationID, HarnessGeneration: uint64(generation.Generation), StoredSubmissionID: row.ID} evidence, err := client.InspectDeliveryRecordEvidence(ctx, request) if err != nil || evidence == nil { return nil, err @@ -118,7 +118,7 @@ func readLegacyRestartInventory(ctx context.Context, client deliveryRecordEviden return nil, errors.New("legacy inventory changed") } descriptor = &evidence.Descriptor - if matchingRetiredRestartSubmission(inputs, row, evidence.RetiredSubmission) { + if matchingRetiredRestartSubmission(inputs, row, evidence.StoredSubmission) { continue } if selected != nil || row.TerminalSequence != 0 || !legacyNativeRestartEvidenceMatches(evidence, row, block) { diff --git a/apps/backend/internal/orchestrator/legacy_native_restart_test.go b/apps/backend/internal/orchestrator/legacy_native_restart_test.go index 5fb0068eead..afc05d87619 100644 --- a/apps/backend/internal/orchestrator/legacy_native_restart_test.go +++ b/apps/backend/internal/orchestrator/legacy_native_restart_test.go @@ -27,7 +27,7 @@ func (m *legacyNativeTestManager) ReadDeliveryRecordSubmission(context.Context, } func TestLegacyNativeRestartWithoutRecoveryNotice(t *testing.T) { - for _, condition := range []string{"dead owner", "live owner", "hash mismatch", "retained terminal", "changed owner", "independent work"} { + for _, condition := range []string{"dead owner", "queue", "retired history", "generic history", "unprojected output", "live owner", "hash mismatch", "retained terminal", "changed owner", "independent work"} { t.Run(condition, func(t *testing.T) { ctx := context.Background() service, manager, block, row, launched, _ := nativeResumeFixture(t, "unknown_prompt_outcome", false, false) @@ -36,20 +36,52 @@ func TestLegacyNativeRestartWithoutRecoveryNotice(t *testing.T) { require.NoError(t, repo.SetSessionMetadataKey(ctx, row.SessionID, models.SessionMetaKeyAgentDeliveryRecovery, nil)) _, err := repo.DB().ExecContext(ctx, `UPDATE session_recovery_blocks SET delivery_stream_id = '' WHERE id = ?`, block.ID) require.NoError(t, err) + if condition == "queue" { + _, err = repo.DB().ExecContext(ctx, `UPDATE agent_delivery_submissions SET id = 'queue:delivery' WHERE id = ?`, row.ID) + require.NoError(t, err) + row.ID = "queue:delivery" + block.DeliverySubmissionID = row.ID + _, err = repo.DB().ExecContext(ctx, `UPDATE session_recovery_blocks SET delivery_submission_id = ? WHERE id = ?`, row.ID, block.ID) + require.NoError(t, err) + } store, err := journal.Open(journal.Config{Path: filepath.Join(t.TempDir(), "journal.bbolt")}) require.NoError(t, err) t.Cleanup(func() { _ = store.Close() }) peer := journal.Submission{ID: row.ID, SessionID: row.SessionID, IncarnationID: row.IncarnationID, HarnessGeneration: 1, StreamID: block.DeliveryStreamID, Hash: row.PayloadHash, Payload: row.Payload, State: journal.SubmissionFailed} _, err = store.PutSubmission(ctx, peer) require.NoError(t, err) + if condition == "unprojected output" { + _, err = store.Append(ctx, journal.Event{SessionID: peer.SessionID, IncarnationID: peer.IncarnationID, HarnessGeneration: 1, StreamID: peer.StreamID, SubmissionID: peer.ID, Type: "tool_call", Payload: []byte("unprojected")}) + require.NoError(t, err) + } + if condition == "retired history" || condition == "generic history" { + older := *row + older.ID = "prior" + older.PayloadHash = "prior-hash" + _, err = repo.PrepareAgentDeliverySubmission(ctx, &older) + require.NoError(t, err) + retired := peer + retired.ID = older.ID + retired.Hash = older.PayloadHash + retired.Retired = true + _, err = store.PutSubmission(ctx, retired) + require.NoError(t, err) + if condition == "generic history" { + _, err = repo.DB().ExecContext(ctx, `UPDATE session_recovery_blocks SET delivery_submission_id = '', reason = 'unresolved_durable_work' WHERE id = ?`, block.ID) + require.NoError(t, err) + } + } calls := 0 identity := agentruntime.AgentDeliveryRecoveryIdentity{TaskID: "task1", SessionID: row.SessionID, ExecutionID: "old-execution", IncarnationID: row.IncarnationID, HarnessGeneration: 1, StreamID: peer.StreamID, SubmissionID: peer.ID, PromptGeneration: 1, OriginalRuntime: processidentity.Identity{PID: 42, GroupID: 42, SessionID: 42, BirthToken: "linux:boot:1"}} wrapper := &legacyNativeTestManager{nativeProofTestManager: &nativeProofTestManager{nativeResumeTestManager: manager}} - wrapper.inspect = func(ctx context.Context, _ agentruntime.DeliveryRecordEvidenceRequest) (*agentruntime.DeliveryRecordEvidence, error) { + wrapper.inspect = func(ctx context.Context, request agentruntime.DeliveryRecordEvidenceRequest) (*agentruntime.DeliveryRecordEvidence, error) { calls++ d, err := store.RecoveryDescriptor(ctx, row.SessionID, row.IncarnationID, 1, peer.StreamID) require.NoError(t, err) - p, err := store.GetSubmission(ctx, peer.ID) + if request.StoredSubmissionID == "" { + return &agentruntime.DeliveryRecordEvidence{Descriptor: d}, nil + } + p, err := store.GetSubmission(ctx, request.StoredSubmissionID) require.NoError(t, err) dead := condition != "live owner" binding := identity @@ -66,7 +98,7 @@ func TestLegacyNativeRestartWithoutRecoveryNotice(t *testing.T) { case "independent work": d.SubmissionCount = 1 } - return &agentruntime.DeliveryRecordEvidence{Descriptor: d, RetiredSubmission: &p, ControlIdentity: &binding, ProcessTerminated: &dead}, nil + return &agentruntime.DeliveryRecordEvidence{Descriptor: d, StoredSubmission: &p, ControlIdentity: &binding, ProcessTerminated: &dead}, nil } originalAck := manager.acknowledge manager.acknowledge = func(ctx context.Context, sessionID string) error { @@ -80,7 +112,11 @@ func TestLegacyNativeRestartWithoutRecoveryNotice(t *testing.T) { retired, err := store.GetSubmission(ctx, peer.ID) require.NoError(t, err) require.True(t, retired.Retired) - return &models.NativeResumeDeliveryEvidence{SessionID: sessionID, IncarnationID: row.IncarnationID, HarnessGeneration: 1, NativeSessionID: "native-conversation", ExecutionID: running.AgentExecutionID, StreamID: peer.StreamID, Submissions: []models.NativeResumeSubmissionEvidence{{ID: peer.ID, HarnessGeneration: 1, PayloadHash: peer.Hash, StreamID: peer.StreamID}}}, nil + proof := &models.NativeResumeDeliveryEvidence{SessionID: sessionID, IncarnationID: row.IncarnationID, HarnessGeneration: 1, NativeSessionID: "native-conversation", ExecutionID: running.AgentExecutionID, StreamID: peer.StreamID, Submissions: []models.NativeResumeSubmissionEvidence{{ID: peer.ID, HarnessGeneration: 1, PayloadHash: peer.Hash, StreamID: peer.StreamID}}} + if condition == "retired history" || condition == "generic history" { + proof.Submissions = append(proof.Submissions, models.NativeResumeSubmissionEvidence{ID: "prior", HarnessGeneration: 1, PayloadHash: "prior-hash", StreamID: peer.StreamID}) + } + return proof, nil } service.agentManager = wrapper session, err := repo.GetTaskSession(ctx, row.SessionID) @@ -91,7 +127,8 @@ func TestLegacyNativeRestartWithoutRecoveryNotice(t *testing.T) { } else { require.NoError(t, err) } - require.Equal(t, condition == "dead owner", restored) + want := condition == "dead owner" || condition == "queue" || condition == "retired history" || condition == "generic history" + require.Equal(t, want, restored) if restored { require.NotNil(t, *launched) saved, err := store.GetSubmission(ctx, peer.ID) diff --git a/apps/backend/internal/orchestrator/restart_submission_reconciliation.go b/apps/backend/internal/orchestrator/restart_submission_reconciliation.go index eb04598894a..a1b421b79c4 100644 --- a/apps/backend/internal/orchestrator/restart_submission_reconciliation.go +++ b/apps/backend/internal/orchestrator/restart_submission_reconciliation.go @@ -148,7 +148,7 @@ func (s *Service) verifyRetiredRestartHistory(ctx context.Context, inputs *silen } func matchingRetiredRestartSubmission(inputs *silentRestoreInputs, row *models.AgentDeliverySubmission, peer *journal.Submission) bool { - return peer != nil && peer.Retired && journal.HasUnprovenOutcome(*peer) && row.State == models.DeliverySubmissionInterruptedUnknown && row.TerminalSequence == 0 && peer.ID == row.ID && + return peer != nil && peer.Retired && journal.HasUnprovenOutcome(*peer) && row.State == models.DeliverySubmissionInterruptedUnknown && (peer.State == journal.SubmissionInterruptedUnknown || row.TerminalSequence == 0) && peer.ID == row.ID && peer.StreamID != "" && peer.SessionID == inputs.session.ID && row.SessionID == peer.SessionID && peer.IncarnationID == inputs.recovery.IncarnationID && row.IncarnationID == peer.IncarnationID && row.HarnessGeneration > 0 && row.HarnessGeneration <= inputs.recovery.HarnessGeneration && diff --git a/apps/backend/internal/orchestrator/session_delivery_reconstruction_test.go b/apps/backend/internal/orchestrator/session_delivery_reconstruction_test.go index efa21e7fd82..dbedf67b42b 100644 --- a/apps/backend/internal/orchestrator/session_delivery_reconstruction_test.go +++ b/apps/backend/internal/orchestrator/session_delivery_reconstruction_test.go @@ -30,12 +30,12 @@ func (m *retainedJournalRecoveryManager) InspectDeliveryRecordEvidence( m.inspectCalls++ retained, err := journal.ReadRetainedReconstructionEvidence(ctx, journal.RetainedReconstructionEvidenceRequest{ Root: m.root, SessionID: request.SessionID, IncarnationID: request.IncarnationID, - HarnessGeneration: request.HarnessGeneration, RetiredSubmissionID: request.RetiredSubmissionID, + HarnessGeneration: request.HarnessGeneration, RetiredSubmissionID: request.RetiredSubmissionID, StoredSubmissionID: request.StoredSubmissionID, }) if err != nil { return nil, err } - return &lifecycle.DeliveryRecordEvidence{Descriptor: retained.Descriptor, ControlIdentity: m.controlIdentity, RetiredSubmission: retained.RetiredSubmission}, nil + return &lifecycle.DeliveryRecordEvidence{Descriptor: retained.Descriptor, ControlIdentity: m.controlIdentity, RetiredSubmission: retained.RetiredSubmission, StoredSubmission: retained.StoredSubmission}, nil } func (m *retainedJournalRecoveryManager) ReadDeliveryRecordSubmission( diff --git a/apps/backend/internal/task/repository/sqlite/native_resume_attempt.go b/apps/backend/internal/task/repository/sqlite/native_resume_attempt.go new file mode 100644 index 00000000000..5ba30b92d28 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/native_resume_attempt.go @@ -0,0 +1,60 @@ +package sqlite + +import ( + "context" + "time" + + "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/task/models" +) + +func (r *Repository) finishPreparedNativeAttemptTx(ctx context.Context, tx *sqlx.Tx, proof *models.NativeResumeDeliveryEvidence, blocks []models.SessionRecoveryBlock, completedAt time.Time) (bool, error) { + var ids []string + if err := tx.SelectContext(ctx, &ids, r.db.Rebind(`SELECT id FROM session_restore_attempts WHERE session_id = ? AND completed_at IS NULL`), proof.SessionID); err != nil { + return false, err + } + if len(ids) == 0 { + return true, nil + } + if len(ids) != 1 { + return false, nil + } + attempt, encoded, err := readSilentRestoreAttemptCheckpointTx(ctx, tx, r.db.DriverName(), ids[0]) + if err != nil { + return false, err + } + if !preparedNativeAttemptMatches(attempt, proof, blocks) { + return false, nil + } + checkpoint := attempt.Checkpoint + var ownerMatches bool + err = tx.QueryRowxContext(ctx, r.db.Rebind(`SELECT EXISTS (SELECT 1 FROM task_sessions s JOIN tasks t ON t.id = s.task_id JOIN workspaces w ON w.id = t.workspace_id WHERE s.id = ? AND t.id = ? AND w.id = ? AND w.owner_id = ? AND w.org_id = ?)`), proof.SessionID, checkpoint.TaskID, checkpoint.WorkspaceID, checkpoint.WorkspaceOwnerID, checkpoint.WorkspaceOrgID).Scan(&ownerMatches) + if err != nil || !ownerMatches { + return false, err + } + return completeSilentRestoreAttemptTx(ctx, tx, r.db.Rebind, attempt.ID, attempt, encoded, *checkpoint, completedAt) +} + +func preparedNativeAttemptMatches(attempt *models.RestoreAttempt, proof *models.NativeResumeDeliveryEvidence, blocks []models.SessionRecoveryBlock) bool { + if attempt == nil || attempt.Checkpoint == nil || !attempt.Authorized || attempt.Action != models.SilentRestoreAction || attempt.Outcome != models.SilentRestoreOutcomePending || attempt.SessionID != proof.SessionID || attempt.IncarnationID != proof.IncarnationID || attempt.ExpectedGeneration != proof.HarnessGeneration { + return false + } + checkpoint := attempt.Checkpoint + if !preparedNativeCheckpointMatches(checkpoint, proof) { + return false + } + for _, block := range blocks { + if block.ID == checkpoint.SourceBlock.ID { + return true + } + } + return false +} + +func preparedNativeCheckpointMatches(checkpoint *models.SilentRestoreCheckpoint, proof *models.NativeResumeDeliveryEvidence) bool { + source := checkpoint.SourceGeneration + return checkpoint.Version == 1 && checkpoint.Stage == models.SilentRestoreStagePrepared && checkpoint.CandidateExecutionID == "" && + source.SessionID == proof.SessionID && source.IncarnationID == proof.IncarnationID && source.Generation == proof.HarnessGeneration && source.NativeSessionID == proof.NativeSessionID && + checkpoint.SourceRecovery.SessionID == proof.SessionID && checkpoint.SourceRecovery.IncarnationID == proof.IncarnationID && checkpoint.SourceRecovery.HarnessGeneration == proof.HarnessGeneration && + checkpoint.SourceRecovery.AgentExecutionID != "" && checkpoint.SourceRecovery.AgentExecutionID != proof.ExecutionID && nativeResumeProofCoversNotice(proof, checkpoint.SourceRecovery) +} diff --git a/apps/backend/internal/task/repository/sqlite/native_resume_binding.go b/apps/backend/internal/task/repository/sqlite/native_resume_binding.go index 6de03e9bf1a..9c0b629453c 100644 --- a/apps/backend/internal/task/repository/sqlite/native_resume_binding.go +++ b/apps/backend/internal/task/repository/sqlite/native_resume_binding.go @@ -11,6 +11,11 @@ import ( "github.com/kandev/kandev/internal/task/models" ) +const ( + deliveryUnknownPromptReason = "unknown_prompt_outcome" + deliveryUnresolvedWorkReason = "unresolved_durable_work" +) + // ResolveSessionRecoveryBlockWithNativeEvidence commits only after authenticated // native retirement. All incident fences are verified before any becomes usable. func (r *Repository) ResolveSessionRecoveryBlockWithNativeEvidence(ctx context.Context, id string, proof *models.NativeResumeDeliveryEvidence, completedAt time.Time) (bool, error) { @@ -22,10 +27,7 @@ func (r *Repository) ResolveSessionRecoveryBlockWithNativeEvidence(ctx context.C return false, err } defer func() { _ = tx.Rollback() }() - if err := lockSessionTurnWrites(ctx, tx, r.db.DriverName(), proof.SessionID); err != nil { - return false, err - } - if err := lockAgentDeliveryWrites(ctx, tx, r.db.DriverName(), []string{proof.SessionID}, []string{proof.StreamID}); err != nil { + if err := r.lockNativeResumeProofTx(ctx, tx, proof); err != nil { return false, err } valid, err := r.verifyNativeResumeSnapshotTx(ctx, tx, proof) @@ -36,7 +38,11 @@ func (r *Repository) ResolveSessionRecoveryBlockWithNativeEvidence(ctx context.C if err != nil || len(blocks) == 0 { return false, err } - valid, err = r.restoreAuthenticatedNativeNoticeTx(ctx, tx, proof, completedAt) + valid, err = r.finishPreparedNativeAttemptTx(ctx, tx, proof, blocks, completedAt) + if err != nil || !valid { + return false, err + } + valid, err = r.restoreAuthenticatedNativePresentationTx(ctx, tx, proof, completedAt) if err != nil || !valid { return false, err } @@ -58,7 +64,6 @@ func (r *Repository) verifyNativeResumeSnapshotTx(ctx context.Context, tx *sqlx. WHERE ts.id = ? AND ts.queue_incarnation_id = ? AND ts.state <> 'CANCELLED' AND g.generation = ? AND g.native_session_id = ? AND e.agent_execution_id = ? AND NOT EXISTS (SELECT 1 FROM harness_session_generations newer WHERE newer.session_id = ts.id AND newer.incarnation_id = g.incarnation_id AND newer.generation > g.generation) - AND NOT EXISTS (SELECT 1 FROM session_restore_attempts WHERE session_id = ts.id AND completed_at IS NULL) AND NOT EXISTS (SELECT 1 FROM agent_delivery_cursors WHERE session_id = ts.id AND incarnation_id = g.incarnation_id AND harness_generation = g.generation AND received_sequence <> projected_sequence) )`), proof.SessionID, proof.IncarnationID, proof.HarnessGeneration, proof.NativeSessionID, proof.ExecutionID).Scan(&valid) if err != nil || !valid { @@ -147,7 +152,7 @@ func nativeResumeIncidentBlockMatches(block models.SessionRecoveryBlock, proof * return false } if block.DeliverySubmissionID == "" { - return block.Reason == "unresolved_durable_work" && block.DeliveryStreamID == "" + return block.Reason == deliveryUnresolvedWorkReason && block.DeliveryStreamID == "" } if block.DeliveryStreamID != "" && block.DeliveryStreamID != proof.StreamID { return false @@ -249,8 +254,8 @@ func nativeResumeSubmissionMap(proof *models.NativeResumeDeliveryEvidence) (map[ func nativeResumeIncidentOwnerMatches(block models.SessionRecoveryBlock, proof *models.NativeResumeDeliveryEvidence) bool { if block.IncarnationID != proof.IncarnationID || block.ExpectedGeneration != proof.HarnessGeneration || block.ConsumerReference != reconstructionDeliveryConsumer || - (block.Reason != "unknown_prompt_outcome" && block.Reason != "unresolved_durable_work") || - block.DeliverySequence != 0 || block.DeliveryTurnID != "" || (block.DeliveryOutcome != "" && block.DeliveryOutcome != "unknown") { + (block.Reason != deliveryUnknownPromptReason && block.Reason != deliveryUnresolvedWorkReason) || + block.DeliverySequence != 0 || block.DeliveryTurnID != "" || (block.DeliveryOutcome != "" && block.DeliveryOutcome != reconstructionUnknownOutcome) { return false } return true @@ -265,8 +270,15 @@ func nativeResumeNoticeMatches(proof *models.NativeResumeDeliveryEvidence, recov for _, row := range proof.Submissions { found = found || row.ID == recovery.SubmissionID && row.HarnessGeneration == recovery.HarnessGeneration } - if !found { - return false + return found +} + +func (r *Repository) lockNativeResumeProofTx(ctx context.Context, tx *sqlx.Tx, proof *models.NativeResumeDeliveryEvidence) error { + if err := lockSessionTurnWrites(ctx, tx, r.db.DriverName(), proof.SessionID); err != nil { + return err } - return true + if err := lockAgentDeliveryWrites(ctx, tx, r.db.DriverName(), []string{proof.SessionID}, []string{proof.StreamID}); err != nil { + return err + } + return nil } diff --git a/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go b/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go index d5e11ae74e6..f50f91b4644 100644 --- a/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go +++ b/apps/backend/internal/task/repository/sqlite/native_resume_binding_test.go @@ -29,7 +29,7 @@ func TestPostgresNativeResumeAuthenticatedMissingBinding(t *testing.T) { } func testNativeResumeAuthenticatedMissingBinding(t *testing.T, open func(*testing.T) *Repository) { - for _, scenario := range []string{"absent notice", "partial notice", "resolved earlier notice", "matching fences", "independent fence", "foreign native", "hash changed", "active work", "unmatched unknown", "foreign stream", "foreign submission stream", "missing submission stream", "reconstructed unknown", "known outcome", "rollback"} { + for _, scenario := range []string{"absent notice", "partial notice", "resolved earlier notice", "matching fences", "independent fence", "foreign native", "hash changed", "active work", "unmatched unknown", "foreign stream", "foreign submission stream", "missing submission stream", "reconstructed unknown", "known outcome", "prepared attempt", "independent attempt", "legacy error", "independent error", "rollback"} { t.Run(scenario, func(t *testing.T) { repo := open(t) ctx := context.Background() @@ -73,6 +73,13 @@ func testNativeResumeAuthenticatedMissingBinding(t *testing.T, open func(*testin partial.StreamID = "" metadata[models.SessionMetaKeyAgentDeliveryRecovery] = partial } + if scenario == "legacy error" || scenario == "independent error" { + details := submission.ID + if scenario == "independent error" { + details = "independent" + } + metadata[models.SessionMetaKeyLastAgentError] = models.LastAgentError{Code: durableDeliveryUncertainErrorCode, Message: "uncertain", Details: details, AgentExecutionID: "original"} + } raw, err := json.Marshal(metadata) require.NoError(t, err) _, err = repo.DB().ExecContext(ctx, repo.db.Rebind(`UPDATE task_sessions SET metadata = ? WHERE id = ?`), string(raw), fixture.session.ID) @@ -100,6 +107,15 @@ func testNativeResumeAuthenticatedMissingBinding(t *testing.T, open func(*testin proof.Submissions[0].StreamID = "foreign" case "missing submission stream": proof.Submissions[0].StreamID = "" + case "prepared attempt", "independent attempt": + attempt := fixture.attempt("pending-native-attempt") + attempt.Authorized = true + attempt.Checkpoint.SourceRecovery.AgentExecutionID = "original-owner" + if scenario == "independent attempt" { + attempt.Checkpoint.Stage = models.SilentRestoreStageCandidateLaunching + attempt.Checkpoint.CandidateExecutionID = "independent" + } + require.NoError(t, repo.CreateRestoreAttempt(ctx, attempt)) case "foreign native": proof.NativeSessionID = "foreign" case "hash changed": @@ -131,8 +147,13 @@ func testNativeResumeAuthenticatedMissingBinding(t *testing.T, open func(*testin } else { require.NoError(t, err) } - want := scenario == "reconstructed unknown" || scenario == "absent notice" || scenario == "partial notice" || scenario == "resolved earlier notice" || scenario == "matching fences" + want := scenario == "legacy error" || scenario == "independent error" || scenario == "prepared attempt" || scenario == "reconstructed unknown" || scenario == "absent notice" || scenario == "partial notice" || scenario == "resolved earlier notice" || scenario == "matching fences" require.Equal(t, want, changed) + if scenario == "prepared attempt" || scenario == "independent attempt" { + attempt, err := repo.GetRestoreAttempt(ctx, "pending-native-attempt") + require.NoError(t, err) + require.Equal(t, scenario == "prepared attempt", attempt.CompletedAt != nil) + } block, err := repo.GetSessionRecoveryBlock(ctx, fixture.block.ID) require.NoError(t, err) if want { @@ -149,6 +170,11 @@ func testNativeResumeAuthenticatedMissingBinding(t *testing.T, open func(*testin require.Empty(t, block.DeliveryStreamID) } } + if scenario == "legacy error" || scenario == "independent error" { + current, err := repo.GetTaskSession(ctx, proof.SessionID) + require.NoError(t, err) + require.Equal(t, scenario == "legacy error", current.Metadata[models.SessionMetaKeyLastAgentError] == nil) + } stored, err := repo.GetAgentDeliverySubmission(ctx, submission.ID) require.NoError(t, err) require.Equal(t, submission.State, stored.State) diff --git a/apps/backend/internal/task/repository/sqlite/native_resume_error.go b/apps/backend/internal/task/repository/sqlite/native_resume_error.go new file mode 100644 index 00000000000..3f3ffb4b825 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/native_resume_error.go @@ -0,0 +1,74 @@ +package sqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "time" + + "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/task/models" +) + +func (r *Repository) restoreAuthenticatedNativePresentationTx(ctx context.Context, tx *sqlx.Tx, proof *models.NativeResumeDeliveryEvidence, completedAt time.Time) (bool, error) { + valid, err := r.restoreAuthenticatedNativeNoticeTx(ctx, tx, proof, completedAt) + if err != nil || !valid { + return false, err + } + return true, r.clearAuthenticatedNativeErrorTx(ctx, tx, proof, completedAt) +} + +func (r *Repository) clearAuthenticatedNativeErrorTx(ctx context.Context, tx *sqlx.Tx, proof *models.NativeResumeDeliveryEvidence, completedAt time.Time) error { + var raw sql.NullString + if err := tx.QueryRowxContext(ctx, r.db.Rebind(`SELECT metadata FROM task_sessions WHERE id = ?`), proof.SessionID).Scan(&raw); err != nil { + return err + } + metadata, err := decodeAgentDeliveryMetadata(raw) + if err != nil { + return err + } + if !nativeErrorMatchesRetiredSubmission(metadata[models.SessionMetaKeyLastAgentError], proof) { + return nil + } + // Explicit null also clears a connected client's legacy error merge when + // no recovery notice exists to carry a newer restored revision. + metadata[models.SessionMetaKeyLastAgentError] = json.RawMessage("null") + encoded, err := json.Marshal(metadata) + if err != nil { + return err + } + result, err := tx.ExecContext(ctx, r.db.Rebind(`UPDATE task_sessions SET metadata = ?, updated_at = ? WHERE id = ? AND COALESCE(metadata, '') = ?`), string(encoded), completedAt, proof.SessionID, nullableMetadataValue(raw)) + if err != nil { + return err + } + count, err := result.RowsAffected() + if err != nil { + return err + } + if count != 1 { + return errors.New("native recovery error metadata changed") + } + return nil +} + +func nativeErrorMatchesRetiredSubmission(raw json.RawMessage, proof *models.NativeResumeDeliveryEvidence) bool { + var record map[string]any + if len(raw) == 0 || json.Unmarshal(raw, &record) != nil { + return false + } + last, exists := models.LoadLastAgentError(map[string]any{models.SessionMetaKeyLastAgentError: record}) + if !exists || last.Code != durableDeliveryUncertainErrorCode { + return false + } + submissionID := last.Details + if structured, present := record["delivery_submission_id"]; present { + submissionID, _ = structured.(string) + } + for _, row := range proof.Submissions { + if row.ID == submissionID { + return true + } + } + return false +} diff --git a/docs/plans/session-recovery-convergence/task-02-native-convergence.md b/docs/plans/session-recovery-convergence/task-02-native-convergence.md index 0b2cbf49701..bd30e809521 100644 --- a/docs/plans/session-recovery-convergence/task-02-native-convergence.md +++ b/docs/plans/session-recovery-convergence/task-02-native-convergence.md @@ -48,7 +48,11 @@ PostgreSQL variants run in CI against real PostgreSQL. The 49-pair lifecycle matrix and SQLite authenticated-binding suite pass. Foreign/missing submission streams and reconstructed unknown outcomes were -reproduced before correction. Automatic entrypoint checks and final CI are -in progress. +reproduced before correction. Automatic loaded-successor and legacy restart checks pass, including queue-origin +and retired-history cases. Prepared checkpoint and stale composer-error SQL +controls pass. CI exposed the distinction between stored-record inspection and +retirement proof; the retained-reader correction passes its focused journal test, +with the affected established restart/gap regressions being rechecked. Final CI +is pending. [Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/specs/platform/system-design/session-recovery-convergence.md b/docs/specs/platform/system-design/session-recovery-convergence.md index fdd4ead4009..46dfb10b4ad 100644 --- a/docs/specs/platform/system-design/session-recovery-convergence.md +++ b/docs/specs/platform/system-design/session-recovery-convergence.md @@ -64,7 +64,13 @@ Absent metadata and an earlier already-resolved notice are presentation states, not new outcome authority. Restore the current notice and resolve only matching incident fences atomically. A changed source, independent fence, pending restore or unprojected receipt refuses resolution. A transaction rollback changes none -of the binding, notice or fence. A retry reuses retired journal proof. +of the binding, notice or fence. A retry reuses retired journal proof. Retained inspection has a separate +metadata-only stored-submission request for unretired legacy history; retirement +proof still requires the actual retired marker and preserves the original label. +A matching prepared checkpoint with no candidate is completed in that transaction; +an allocated/launching or unrelated attempt still blocks it. Legacy composer +uncertainty is cleared only for an exactly retired submission. An explicit null +propagates that clearing to clients even when no recovery notice existed. ## Entry points From d750c57189effc0235d6c4e31d0037fcb0860642 Mon Sep 17 00:00:00 2001 From: Kandev Agent Date: Mon, 12 Oct 2026 03:43:04 +0100 Subject: [PATCH 4/7] fix: simplify retained recovery inspection --- .../internal/agentctl/journal/retained.go | 38 +++++++++++-------- .../session-recovery-convergence/plan.md | 13 ++++--- .../task-02-native-convergence.md | 4 +- .../task-03-recovery-contract.md | 6 ++- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/apps/backend/internal/agentctl/journal/retained.go b/apps/backend/internal/agentctl/journal/retained.go index 3bdbbb97548..353347639a6 100644 --- a/apps/backend/internal/agentctl/journal/retained.go +++ b/apps/backend/internal/agentctl/journal/retained.go @@ -107,22 +107,8 @@ func ReadRetainedReconstructionEvidence( retired.Payload = nil result.RetiredSubmission = &retired } - if request.StoredSubmissionID != "" { - stored, err := journal.GetSubmission(ctx, request.StoredSubmissionID) - if err != nil { - return nil, err - } - if !storedSubmissionOwnerMatches(stored, request) { - return nil, ErrOwnerMismatch - } - stored.Payload = nil - result.StoredSubmission = &stored - if result.Descriptor.StreamID == "" && stored.HarnessGeneration == request.HarnessGeneration { - result.Descriptor, err = journal.RecoveryDescriptor(ctx, request.SessionID, request.IncarnationID, request.HarnessGeneration, stored.StreamID) - if err != nil { - return nil, err - } - } + if err := journal.readStoredReconstructionEvidence(ctx, request, result); err != nil { + return nil, err } if request.OriginalRuntime != nil { if err := request.OriginalRuntime.Validate(); err == nil { @@ -134,6 +120,26 @@ func ReadRetainedReconstructionEvidence( return result, nil } +func (j *Journal) readStoredReconstructionEvidence(ctx context.Context, request RetainedReconstructionEvidenceRequest, result *RetainedReconstructionEvidence) error { + if request.StoredSubmissionID == "" { + return nil + } + stored, err := j.GetSubmission(ctx, request.StoredSubmissionID) + if err != nil { + return err + } + if !storedSubmissionOwnerMatches(stored, request) { + return ErrOwnerMismatch + } + stored.Payload = nil + result.StoredSubmission = &stored + if result.Descriptor.StreamID != "" || stored.HarnessGeneration != request.HarnessGeneration { + return nil + } + result.Descriptor, err = j.RecoveryDescriptor(ctx, request.SessionID, request.IncarnationID, request.HarnessGeneration, stored.StreamID) + return err +} + // ReadRetainedReconstructionSubmission returns the payload only when the // caller's summary still names the sole unresolved candidate. func ReadRetainedReconstructionSubmission( diff --git a/docs/plans/session-recovery-convergence/plan.md b/docs/plans/session-recovery-convergence/plan.md index a51f45c032a..163c0a611d4 100644 --- a/docs/plans/session-recovery-convergence/plan.md +++ b/docs/plans/session-recovery-convergence/plan.md @@ -42,7 +42,7 @@ remain blocked with a specific supported action; they are not fabricated away. ## Validation and delivery -Focused Go tests use GOMEMLIMIT=768MiB GOMAXPROCS=2, -p 1, -race, -trimpath, +Focused Go tests use GOMEMLIMIT=2048MiB GOMAXPROCS=2, -p 1, -race, -trimpath, -tags fts5; only one heavy local operation runs at once. Required PostgreSQL and all desktop/phone browser suites run in CI. Push and open a follow-up PR promptly, address every actionable CI/review finding, and merge the actual passing head. @@ -50,8 +50,11 @@ No completion claim follows merely from recovering the four observed records. ## Results -PR #4432 is open. Focused producer, SQL and lifecycle race checks pass, including -the 49-state-pair matrix. CI identified documentation metadata and function -complexity failures; corrections and review regressions are in progress. -Automatic restart/admission closure and final CI remain pending. No local E2E +PR #4432 implements all three work orders. Focused producer, journal, SQL, +lifecycle and orchestrator race checks pass, including the 49-state-pair matrix, +legacy shutdown history, prepared checkpoints, missing/stale fences and repeated +recovery. The four established recovery regressions reported by CI now pass +after separating stored-record inspection from retired-record proof. +All eight initial review threads were addressed and resolved. Final-head CI, +any subsequent review findings and merge remain the delivery gate. No local E2E has run. diff --git a/docs/plans/session-recovery-convergence/task-02-native-convergence.md b/docs/plans/session-recovery-convergence/task-02-native-convergence.md index bd30e809521..26e15d4b013 100644 --- a/docs/plans/session-recovery-convergence/task-02-native-convergence.md +++ b/docs/plans/session-recovery-convergence/task-02-native-convergence.md @@ -52,7 +52,7 @@ reproduced before correction. Automatic loaded-successor and legacy restart chec and retired-history cases. Prepared checkpoint and stale composer-error SQL controls pass. CI exposed the distinction between stored-record inspection and retirement proof; the retained-reader correction passes its focused journal test, -with the affected established restart/gap regressions being rechecked. Final CI -is pending. +and the affected established restart/gap regressions pass with race detection. +Final CI is pending. [Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/plans/session-recovery-convergence/task-03-recovery-contract.md b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md index e20ee5eff61..9226863a10f 100644 --- a/docs/plans/session-recovery-convergence/task-03-recovery-contract.md +++ b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md @@ -49,7 +49,9 @@ Desktop and phone E2E are CI-only; never run locally. PR #4432 is open and running CI. Documentation coverage passes locally after correcting work-order metadata. Review remediation covers terminal drain ordering, complete stream snapshot validation, exact submission stream binding and unknown -reconstruction outcomes. Final automatic restart checks, CI and review remain -in progress. No local E2E has run. +reconstruction outcomes. Focused automatic restart and repeated recovery checks pass. CI-reported +recovery fixtures pass after correcting metadata-only retained inspection. +All eight initial review findings are resolved; final-head CI and any new +review findings remain the delivery gate. No local E2E has run. [Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). From 1e4c3ee44a5aa974d2f5cf896802fecbdd131736 Mon Sep 17 00:00:00 2001 From: Kandev Agent Date: Mon, 12 Oct 2026 04:36:49 +0100 Subject: [PATCH 5/7] fix: recover canonical internal prompts after restart --- .../reconstruction_message_authority.go | 30 ++++++++++++++++ .../restart_reconciliation_test.go | 34 ++++++++++++------ ...ession_delivery_reconstruction_evidence.go | 17 +++++---- .../task/models/session_continuity.go | 35 ++++++++++--------- .../sqlite/agent_delivery_reconstruction.go | 10 ++++-- .../sqlite/restart_reconstruction_test.go | 25 +++++++++++-- .../provider-interruption-continuation.ts | 6 ++-- ...provider-interruption-continuation.spec.ts | 16 +++++++-- .../task-03-recovery-contract.md | 8 +++-- .../session-recovery-convergence.md | 6 ++++ 10 files changed, 143 insertions(+), 44 deletions(-) create mode 100644 apps/backend/internal/orchestrator/reconstruction_message_authority.go diff --git a/apps/backend/internal/orchestrator/reconstruction_message_authority.go b/apps/backend/internal/orchestrator/reconstruction_message_authority.go new file mode 100644 index 00000000000..41f03943a68 --- /dev/null +++ b/apps/backend/internal/orchestrator/reconstruction_message_authority.go @@ -0,0 +1,30 @@ +package orchestrator + +import ( + "context" + "database/sql" + "errors" + + "github.com/kandev/kandev/internal/agentctl/journal" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository" +) + +func (s *Service) reconstructionMessageAuthority(ctx context.Context, session *models.TaskSession, candidate journal.SubmissionSummary, messageID string) (bool, string, error) { + store, ok := s.repo.(repository.AgentDeliveryRepository) + if !ok { + return false, recoveryStateUnavailableReason, nil + } + canonical, err := store.GetAgentDeliverySubmission(ctx, candidate.ID) + if errors.Is(err, sql.ErrNoRows) || errors.Is(err, repository.ErrAgentDeliverySubmissionNotFound) { + reason, messageErr := s.validateReconstructionMessage(ctx, session, messageID) + return false, reason, messageErr + } + if err != nil { + return false, "", err + } + if canonical == nil || canonical.SessionID != session.ID || canonical.IncarnationID != candidate.IncarnationID || canonical.HarnessGeneration != int64(candidate.HarnessGeneration) || canonical.DispatchAttemptID != messageID || canonical.PayloadHash != candidate.Hash { + return false, deliveryIdentityMismatchReason, nil + } + return true, "", nil +} diff --git a/apps/backend/internal/orchestrator/restart_reconciliation_test.go b/apps/backend/internal/orchestrator/restart_reconciliation_test.go index b1d2587a348..fbe76bdcafc 100644 --- a/apps/backend/internal/orchestrator/restart_reconciliation_test.go +++ b/apps/backend/internal/orchestrator/restart_reconciliation_test.go @@ -132,22 +132,26 @@ func (m *alreadyResumedRestartManager) ReconcileAlreadyResumedDelivery(ctx conte // @covers AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-001.2 func TestRestartRecoveryUsesAlreadyLoadedNativeConversation(t *testing.T) { - testAlreadyLoadedRestartRecovery(t, false, false, false) + testAlreadyLoadedRestartRecovery(t, false, false, false, false) } func TestRestartRecoveryRepairsBothDispatchingWithoutMetadata(t *testing.T) { - testAlreadyLoadedRestartRecovery(t, true, false, false) + testAlreadyLoadedRestartRecovery(t, true, false, false, false) } func TestAlreadyLoadedRestartRecoveryPreservesIndependentBlock(t *testing.T) { - testAlreadyLoadedRestartRecovery(t, false, true, false) + testAlreadyLoadedRestartRecovery(t, false, true, false, false) } func TestRestartRecoveryRepairsIncompleteBlockWithKnownOwner(t *testing.T) { - testAlreadyLoadedRestartRecovery(t, false, false, true) + testAlreadyLoadedRestartRecovery(t, false, false, true, false) } -func testAlreadyLoadedRestartRecovery(t *testing.T, missingMetadata, independentBlock, partialBlock bool) { +func TestRestartRecoveryRepairsCanonicalContinuationWithoutUserMessage(t *testing.T) { + testAlreadyLoadedRestartRecovery(t, true, false, false, true) +} + +func testAlreadyLoadedRestartRecovery(t *testing.T, missingMetadata, independentBlock, partialBlock, continuation bool) { service, repo, base, session, store := newSilentRestoreEligibilityFixture(t, "native", "loaded") ctx := context.Background() root := t.TempDir() @@ -157,11 +161,17 @@ func testAlreadyLoadedRestartRecovery(t *testing.T, missingMetadata, independent submissionID := "loaded-submission" state := journal.SubmissionInterruptedUnknown if missingMetadata { - submissionID, state = "prompt:crash-message", journal.SubmissionDispatching + messageID := "crash-message" + if continuation { + messageID = "continuation:crash" + } + submissionID, state = "prompt:"+messageID, journal.SubmissionDispatching require.NoError(t, repo.CreateTurn(ctx, &models.Turn{ID: "crash-turn", TaskID: session.TaskID, TaskSessionID: session.ID, StartedAt: time.Now().UTC()})) - require.NoError(t, repo.CreateMessage(ctx, &models.Message{ID: "crash-message", TaskID: session.TaskID, TaskSessionID: session.ID, - TurnID: "crash-turn", AuthorType: models.MessageAuthorUser, Content: "original prompt"})) - _, err = repo.DB().ExecContext(ctx, `UPDATE agent_delivery_submissions SET id = ?, dispatch_attempt_id = 'crash-message', state = 'dispatching' WHERE id = ?`, submissionID, "loaded-submission") + if !continuation { + require.NoError(t, repo.CreateMessage(ctx, &models.Message{ID: messageID, TaskID: session.TaskID, TaskSessionID: session.ID, + TurnID: "crash-turn", AuthorType: models.MessageAuthorUser, Content: "original prompt"})) + } + _, err = repo.DB().ExecContext(ctx, `UPDATE agent_delivery_submissions SET id = ?, dispatch_attempt_id = ?, state = 'dispatching' WHERE id = ?`, submissionID, messageID, "loaded-submission") require.NoError(t, err) _, err = repo.DB().ExecContext(ctx, `UPDATE task_sessions SET metadata = '{}' WHERE id = ?`, session.ID) require.NoError(t, err) @@ -217,7 +227,11 @@ func testAlreadyLoadedRestartRecovery(t *testing.T, missingMetadata, independent require.NoError(t, err) require.Equal(t, models.DeliverySubmissionInterruptedUnknown, canonical.State) require.Equal(t, payload, canonical.Payload) - require.Empty(t, canonical.Outcome) + if missingMetadata { + require.Equal(t, "unknown", canonical.Outcome) + } else { + require.Empty(t, canonical.Outcome) + } retiredStore, err := journal.Open(journal.Config{Path: journal.CheckStorage(root, session.ID).Path}) require.NoError(t, err) retired, err := retiredStore.GetSubmission(ctx, submissionID) diff --git a/apps/backend/internal/orchestrator/session_delivery_reconstruction_evidence.go b/apps/backend/internal/orchestrator/session_delivery_reconstruction_evidence.go index 01d5c6b683e..afb08f87bdc 100644 --- a/apps/backend/internal/orchestrator/session_delivery_reconstruction_evidence.go +++ b/apps/backend/internal/orchestrator/session_delivery_reconstruction_evidence.go @@ -11,10 +11,11 @@ import ( ) type reconstructionSelection struct { - evidence *agentruntime.DeliveryRecordEvidence - submission *journal.Submission - stream *journal.Stream - messageID string + evidence *agentruntime.DeliveryRecordEvidence + submission *journal.Submission + stream *journal.Stream + messageID string + existingOnly bool } func (s *Service) readReconstructionSelection(ctx context.Context, client deliveryRecordEvidenceClient, session *models.TaskSession, incarnationID string, generation int64, block *models.SessionRecoveryBlock) (*reconstructionSelection, string, error) { @@ -34,7 +35,8 @@ func (s *Service) readReconstructionSelection(ctx context.Context, client delive if !ok { return blocked(deliveryIdentityMismatchReason) } - if reason, err := s.validateReconstructionMessage(ctx, session, messageID); err != nil { + existingOnly, reason, err := s.reconstructionMessageAuthority(ctx, session, candidate, messageID) + if err != nil { return nil, "", err } else if reason != "" { return blocked(reason) @@ -49,7 +51,7 @@ func (s *Service) readReconstructionSelection(ctx context.Context, client delive if selected == nil || journal.VerifyReconstructionSubmission(*selected, candidate) != nil { return blocked(deliveryIdentityMismatchReason) } - return &reconstructionSelection{evidence: evidence, submission: selected, stream: stream, messageID: messageID}, "", nil + return &reconstructionSelection{evidence: evidence, submission: selected, stream: stream, messageID: messageID, existingOnly: existingOnly}, "", nil } func (s *Service) prepareReconstructionRequest(ctx context.Context, session *models.TaskSession, incarnationID string, generation int64, block *models.SessionRecoveryBlock, selection *reconstructionSelection) (*models.AgentDeliveryReconstructionRequest, string, error) { @@ -86,7 +88,8 @@ func (s *Service) prepareReconstructionRequest(ctx context.Context, session *mod payloadHash := selected.Hash state := models.DeliverySubmissionInterruptedUnknown request := &models.AgentDeliveryReconstructionRequest{ - TaskID: session.TaskID, SessionID: session.ID, IncarnationID: incarnationID, + ExistingSubmissionOnly: selection.existingOnly, + TaskID: session.TaskID, SessionID: session.ID, IncarnationID: incarnationID, HarnessGeneration: generation, ExpectedExecutionID: session.AgentExecutionID, ExpectedSessionState: session.State, ExpectedWorkspacePath: session.WorkspacePath, NativeSessionID: current.NativeSessionID, MessageID: messageID, ExpectedBlock: *block, diff --git a/apps/backend/internal/task/models/session_continuity.go b/apps/backend/internal/task/models/session_continuity.go index 3ce1c773565..4ae2c5b86d4 100644 --- a/apps/backend/internal/task/models/session_continuity.go +++ b/apps/backend/internal/task/models/session_continuity.go @@ -73,22 +73,25 @@ type AgentDeliveryReconstructionRequest struct { // Retirement evidence is verified through the authenticated runtime before // these immutable canonical snapshots are checked under the SQL write lock. VerifiedRetiredSubmissions []AgentDeliverySubmission - TaskID string - SessionID string - IncarnationID string - HarnessGeneration int64 - ExpectedExecutionID string - ExpectedSessionState TaskSessionState - ExpectedWorkspacePath string - NativeSessionID string - MessageID string - ExpectedBlock SessionRecoveryBlock - SourceStreamHighWater int64 - SourceStreamAcknowledged int64 - SourceStreamFirstRetained int64 - ExpectedCursor *AgentDeliveryCursor - Submission AgentDeliverySubmission - Recovery AgentDeliveryRecovery + // ExistingSubmissionOnly repairs presentation from a canonical SQL row and + // cannot insert a missing submission or infer a user-message association. + ExistingSubmissionOnly bool + TaskID string + SessionID string + IncarnationID string + HarnessGeneration int64 + ExpectedExecutionID string + ExpectedSessionState TaskSessionState + ExpectedWorkspacePath string + NativeSessionID string + MessageID string + ExpectedBlock SessionRecoveryBlock + SourceStreamHighWater int64 + SourceStreamAcknowledged int64 + SourceStreamFirstRetained int64 + ExpectedCursor *AgentDeliveryCursor + Submission AgentDeliverySubmission + Recovery AgentDeliveryRecovery } // AgentDeliveryReconstructionResult reports the committed recovery identity. diff --git a/apps/backend/internal/task/repository/sqlite/agent_delivery_reconstruction.go b/apps/backend/internal/task/repository/sqlite/agent_delivery_reconstruction.go index 693bafe295f..c526b39fb50 100644 --- a/apps/backend/internal/task/repository/sqlite/agent_delivery_reconstruction.go +++ b/apps/backend/internal/task/repository/sqlite/agent_delivery_reconstruction.go @@ -52,6 +52,9 @@ func (r *Repository) ReconstructAgentDeliverySubmission( if existing, exists := rows[request.Submission.ID]; exists { return r.reconcileExistingReconstructionTx(ctx, tx, owner, existing, block, request) } + if request.ExistingSubmissionOnly { + return nil, repoerrors.ErrAgentDeliveryReconstructionStale + } if owner.exists && !owner.historical { return nil, repoerrors.ErrAgentDeliveryReconstructionConflict } @@ -272,6 +275,9 @@ func (r *Repository) validateReconstructionMessageTx( tx *sqlx.Tx, request *models.AgentDeliveryReconstructionRequest, ) error { + if request.ExistingSubmissionOnly { + return nil + } query := `SELECT task_id, task_session_id, author_type FROM task_session_messages WHERE id = ?` if dialect.IsPostgres(r.db.DriverName()) { query += ` FOR KEY SHARE` @@ -584,8 +590,8 @@ func (r *Repository) reconstructExistingDispatchTx(ctx context.Context, tx *sqlx if proof == nil || !*proof { return nil, repoerrors.ErrAgentDeliveryReconstructionConflict } - result, err := tx.ExecContext(ctx, r.db.Rebind(`UPDATE agent_delivery_submissions SET state = ? WHERE id = ? AND state = ?`), - models.DeliverySubmissionInterruptedUnknown, existing.ID, existing.State) + result, err := tx.ExecContext(ctx, r.db.Rebind(`UPDATE agent_delivery_submissions SET state = ?, outcome = CASE WHEN outcome = '' THEN ? ELSE outcome END, updated_at = ? WHERE id = ? AND state = ?`), + models.DeliverySubmissionInterruptedUnknown, reconstructionUnknownOutcome, r.nowUTC(), existing.ID, existing.State) if err != nil { return nil, err } diff --git a/apps/backend/internal/task/repository/sqlite/restart_reconstruction_test.go b/apps/backend/internal/task/repository/sqlite/restart_reconstruction_test.go index 150a2f08c7a..5baa923ce60 100644 --- a/apps/backend/internal/task/repository/sqlite/restart_reconstruction_test.go +++ b/apps/backend/internal/task/repository/sqlite/restart_reconstruction_test.go @@ -17,7 +17,7 @@ func TestRestartRecoveryRepairsMissingCanonicalProjection(t *testing.T) { } func TestRestartRecoveryPreservesCanonicalLiveDispatch(t *testing.T) { - for _, condition := range []string{"live", "no death proof", "dead", "lost CAS", "retired verified", "retired changed", "retired live"} { + for _, condition := range []string{"live", "no death proof", "dead", "lost CAS", "retired verified", "retired changed", "retired live", "continuation", "continuation missing", "continuation changed"} { t.Run(condition, func(t *testing.T) { testMissingProjectionReconstruction(t, condition) }) } } @@ -48,12 +48,17 @@ func testMissingProjectionReconstruction(t *testing.T, condition string) { PayloadHash: journal.SubmissionHash(payload), State: models.DeliverySubmissionInterruptedUnknown, Outcome: "unknown"} if condition != "unknown" { submission.State = models.DeliverySubmissionDispatching + submission.Outcome = "" + } + continuation := condition == "continuation" || condition == "continuation missing" || condition == "continuation changed" + if continuation { + submission.ID, submission.DispatchAttemptID = "prompt:continuation:attempt", "continuation:attempt" } _, err := repo.PrepareAgentDeliverySubmission(ctx, &submission) require.NoError(t, err) request := &models.AgentDeliveryReconstructionRequest{TaskID: "task", SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: 1, ExpectedSessionState: models.TaskSessionStateWaitingForInput, NativeSessionID: "native", - MessageID: "message", ExpectedBlock: *block, Submission: submission, SourceStreamFirstRetained: 1, + ExistingSubmissionOnly: continuation, MessageID: submission.DispatchAttemptID, ExpectedBlock: *block, Submission: submission, SourceStreamFirstRetained: 1, Recovery: models.AgentDeliveryRecovery{Phase: models.AgentDeliveryRecoveryUncertain, SessionID: "session", SubmissionID: submission.ID, StreamID: "stream", IncarnationID: "incarnation", HarnessGeneration: 1, Reconstruction: &models.AgentDeliveryReconstructionProvenance{SourceSessionID: "session", SourceIncarnationID: "incarnation", @@ -104,7 +109,23 @@ func testMissingProjectionReconstruction(t *testing.T, condition string) { request.Recovery.Revision = prior.Revision } + if condition == "continuation missing" { + _, err = repo.DB().ExecContext(ctx, `DELETE FROM agent_delivery_submissions WHERE id = ?`, submission.ID) + require.NoError(t, err) + } + if condition == "continuation changed" { + _, err = repo.DB().ExecContext(ctx, `UPDATE agent_delivery_submissions SET payload_hash = 'changed' WHERE id = ?`, submission.ID) + require.NoError(t, err) + } result, err := repo.ReconstructAgentDeliverySubmission(ctx, request) + if condition == "continuation missing" || condition == "continuation changed" { + require.Error(t, err, "canonical-only repair must not insert missing or overwrite changed evidence") + current, readErr := repo.GetTaskSession(ctx, "session") + require.NoError(t, readErr) + _, exists := models.LoadAgentDeliveryRecovery(current.Metadata) + require.False(t, exists) + return + } if condition == "history unresolved" { require.ErrorIs(t, err, repoerrors.ErrAgentDeliveryReconstructionConflict) return diff --git a/apps/web/e2e/helpers/provider-interruption-continuation.ts b/apps/web/e2e/helpers/provider-interruption-continuation.ts index e48c437025d..6737a00bcf5 100644 --- a/apps/web/e2e/helpers/provider-interruption-continuation.ts +++ b/apps/web/e2e/helpers/provider-interruption-continuation.ts @@ -179,9 +179,9 @@ export function assertNativeContinuationTrace(tracePath: string, scenario: strin expect(record.connection_id).toBeTruthy(); expect(record.session_id).toBeTruthy(); } - expect(new Set(nativeRecords.map((record) => record.process_id)).size).toBe(1); - expect(new Set(nativeRecords.map((record) => record.connection_id)).size).toBe(1); - expect(records.filter((record) => record.event === "initialize")).toHaveLength(1); + expect(new Set(nativeRecords.map((record) => record.process_id)).size).toBe(1 + loads); + expect(new Set(nativeRecords.map((record) => record.connection_id)).size).toBe(1 + loads); + expect(records.filter((record) => record.event === "initialize")).toHaveLength(1 + loads); expect( records.filter((record) => record.event === "session_load" && record.session_id === nativeId), ).toHaveLength(loads); diff --git a/apps/web/e2e/tests/session/provider-interruption-continuation.spec.ts b/apps/web/e2e/tests/session/provider-interruption-continuation.spec.ts index 1e5c087cb2f..8788c150a6d 100644 --- a/apps/web/e2e/tests/session/provider-interruption-continuation.spec.ts +++ b/apps/web/e2e/tests/session/provider-interruption-continuation.spec.ts @@ -466,9 +466,21 @@ for (const survives of [false, true]) { (message) => message.metadata?.recovery_disposition === "restart_interrupted", ); expect(recovery.metadata?.attempts_started).toBe(1); - await expect(session.recoveryResumeButton()).toBeVisible(); + await expect + .poll( + async () => { + const restored = (await apiClient.getTaskSession(fixture.sessionId)).session; + const delivery = restored.metadata?.agent_delivery_recovery as + | { phase?: string } + | undefined; + return { state: restored.state, phase: delivery?.phase }; + }, + { timeout: 90_000 }, + ) + .toEqual({ state: "WAITING_FOR_INPUT", phase: "restored" }); + await session.composerReady(); } - assertNativeContinuationTrace(fixture.tracePath, "read-hold"); + assertNativeContinuationTrace(fixture.tracePath, "read-hold", survives ? 0 : 1); await expect(session.activeChat()).toContainText("partial history preserved"); } finally { await fixture.dispose(); diff --git a/docs/plans/session-recovery-convergence/task-03-recovery-contract.md b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md index 9226863a10f..eb56cd4e965 100644 --- a/docs/plans/session-recovery-convergence/task-03-recovery-contract.md +++ b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md @@ -51,7 +51,11 @@ correcting work-order metadata. Review remediation covers terminal drain orderin complete stream snapshot validation, exact submission stream binding and unknown reconstruction outcomes. Focused automatic restart and repeated recovery checks pass. CI-reported recovery fixtures pass after correcting metadata-only retained inspection. -All eight initial review findings are resolved; final-head CI and any new -review findings remain the delivery gate. No local E2E has run. +All eight initial review findings are resolved. CI then exposed an empty outcome +on reconstructed dispatch and a canonical internal continuation incorrectly +requiring a user-message row. Both desktop and phone pruned-history failures +share the former cause. Focused regressions cover empty-outcome settlement and +existing-only continuation repair, including deleted/changed canonical controls. +Final-head CI and any new review findings remain the delivery gate. No local E2E has run. [Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/specs/platform/system-design/session-recovery-convergence.md b/docs/specs/platform/system-design/session-recovery-convergence.md index 46dfb10b4ad..d8709603adc 100644 --- a/docs/specs/platform/system-design/session-recovery-convergence.md +++ b/docs/specs/platform/system-design/session-recovery-convergence.md @@ -67,6 +67,12 @@ or unprojected receipt refuses resolution. A transaction rollback changes none of the binding, notice or fence. A retry reuses retired journal proof. Retained inspection has a separate metadata-only stored-submission request for unretired legacy history; retirement proof still requires the actual retired marker and preserves the original label. +A retained submission with an existing canonical SQL row can repair its recovery +presentation without a user-message row, including internal continuations. The +request marks this as existing-record-only, and the SQL transaction refuses a +missing or changed canonical row instead of inserting one. Reconstruction from +journal-only evidence still requires the exact user-message association. A +proven-dead dispatch records an unknown outcome when its prior outcome is empty. A matching prepared checkpoint with no candidate is completed in that transaction; an allocated/launching or unrelated attempt still blocks it. Legacy composer uncertainty is cleared only for an exactly retired submission. An explicit null From e9ee7b055c3fed957b048abb8858b3aa68a48554 Mon Sep 17 00:00:00 2001 From: Kandev Agent Date: Mon, 12 Oct 2026 05:23:25 +0100 Subject: [PATCH 6/7] fix: preserve initial prompt identity during recovery --- .../reconstruction_message_authority.go | 14 ++++----- .../restart_reconciliation_test.go | 29 +++++++++++++------ ...ession_delivery_reconstruction_evidence.go | 22 ++++++++------ ...gent_delivery_reconstruction_validation.go | 9 +++++- .../sqlite/restart_reconstruction_test.go | 16 ++++++++-- .../sidebar-filter-selected-first-helpers.ts | 2 +- .../task-03-recovery-contract.md | 5 ++++ .../session-recovery-convergence.md | 4 ++- 8 files changed, 70 insertions(+), 31 deletions(-) diff --git a/apps/backend/internal/orchestrator/reconstruction_message_authority.go b/apps/backend/internal/orchestrator/reconstruction_message_authority.go index 41f03943a68..3b249653bef 100644 --- a/apps/backend/internal/orchestrator/reconstruction_message_authority.go +++ b/apps/backend/internal/orchestrator/reconstruction_message_authority.go @@ -10,21 +10,21 @@ import ( "github.com/kandev/kandev/internal/task/repository" ) -func (s *Service) reconstructionMessageAuthority(ctx context.Context, session *models.TaskSession, candidate journal.SubmissionSummary, messageID string) (bool, string, error) { +func (s *Service) reconstructionMessageAuthority(ctx context.Context, session *models.TaskSession, candidate journal.SubmissionSummary, messageID string) (*models.AgentDeliverySubmission, string, error) { store, ok := s.repo.(repository.AgentDeliveryRepository) if !ok { - return false, recoveryStateUnavailableReason, nil + return nil, recoveryStateUnavailableReason, nil } canonical, err := store.GetAgentDeliverySubmission(ctx, candidate.ID) if errors.Is(err, sql.ErrNoRows) || errors.Is(err, repository.ErrAgentDeliverySubmissionNotFound) { reason, messageErr := s.validateReconstructionMessage(ctx, session, messageID) - return false, reason, messageErr + return nil, reason, messageErr } if err != nil { - return false, "", err + return nil, "", err } - if canonical == nil || canonical.SessionID != session.ID || canonical.IncarnationID != candidate.IncarnationID || canonical.HarnessGeneration != int64(candidate.HarnessGeneration) || canonical.DispatchAttemptID != messageID || canonical.PayloadHash != candidate.Hash { - return false, deliveryIdentityMismatchReason, nil + if canonical == nil || canonical.ID != candidate.ID || canonical.SessionID != session.ID || canonical.IncarnationID != candidate.IncarnationID || canonical.HarnessGeneration != int64(candidate.HarnessGeneration) || canonical.DispatchAttemptID == "" || canonical.PayloadHash != candidate.Hash { + return nil, deliveryIdentityMismatchReason, nil } - return true, "", nil + return canonical, "", nil } diff --git a/apps/backend/internal/orchestrator/restart_reconciliation_test.go b/apps/backend/internal/orchestrator/restart_reconciliation_test.go index fbe76bdcafc..7b8bf0cdf03 100644 --- a/apps/backend/internal/orchestrator/restart_reconciliation_test.go +++ b/apps/backend/internal/orchestrator/restart_reconciliation_test.go @@ -132,26 +132,30 @@ func (m *alreadyResumedRestartManager) ReconcileAlreadyResumedDelivery(ctx conte // @covers AC-PLATFORM-DURABLE-RESTART-RECONCILIATION-001.2 func TestRestartRecoveryUsesAlreadyLoadedNativeConversation(t *testing.T) { - testAlreadyLoadedRestartRecovery(t, false, false, false, false) + testAlreadyLoadedRestartRecovery(t, false, false, false, "") } func TestRestartRecoveryRepairsBothDispatchingWithoutMetadata(t *testing.T) { - testAlreadyLoadedRestartRecovery(t, true, false, false, false) + testAlreadyLoadedRestartRecovery(t, true, false, false, "") } func TestAlreadyLoadedRestartRecoveryPreservesIndependentBlock(t *testing.T) { - testAlreadyLoadedRestartRecovery(t, false, true, false, false) + testAlreadyLoadedRestartRecovery(t, false, true, false, "") } func TestRestartRecoveryRepairsIncompleteBlockWithKnownOwner(t *testing.T) { - testAlreadyLoadedRestartRecovery(t, false, false, true, false) + testAlreadyLoadedRestartRecovery(t, false, false, true, "") } func TestRestartRecoveryRepairsCanonicalContinuationWithoutUserMessage(t *testing.T) { - testAlreadyLoadedRestartRecovery(t, true, false, false, true) + testAlreadyLoadedRestartRecovery(t, true, false, false, "continuation") } -func testAlreadyLoadedRestartRecovery(t *testing.T, missingMetadata, independentBlock, partialBlock, continuation bool) { +func TestRestartRecoveryPreservesInitialPromptDispatchAttempt(t *testing.T) { + testAlreadyLoadedRestartRecovery(t, true, false, false, "initial") +} + +func testAlreadyLoadedRestartRecovery(t *testing.T, missingMetadata, independentBlock, partialBlock bool, internalPrompt string) { service, repo, base, session, store := newSilentRestoreEligibilityFixture(t, "native", "loaded") ctx := context.Background() root := t.TempDir() @@ -162,16 +166,20 @@ func testAlreadyLoadedRestartRecovery(t *testing.T, missingMetadata, independent state := journal.SubmissionInterruptedUnknown if missingMetadata { messageID := "crash-message" - if continuation { + if internalPrompt == "continuation" { messageID = "continuation:crash" } + dispatchAttemptID := messageID + if internalPrompt == "initial" { + messageID, dispatchAttemptID = "initial:"+session.ID+":1:loaded-execution", "loaded-execution" + } submissionID, state = "prompt:"+messageID, journal.SubmissionDispatching require.NoError(t, repo.CreateTurn(ctx, &models.Turn{ID: "crash-turn", TaskID: session.TaskID, TaskSessionID: session.ID, StartedAt: time.Now().UTC()})) - if !continuation { + if internalPrompt == "" { require.NoError(t, repo.CreateMessage(ctx, &models.Message{ID: messageID, TaskID: session.TaskID, TaskSessionID: session.ID, TurnID: "crash-turn", AuthorType: models.MessageAuthorUser, Content: "original prompt"})) } - _, err = repo.DB().ExecContext(ctx, `UPDATE agent_delivery_submissions SET id = ?, dispatch_attempt_id = ?, state = 'dispatching' WHERE id = ?`, submissionID, messageID, "loaded-submission") + _, err = repo.DB().ExecContext(ctx, `UPDATE agent_delivery_submissions SET id = ?, dispatch_attempt_id = ?, state = 'dispatching' WHERE id = ?`, submissionID, dispatchAttemptID, "loaded-submission") require.NoError(t, err) _, err = repo.DB().ExecContext(ctx, `UPDATE task_sessions SET metadata = '{}' WHERE id = ?`, session.ID) require.NoError(t, err) @@ -227,6 +235,9 @@ func testAlreadyLoadedRestartRecovery(t *testing.T, missingMetadata, independent require.NoError(t, err) require.Equal(t, models.DeliverySubmissionInterruptedUnknown, canonical.State) require.Equal(t, payload, canonical.Payload) + if internalPrompt == "initial" { + require.Equal(t, "loaded-execution", canonical.DispatchAttemptID) + } if missingMetadata { require.Equal(t, "unknown", canonical.Outcome) } else { diff --git a/apps/backend/internal/orchestrator/session_delivery_reconstruction_evidence.go b/apps/backend/internal/orchestrator/session_delivery_reconstruction_evidence.go index afb08f87bdc..2182925099d 100644 --- a/apps/backend/internal/orchestrator/session_delivery_reconstruction_evidence.go +++ b/apps/backend/internal/orchestrator/session_delivery_reconstruction_evidence.go @@ -11,11 +11,11 @@ import ( ) type reconstructionSelection struct { - evidence *agentruntime.DeliveryRecordEvidence - submission *journal.Submission - stream *journal.Stream - messageID string - existingOnly bool + evidence *agentruntime.DeliveryRecordEvidence + submission *journal.Submission + stream *journal.Stream + messageID string + canonical *models.AgentDeliverySubmission } func (s *Service) readReconstructionSelection(ctx context.Context, client deliveryRecordEvidenceClient, session *models.TaskSession, incarnationID string, generation int64, block *models.SessionRecoveryBlock) (*reconstructionSelection, string, error) { @@ -35,7 +35,7 @@ func (s *Service) readReconstructionSelection(ctx context.Context, client delive if !ok { return blocked(deliveryIdentityMismatchReason) } - existingOnly, reason, err := s.reconstructionMessageAuthority(ctx, session, candidate, messageID) + canonical, reason, err := s.reconstructionMessageAuthority(ctx, session, candidate, messageID) if err != nil { return nil, "", err } else if reason != "" { @@ -51,7 +51,7 @@ func (s *Service) readReconstructionSelection(ctx context.Context, client delive if selected == nil || journal.VerifyReconstructionSubmission(*selected, candidate) != nil { return blocked(deliveryIdentityMismatchReason) } - return &reconstructionSelection{evidence: evidence, submission: selected, stream: stream, messageID: messageID, existingOnly: existingOnly}, "", nil + return &reconstructionSelection{evidence: evidence, submission: selected, stream: stream, messageID: messageID, canonical: canonical}, "", nil } func (s *Service) prepareReconstructionRequest(ctx context.Context, session *models.TaskSession, incarnationID string, generation int64, block *models.SessionRecoveryBlock, selection *reconstructionSelection) (*models.AgentDeliveryReconstructionRequest, string, error) { @@ -85,10 +85,14 @@ func (s *Service) prepareReconstructionRequest(ctx context.Context, session *mod if !okHigh || !okAcknowledged || !okFirst { return blocked(deliveryIdentityMismatchReason) } + dispatchAttemptID := messageID + if selection.canonical != nil { + dispatchAttemptID = selection.canonical.DispatchAttemptID + } payloadHash := selected.Hash state := models.DeliverySubmissionInterruptedUnknown request := &models.AgentDeliveryReconstructionRequest{ - ExistingSubmissionOnly: selection.existingOnly, + ExistingSubmissionOnly: selection.canonical != nil, TaskID: session.TaskID, SessionID: session.ID, IncarnationID: incarnationID, HarnessGeneration: generation, ExpectedExecutionID: session.AgentExecutionID, ExpectedSessionState: session.State, ExpectedWorkspacePath: session.WorkspacePath, @@ -98,7 +102,7 @@ func (s *Service) prepareReconstructionRequest(ctx context.Context, session *mod Submission: models.AgentDeliverySubmission{ ID: selected.ID, SessionID: selected.SessionID, IncarnationID: selected.IncarnationID, HarnessGeneration: int64(selected.HarnessGeneration), OwnerGeneration: int64(selected.HarnessGeneration), - DispatchAttemptID: messageID, PayloadHash: payloadHash, Payload: append([]byte(nil), selected.Payload...), + DispatchAttemptID: dispatchAttemptID, PayloadHash: payloadHash, Payload: append([]byte(nil), selected.Payload...), State: state, Outcome: "reconstructed_unknown", CreatedAt: selected.CreatedAt, UpdatedAt: selected.UpdatedAt, }, Recovery: models.AgentDeliveryRecovery{ diff --git a/apps/backend/internal/task/repository/sqlite/agent_delivery_reconstruction_validation.go b/apps/backend/internal/task/repository/sqlite/agent_delivery_reconstruction_validation.go index cab370f0e68..772fee95c2b 100644 --- a/apps/backend/internal/task/repository/sqlite/agent_delivery_reconstruction_validation.go +++ b/apps/backend/internal/task/repository/sqlite/agent_delivery_reconstruction_validation.go @@ -24,11 +24,18 @@ func validReconstructionSubmission(request *models.AgentDeliveryReconstructionRe submission := request.Submission return submission.ID == "prompt:"+request.MessageID && submission.SessionID == request.SessionID && submission.IncarnationID == request.IncarnationID && submission.HarnessGeneration == request.HarnessGeneration && - submission.OwnerGeneration == request.HarnessGeneration && submission.DispatchAttemptID == request.MessageID && + submission.OwnerGeneration == request.HarnessGeneration && validReconstructionDispatchAttempt(request) && submission.State == models.DeliverySubmissionInterruptedUnknown && !submission.CreatedAt.IsZero() && !submission.UpdatedAt.IsZero() && validReconstructionSubmissionPayload(submission) } +func validReconstructionDispatchAttempt(request *models.AgentDeliveryReconstructionRequest) bool { + if request.ExistingSubmissionOnly { + return request.Submission.DispatchAttemptID != "" + } + return request.Submission.DispatchAttemptID == request.MessageID +} + func validReconstructionSubmissionPayload(submission models.AgentDeliverySubmission) bool { if submission.PayloadHash == "" || len(submission.Payload) == 0 || len(submission.Payload) > agentDeliveryReconstructionPayloadLimit { return false diff --git a/apps/backend/internal/task/repository/sqlite/restart_reconstruction_test.go b/apps/backend/internal/task/repository/sqlite/restart_reconstruction_test.go index 5baa923ce60..f0f711c7cc7 100644 --- a/apps/backend/internal/task/repository/sqlite/restart_reconstruction_test.go +++ b/apps/backend/internal/task/repository/sqlite/restart_reconstruction_test.go @@ -2,6 +2,7 @@ package sqlite import ( "context" + "strings" "testing" "time" @@ -17,7 +18,7 @@ func TestRestartRecoveryRepairsMissingCanonicalProjection(t *testing.T) { } func TestRestartRecoveryPreservesCanonicalLiveDispatch(t *testing.T) { - for _, condition := range []string{"live", "no death proof", "dead", "lost CAS", "retired verified", "retired changed", "retired live", "continuation", "continuation missing", "continuation changed"} { + for _, condition := range []string{"live", "no death proof", "dead", "lost CAS", "retired verified", "retired changed", "retired live", "continuation", "continuation missing", "continuation changed", "initial", "initial attempt changed"} { t.Run(condition, func(t *testing.T) { testMissingProjectionReconstruction(t, condition) }) } } @@ -54,11 +55,15 @@ func testMissingProjectionReconstruction(t *testing.T, condition string) { if continuation { submission.ID, submission.DispatchAttemptID = "prompt:continuation:attempt", "continuation:attempt" } + initial := strings.HasPrefix(condition, "initial") + if initial { + submission.ID, submission.DispatchAttemptID = "prompt:initial:session:1:execution", "execution" + } _, err := repo.PrepareAgentDeliverySubmission(ctx, &submission) require.NoError(t, err) request := &models.AgentDeliveryReconstructionRequest{TaskID: "task", SessionID: "session", IncarnationID: "incarnation", HarnessGeneration: 1, ExpectedSessionState: models.TaskSessionStateWaitingForInput, NativeSessionID: "native", - ExistingSubmissionOnly: continuation, MessageID: submission.DispatchAttemptID, ExpectedBlock: *block, Submission: submission, SourceStreamFirstRetained: 1, + ExistingSubmissionOnly: continuation || initial, MessageID: strings.TrimPrefix(submission.ID, "prompt:"), ExpectedBlock: *block, Submission: submission, SourceStreamFirstRetained: 1, Recovery: models.AgentDeliveryRecovery{Phase: models.AgentDeliveryRecoveryUncertain, SessionID: "session", SubmissionID: submission.ID, StreamID: "stream", IncarnationID: "incarnation", HarnessGeneration: 1, Reconstruction: &models.AgentDeliveryReconstructionProvenance{SourceSessionID: "session", SourceIncarnationID: "incarnation", @@ -117,8 +122,12 @@ func testMissingProjectionReconstruction(t *testing.T, condition string) { _, err = repo.DB().ExecContext(ctx, `UPDATE agent_delivery_submissions SET payload_hash = 'changed' WHERE id = ?`, submission.ID) require.NoError(t, err) } + if condition == "initial attempt changed" { + _, err = repo.DB().ExecContext(ctx, `UPDATE agent_delivery_submissions SET dispatch_attempt_id = 'replacement' WHERE id = ?`, submission.ID) + require.NoError(t, err) + } result, err := repo.ReconstructAgentDeliverySubmission(ctx, request) - if condition == "continuation missing" || condition == "continuation changed" { + if condition == "continuation missing" || condition == "continuation changed" || condition == "initial attempt changed" { require.Error(t, err, "canonical-only repair must not insert missing or overwrite changed evidence") current, readErr := repo.GetTaskSession(ctx, "session") require.NoError(t, readErr) @@ -165,6 +174,7 @@ func testMissingProjectionReconstruction(t *testing.T, condition string) { require.Equal(t, submission.ID, result.Recovery.SubmissionID) retained, err := repo.GetAgentDeliverySubmission(ctx, submission.ID) require.NoError(t, err) + require.Equal(t, submission.DispatchAttemptID, retained.DispatchAttemptID) require.Equal(t, "unknown", retained.Outcome, "projection repair preserves the canonical historical outcome") repeated, err := repo.ReconstructAgentDeliverySubmission(ctx, request) require.NoError(t, err) diff --git a/apps/web/e2e/tests/task/sidebar-filter-selected-first-helpers.ts b/apps/web/e2e/tests/task/sidebar-filter-selected-first-helpers.ts index e96e44f7b9c..5868fd790de 100644 --- a/apps/web/e2e/tests/task/sidebar-filter-selected-first-helpers.ts +++ b/apps/web/e2e/tests/task/sidebar-filter-selected-first-helpers.ts @@ -157,7 +157,7 @@ export async function verifySelectedFilters( expect(labelsAfterSearch.slice(0, seeded.selected.length).sort()).toEqual( [...seeded.selected].sort(), ); - await activate(rows.filter({ hasText: seeded.selected[0] }), mobile); + await activate(picker.getByRole("option", { name: seeded.selected[0], exact: true }), mobile); await expect(rows.nth(0)).toHaveText(seeded.selected[1]); await expect(rows.nth(0)).toHaveAttribute("data-checked", "true"); await search.fill(seeded.unselected[0]); diff --git a/docs/plans/session-recovery-convergence/task-03-recovery-contract.md b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md index eb56cd4e965..7c0adf30778 100644 --- a/docs/plans/session-recovery-convergence/task-03-recovery-contract.md +++ b/docs/plans/session-recovery-convergence/task-03-recovery-contract.md @@ -56,6 +56,11 @@ on reconstructed dispatch and a canonical internal continuation incorrectly requiring a user-message row. Both desktop and phone pruned-history failures share the former cause. Focused regressions cover empty-outcome settlement and existing-only continuation repair, including deleted/changed canonical controls. +The next CI run exposed initial prompts whose canonical dispatch attempt is the +original execution ID rather than the message-derived identity. Recovery now +preserves that stored identity, with initial-prompt and changed-attempt controls. +The same CI run also exposed a substring collision in the sidebar filter test; +its selection now uses the exact option name. Final-head CI and any new review findings remain the delivery gate. No local E2E has run. [Parent plan](plan.md). [Requirements](../../specs/platform/requirements/durable-restart-reconciliation.md). diff --git a/docs/specs/platform/system-design/session-recovery-convergence.md b/docs/specs/platform/system-design/session-recovery-convergence.md index d8709603adc..676ff597b60 100644 --- a/docs/specs/platform/system-design/session-recovery-convergence.md +++ b/docs/specs/platform/system-design/session-recovery-convergence.md @@ -68,7 +68,9 @@ of the binding, notice or fence. A retry reuses retired journal proof. Retained metadata-only stored-submission request for unretired legacy history; retirement proof still requires the actual retired marker and preserves the original label. A retained submission with an existing canonical SQL row can repair its recovery -presentation without a user-message row, including internal continuations. The +presentation without a user-message row, including initial prompts and internal +continuations. It preserves the stored dispatch-attempt identity: an initial +prompt uses the original execution ID, independently of its submission ID. The request marks this as existing-record-only, and the SQL transaction refuses a missing or changed canonical row instead of inserting one. Reconstruction from journal-only evidence still requires the exact user-message association. A From 1365fa26e38263354826a399b29e0974dc6e56d4 Mon Sep 17 00:00:00 2001 From: Kandev Agent Date: Mon, 12 Oct 2026 05:23:32 +0100 Subject: [PATCH 7/7] test: require an unknown outcome after interrupted recovery --- .../runtime/lifecycle/e2e_pruned_delivery_fixture_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/backend/internal/agent/runtime/lifecycle/e2e_pruned_delivery_fixture_test.go b/apps/backend/internal/agent/runtime/lifecycle/e2e_pruned_delivery_fixture_test.go index 653e48bb10b..5081098a6d2 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/e2e_pruned_delivery_fixture_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/e2e_pruned_delivery_fixture_test.go @@ -93,8 +93,7 @@ func assertPrunedDeliveryRestored(t *testing.T, repo *tasksqlite.Repository, ses original, err := repo.GetAgentDeliverySubmission(ctx, "prompt:message-e2e-record-"+sessionID+"-1") require.NoError(t, err) require.Equal(t, models.DeliverySubmissionInterruptedUnknown, original.State) - // Reconstruction preserves the existing outcome; native-resume reconciliation records unknown. - require.Contains(t, []string{"", "unknown"}, original.Outcome) + require.Equal(t, "unknown", original.Outcome) require.Zero(t, original.TerminalSequence) generation, err := repo.GetCurrentHarnessSessionGeneration(ctx, sessionID, original.IncarnationID) require.NoError(t, err)