diff --git a/packages/cli/src/__tests__/activation-command.test.ts b/packages/cli/src/__tests__/activation-command.test.ts index 6a6978daa8..36d5756f21 100644 --- a/packages/cli/src/__tests__/activation-command.test.ts +++ b/packages/cli/src/__tests__/activation-command.test.ts @@ -461,6 +461,57 @@ describe('maka activate JSONL protocol', () => { }); }); + test('blocks a completed invocation whose stream carried a boundary failure', async () => { + // Before #4506 the classifier cleared `sandboxBoundary` when a later tool + // succeeded, so this activation completed with exit 0. The stream event is + // the hard fact; the outcome shape here (`completed`, boundary `none`) is + // exactly what that clearing produced. + const lines: string[] = []; + const boundaryFailure = { + kind: 'text', + text: 'Write requires an approved session sandbox boundary expansion.', + sandboxFailure: { + reason: 'sandbox_boundary_required', + requiredExpansion: { + filesystem: { + entries: [{ path: '/tmp/output', access: 'write', scope: 'subtree' }], + }, + }, + }, + } as const; + const result = await runMakaActivationCli( + [ + '--state-root', + ROOTS.stateRoot, + '--workspace-root', + ROOTS.workspaceRoot, + '--config-root', + ROOTS.configRoot, + ], + { + ...fakeDeps({ + result: completedResult(), + events: [ + { + type: 'tool_result', + id: 'event-boundary-result', + turnId: 'turn-1', + ts: 1, + toolUseId: 'tool-boundary', + isError: true, + content: boundaryFailure, + }, + ], + }), + writeStdout: (text) => lines.push(text.trim()), + }, + ); + + assert.equal(result, 3); + assert.equal(JSON.parse(lines.at(-1)!).status, 'blocked'); + assert.equal(JSON.parse(lines.at(-1)!).reason, 'permission_required'); + }); + test('retries non-permission blocked sessions instead of requesting permission', async () => { for (const blockedReason of ['auth', 'tool_failed'] as const) { const lines: string[] = []; diff --git a/packages/cli/src/__tests__/runtime-host-run-command.test.ts b/packages/cli/src/__tests__/runtime-host-run-command.test.ts index 49d76680c8..0922c5e562 100644 --- a/packages/cli/src/__tests__/runtime-host-run-command.test.ts +++ b/packages/cli/src/__tests__/runtime-host-run-command.test.ts @@ -313,6 +313,56 @@ describe('Runtime Host maka run adapter', () => { ); }); + test('returns exit code 1 when a same-named tool retries a different target (live)', async () => { + const fixture = runFixture({ + turnEvents: sandboxBoundaryEvents( + 'turn-1', + 'step-1', + 'step-2', + 'Partial answer', + 'Read', + ), + }); + + const exitCode = await runFixtureCommand(fixture, ['accept same-name different target']); + + assert.equal(exitCode, 1); + }); + + test('returns exit code 1 when a same-named tool retries a different target (durable)', async () => { + const fixture = runFixture({ + graph: true, + finalMessages: sandboxBoundaryMessages('step-1', 'step-2', 'Read'), + }); + + const exitCode = await runFixtureCommand(fixture, [ + 'accept durable same-name different target', + '--graph', + ]); + + assert.equal(exitCode, 1); + }); + + test('returns exit code 1 when a denied widening precedes any later tool success', async () => { + const stderr: string[] = []; + const fixture = runFixture({ + turnEvents: deniedWideningEvents('turn-1'), + }); + + const exitCode = await runFixtureCommand( + fixture, + ['accept post-denial success'], + () => {}, + (text) => stderr.push(text), + ); + + assert.equal(exitCode, 1); + assert.equal( + stderr.join(''), + 'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n', + ); + }); + test('returns exit code 1 when reconnect restores a missed sandbox failure', async () => { let publishReplacement = () => {}; const fixture = runFixture({ @@ -351,18 +401,26 @@ describe('Runtime Host maka run adapter', () => { assert.equal(stdout.join(''), 'Final graph answer\n'); }); - test('returns exit code 0 when a root Graph boundary failure recovers', async () => { + test('keeps a root Graph boundary failure unresolved even when a later same-named call succeeds', async () => { const stdout: string[] = []; + const stderr: string[] = []; const fixture = runFixture({ graph: true, turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'), }); - const exitCode = await runFixtureCommand(fixture, ['recover once', '--graph'], (text) => - stdout.push(text), + const exitCode = await runFixtureCommand( + fixture, + ['recover once', '--graph'], + (text) => stdout.push(text), + (text) => stderr.push(text), ); - assert.equal(exitCode, 0); - assert.equal(stdout.join(''), 'Final graph answer\n'); + assert.equal(exitCode, 1); + assert.equal(stdout.join(''), ''); + assert.equal( + stderr.join(''), + 'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n', + ); }); test('returns exit code 1 when a same-step Graph sibling succeeds after a sandbox failure', async () => { @@ -465,7 +523,7 @@ describe('Runtime Host maka run adapter', () => { assert.equal(observed.at(-1)?.finalOutput, 'Final graph answer'); }); - test('reports a recovered sandbox boundary from live and durable Turns', async () => { + test('keeps the sandbox boundary unresolved across live and durable Turns', async () => { const live = await observeFixtureOutcome({ turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'), }); @@ -474,8 +532,8 @@ describe('Runtime Host maka run adapter', () => { finalMessages: sandboxBoundaryMessages('step-1', 'step-2'), }); - assert.equal(live.sandboxBoundary, 'recovered'); - assert.equal(durable.sandboxBoundary, 'recovered'); + assert.equal(live.sandboxBoundary, 'unresolved'); + assert.equal(durable.sandboxBoundary, 'unresolved'); }); test('leaves sandbox failures unresolved when their provider steps are unavailable', async () => { @@ -491,7 +549,7 @@ describe('Runtime Host maka run adapter', () => { assert.equal(durable.sandboxBoundary, 'unresolved'); }); - test('returns a recovered boundary to unresolved after a later sandbox failure', async () => { + test('keeps the boundary unresolved across interleaved successes and a later sandbox failure', async () => { const outcome = await observeFixtureOutcome({ turnEvents: sandboxFailureAfterRecoveryEvents('turn-1'), }); @@ -1356,8 +1414,12 @@ function sandboxBoundaryMessages( const sameStep = failureStepId !== undefined && failureStepId === successStepId; return [ ...graphMessages(false), - ...(failureStepId === undefined ? [] : [storedToolCall('turn-2', 'tool-1', failureStepId, 5)]), - ...(sameStep ? [storedToolCall('turn-2', 'tool-2', successStepId, 6, successToolName)] : []), + ...(failureStepId === undefined + ? [] + : [storedToolCall('turn-2', 'tool-1', failureStepId, 5, 'Read')]), + ...(sameStep + ? [storedToolCall('turn-2', 'tool-2', successStepId, 6, successToolName)] + : []), sandboxFailureToolResult('turn-2', 7), ...(successStepId === undefined || sameStep ? [] @@ -1501,8 +1563,12 @@ async function* sandboxBoundaryEvents( successToolName = 'Read', ): AsyncIterable { const sameStep = failureStepId !== undefined && failureStepId === successStepId; - if (failureStepId !== undefined) yield toolStart(turnId, 'tool-1', failureStepId, 1); - if (sameStep) yield toolStart(turnId, 'tool-2', successStepId, 2, successToolName); + if (failureStepId !== undefined) { + yield toolStart(turnId, 'tool-1', failureStepId, 1, 'Read'); + } + if (sameStep) { + yield toolStart(turnId, 'tool-2', successStepId, 2, successToolName); + } yield sandboxFailureToolResult(turnId, 3); if (successStepId !== undefined && !sameStep) { yield toolStart(turnId, 'tool-2', successStepId, 4, successToolName); @@ -1511,6 +1577,16 @@ async function* sandboxBoundaryEvents( yield* eventsFor(turnId, text, 6); } +async function* deniedWideningEvents(turnId: string): AsyncIterable { + yield toolStart(turnId, 'tool-1', 'step-1', 1, 'Read', { path: '/outside/secret.txt' }); + yield sandboxFailureToolResult(turnId, 2); + yield toolStart(turnId, 'tool-2', 'step-2', 3, 'request_sandbox_boundary'); + yield successfulToolResult(turnId, 4, 'tool-2'); + yield toolStart(turnId, 'tool-3', 'step-3', 5, 'Read', { path: '/workspace/README.md' }); + yield successfulToolResult(turnId, 6, 'tool-3'); + yield* eventsFor(turnId, 'Recovered answer', 7); +} + async function* projectedSameStepSandboxFailureEvents(turnId: string): AsyncIterable { yield toolStart(turnId, 'tool-1', 'step-1', 1); yield toolStart(turnId, 'tool-2', 'step-1', 2); @@ -1654,6 +1730,7 @@ function toolStart( stepId: string, ts: number, toolName = 'Read', + args: unknown = {}, ): Extract { return { type: 'tool_start', @@ -1662,7 +1739,7 @@ function toolStart( ts, toolUseId, toolName, - args: {}, + args, stepId, }; } @@ -1673,6 +1750,7 @@ function storedToolCall( stepId: string, ts: number, toolName = 'Read', + args: unknown = {}, ): Extract { return { type: 'tool_call', @@ -1680,7 +1758,7 @@ function storedToolCall( turnId, ts, toolName, - args: {}, + args, stepId, }; } diff --git a/packages/cli/src/activation-command.ts b/packages/cli/src/activation-command.ts index aa0fce3f51..17511f233b 100644 --- a/packages/cli/src/activation-command.ts +++ b/packages/cli/src/activation-command.ts @@ -588,10 +588,7 @@ export async function runMakaActivationCli( if (invocation?.failure?.class === 'permission_denied') { return finish('blocked', 'permission_denied', undefined, 'grant_permission'); } - if ( - (streamBoundaryFailure && invocation?.sandboxBoundary !== 'recovered') || - invocation?.sandboxBoundary === 'unresolved' - ) { + if (streamBoundaryFailure || invocation?.sandboxBoundary === 'unresolved') { return finish('blocked', 'permission_required', undefined, 'grant_permission'); } if (!invocation) return finish('fatal_failure', 'missing_invocation'); diff --git a/packages/cli/src/run-command-core.ts b/packages/cli/src/run-command-core.ts index 2fba37b35e..2590b7a2a1 100644 --- a/packages/cli/src/run-command-core.ts +++ b/packages/cli/src/run-command-core.ts @@ -82,7 +82,7 @@ export interface MakaRunOutcome { status: 'completed' | 'failed'; finalOutput?: string; failure?: { class: string; message?: string }; - sandboxBoundary: 'none' | 'unresolved' | 'recovered'; + sandboxBoundary: 'none' | 'unresolved'; } export interface MakaRunContextInput { @@ -311,10 +311,7 @@ export async function runMakaTextCliCore( ...(parsed.options.hostProfileId ? { hostProfileId: parsed.options.hostProfileId } : {}), ...(parsed.options.projectId ? { projectId: parsed.options.projectId } : {}), runOutcomeObserver: (result) => { - if (result.sandboxBoundary === 'recovered') { - boundaryFailureInvocationIds.delete(result.outcomeId); - unclassifiedBoundaryFailure = false; - } else if (result.sandboxBoundary === 'unresolved') { + if (result.sandboxBoundary === 'unresolved') { boundaryFailureInvocationIds.add(result.outcomeId); unclassifiedBoundaryFailure = false; } diff --git a/packages/cli/src/runtime-host-run-command.ts b/packages/cli/src/runtime-host-run-command.ts index 92b84cc285..2571bdfaf4 100644 --- a/packages/cli/src/runtime-host-run-command.ts +++ b/packages/cli/src/runtime-host-run-command.ts @@ -570,12 +570,6 @@ type TurnOutcomeObservation = readonly status: 'failed'; readonly failure: NonNullable; } - | { - readonly kind: 'tool_call'; - readonly toolUseId: string; - readonly stepId: string | undefined; - readonly toolName: string; - } | { readonly kind: 'tool_result'; readonly toolUseId: string; @@ -586,20 +580,9 @@ type TerminalOutcomeObservation = Extract(); - readonly #unresolvedSandboxFailures = new Map< - string, - { - readonly failedStepId: string | undefined; - readonly failedToolName: string | undefined; - } - >(); + readonly #unresolvedSandboxFailures = new Set(); #finalOutput: string | undefined; #terminal: TerminalOutcomeObservation | undefined; - #sandboxBoundaryRecovered = false; constructor(outcomeId: string) { this.#outcomeId = outcomeId; @@ -617,36 +600,13 @@ class TurnOutcomeClassifier { this.#terminal = observation; } return; - case 'tool_call': - this.#callByToolUseId.set(observation.toolUseId, { - stepId: observation.stepId, - toolName: observation.toolName, - }); - return; case 'tool_result': { - const call = this.#callByToolUseId.get(observation.toolUseId); if (observation.outcome === 'sandbox_failure') { - this.#unresolvedSandboxFailures.set(observation.toolUseId, { - failedStepId: call?.stepId, - failedToolName: call?.toolName, - }); - return; - } - const unresolved = [...this.#unresolvedSandboxFailures.values()]; - // The wire has no retry identity. A later success can only prove recovery - // when there is exactly one unresolved candidate. - if ( - observation.outcome === 'success' && - call?.toolName !== 'request_sandbox_boundary' && - unresolved.length === 1 && - call?.stepId !== undefined && - unresolved[0]?.failedStepId !== undefined && - call.stepId !== unresolved[0].failedStepId && - call.toolName === unresolved[0].failedToolName - ) { - this.#unresolvedSandboxFailures.clear(); - this.#sandboxBoundaryRecovered = true; + this.#unresolvedSandboxFailures.add(observation.toolUseId); } + // No clearing path: `maka run` denies every widening request, so the + // boundary cannot move mid-Turn and a later success cannot prove that + // a blocked call recovered. The failure stays unresolved to the end. return; } } @@ -658,12 +618,7 @@ class TurnOutcomeClassifier { const terminal = this.#terminal; if (!terminal && incomplete === 'pending') return undefined; const completed = terminal?.status === 'completed'; - const sandboxBoundary = - this.#unresolvedSandboxFailures.size > 0 - ? 'unresolved' - : this.#sandboxBoundaryRecovered - ? 'recovered' - : 'none'; + const sandboxBoundary = this.#unresolvedSandboxFailures.size > 0 ? 'unresolved' : 'none'; const failure = terminal?.status === 'failed' ? terminal.failure @@ -704,14 +659,6 @@ function observationFromSessionEvent(event: SessionEvent): TurnOutcomeObservatio if (event.type === 'complete') { return observationFromCompleteEvent(event); } - if (event.type === 'tool_start') { - return { - kind: 'tool_call', - toolUseId: event.toolUseId, - stepId: event.stepId, - toolName: event.toolName, - }; - } return event.type === 'tool_result' ? observationFromToolResult(event) : undefined; } @@ -741,14 +688,6 @@ function observationFromStoredMessage(message: StoredMessage): TurnOutcomeObserv }, }; } - if (message.type === 'tool_call') { - return { - kind: 'tool_call', - toolUseId: message.id, - stepId: message.stepId, - toolName: message.toolName, - }; - } return message.type === 'tool_result' ? observationFromToolResult(message) : undefined; }