Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 125 additions & 17 deletions packages/cli/src/__tests__/runtime-host-run-command.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,82 @@ describe('Runtime Host maka run adapter', () => {
);
});

test('returns exit code 1 when a same-named tool retries a different target (live)', async () => {
const fixture = runFixture({
turnEvents: sandboxBoundaryEvents(
'turn-1',
'step-1',
'step-2',
'Partial answer',
'Read',
{ path: '/workspace/README.md' },
{ path: '/outside/secret.txt' },
),
});

const exitCode = await runFixtureCommand(fixture, ['accept same-name different target']);

assert.equal(exitCode, 1);
});

test('returns exit code 1 when a same-named tool retries a different target (durable)', async () => {
const fixture = runFixture({
graph: true,
finalMessages: sandboxBoundaryMessages(
'step-1',
'step-2',
'Read',
{ path: '/workspace/README.md' },
{ path: '/outside/secret.txt' },
),
});

const exitCode = await runFixtureCommand(fixture, [
'accept durable same-name different target',
'--graph',
]);

assert.equal(exitCode, 1);
});

test('keeps a same-target retry unresolved because the boundary cannot move (live)', async () => {
const fixture = runFixture({
turnEvents: sandboxBoundaryEvents(
'turn-1',
'step-1',
'step-2',
'Recovered answer',
'Read',
{ path: '/workspace/README.md' },
{ path: '/workspace/README.md' },
),
});

const exitCode = await runFixtureCommand(fixture, ['accept same-target retry']);

assert.equal(exitCode, 1);
});

test('returns exit code 1 when a denied widening precedes any later tool success', async () => {
const stderr: string[] = [];
const fixture = runFixture({
turnEvents: deniedWideningEvents('turn-1'),
});

const exitCode = await runFixtureCommand(
fixture,
['accept post-denial success'],
() => {},
(text) => stderr.push(text),
);

assert.equal(exitCode, 1);
assert.equal(
stderr.join(''),
'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n',
);
});

test('returns exit code 1 when reconnect restores a missed sandbox failure', async () => {
let publishReplacement = () => {};
const fixture = runFixture({
Expand Down Expand Up @@ -351,18 +427,26 @@ describe('Runtime Host maka run adapter', () => {
assert.equal(stdout.join(''), 'Final graph answer\n');
});

test('returns exit code 0 when a root Graph boundary failure recovers', async () => {
test('keeps a root Graph boundary failure unresolved even when a later same-named call succeeds', async () => {
const stdout: string[] = [];
const stderr: string[] = [];
const fixture = runFixture({
graph: true,
turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'),
});
const exitCode = await runFixtureCommand(fixture, ['recover once', '--graph'], (text) =>
stdout.push(text),
const exitCode = await runFixtureCommand(
fixture,
['recover once', '--graph'],
(text) => stdout.push(text),
(text) => stderr.push(text),
);

assert.equal(exitCode, 0);
assert.equal(stdout.join(''), 'Final graph answer\n');
assert.equal(exitCode, 1);
assert.equal(stdout.join(''), '');
assert.equal(
stderr.join(''),
'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n',
);
});

test('returns exit code 1 when a same-step Graph sibling succeeds after a sandbox failure', async () => {
Expand Down Expand Up @@ -465,7 +549,7 @@ describe('Runtime Host maka run adapter', () => {
assert.equal(observed.at(-1)?.finalOutput, 'Final graph answer');
});

test('reports a recovered sandbox boundary from live and durable Turns', async () => {
test('keeps the sandbox boundary unresolved across live and durable Turns', async () => {
const live = await observeFixtureOutcome({
turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'),
});
Expand All @@ -474,8 +558,8 @@ describe('Runtime Host maka run adapter', () => {
finalMessages: sandboxBoundaryMessages('step-1', 'step-2'),
});

assert.equal(live.sandboxBoundary, 'recovered');
assert.equal(durable.sandboxBoundary, 'recovered');
assert.equal(live.sandboxBoundary, 'unresolved');
assert.equal(durable.sandboxBoundary, 'unresolved');
});

test('leaves sandbox failures unresolved when their provider steps are unavailable', async () => {
Expand All @@ -491,7 +575,7 @@ describe('Runtime Host maka run adapter', () => {
assert.equal(durable.sandboxBoundary, 'unresolved');
});

test('returns a recovered boundary to unresolved after a later sandbox failure', async () => {
test('keeps the boundary unresolved across interleaved successes and a later sandbox failure', async () => {
const outcome = await observeFixtureOutcome({
turnEvents: sandboxFailureAfterRecoveryEvents('turn-1'),
});
Expand Down Expand Up @@ -1352,16 +1436,22 @@ function sandboxBoundaryMessages(
failureStepId: string | undefined,
successStepId: string | undefined,
successToolName = 'Read',
successArgs: unknown = {},
failureArgs: unknown = {},
): StoredMessage[] {
const sameStep = failureStepId !== undefined && failureStepId === successStepId;
return [
...graphMessages(false),
...(failureStepId === undefined ? [] : [storedToolCall('turn-2', 'tool-1', failureStepId, 5)]),
...(sameStep ? [storedToolCall('turn-2', 'tool-2', successStepId, 6, successToolName)] : []),
...(failureStepId === undefined
? []
: [storedToolCall('turn-2', 'tool-1', failureStepId, 5, 'Read', failureArgs)]),
...(sameStep
? [storedToolCall('turn-2', 'tool-2', successStepId, 6, successToolName, successArgs)]
: []),
sandboxFailureToolResult('turn-2', 7),
...(successStepId === undefined || sameStep
? []
: [storedToolCall('turn-2', 'tool-2', successStepId, 8, successToolName)]),
: [storedToolCall('turn-2', 'tool-2', successStepId, 8, successToolName, successArgs)]),
successfulToolResult('turn-2', 9),
{
type: 'turn_state',
Expand Down Expand Up @@ -1499,18 +1589,34 @@ async function* sandboxBoundaryEvents(
successStepId: string | undefined,
text: string,
successToolName = 'Read',
successArgs: unknown = {},
failureArgs: unknown = {},
): AsyncIterable<SessionEvent> {
const sameStep = failureStepId !== undefined && failureStepId === successStepId;
if (failureStepId !== undefined) yield toolStart(turnId, 'tool-1', failureStepId, 1);
if (sameStep) yield toolStart(turnId, 'tool-2', successStepId, 2, successToolName);
if (failureStepId !== undefined) {
yield toolStart(turnId, 'tool-1', failureStepId, 1, 'Read', failureArgs);
}
if (sameStep) {
yield toolStart(turnId, 'tool-2', successStepId, 2, successToolName, successArgs);
}
yield sandboxFailureToolResult(turnId, 3);
if (successStepId !== undefined && !sameStep) {
yield toolStart(turnId, 'tool-2', successStepId, 4, successToolName);
yield toolStart(turnId, 'tool-2', successStepId, 4, successToolName, successArgs);
}
yield successfulToolResult(turnId, 5);
yield* eventsFor(turnId, text, 6);
}

async function* deniedWideningEvents(turnId: string): AsyncIterable<SessionEvent> {
yield toolStart(turnId, 'tool-1', 'step-1', 1, 'Read', { path: '/outside/secret.txt' });
yield sandboxFailureToolResult(turnId, 2);
yield toolStart(turnId, 'tool-2', 'step-2', 3, 'request_sandbox_boundary');
yield successfulToolResult(turnId, 4, 'tool-2');
yield toolStart(turnId, 'tool-3', 'step-3', 5, 'Read', { path: '/workspace/README.md' });
yield successfulToolResult(turnId, 6, 'tool-3');
yield* eventsFor(turnId, 'Recovered answer', 7);
}

async function* projectedSameStepSandboxFailureEvents(turnId: string): AsyncIterable<SessionEvent> {
yield toolStart(turnId, 'tool-1', 'step-1', 1);
yield toolStart(turnId, 'tool-2', 'step-1', 2);
Expand Down Expand Up @@ -1654,6 +1760,7 @@ function toolStart(
stepId: string,
ts: number,
toolName = 'Read',
args: unknown = {},
): Extract<SessionEvent, { type: 'tool_start' }> {
return {
type: 'tool_start',
Expand All @@ -1662,7 +1769,7 @@ function toolStart(
ts,
toolUseId,
toolName,
args: {},
args,
stepId,
};
}
Expand All @@ -1673,14 +1780,15 @@ function storedToolCall(
stepId: string,
ts: number,
toolName = 'Read',
args: unknown = {},
): Extract<StoredMessage, { type: 'tool_call' }> {
return {
type: 'tool_call',
id: toolUseId,
turnId,
ts,
toolName,
args: {},
args,
stepId,
};
}
Expand Down
5 changes: 1 addition & 4 deletions packages/cli/src/activation-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -588,10 +588,7 @@ export async function runMakaActivationCli(
if (invocation?.failure?.class === 'permission_denied') {
return finish('blocked', 'permission_denied', undefined, 'grant_permission');
}
if (
(streamBoundaryFailure && invocation?.sandboxBoundary !== 'recovered') ||
invocation?.sandboxBoundary === 'unresolved'
) {
if (streamBoundaryFailure || invocation?.sandboxBoundary === 'unresolved') {
return finish('blocked', 'permission_required', undefined, 'grant_permission');
}
if (!invocation) return finish('fatal_failure', 'missing_invocation');
Expand Down
7 changes: 2 additions & 5 deletions packages/cli/src/run-command-core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ export interface MakaRunOutcome {
status: 'completed' | 'failed';
finalOutput?: string;
failure?: { class: string; message?: string };
sandboxBoundary: 'none' | 'unresolved' | 'recovered';
sandboxBoundary: 'none' | 'unresolved';
}

export interface MakaRunContextInput {
Expand Down Expand Up @@ -311,10 +311,7 @@ export async function runMakaTextCliCore(
...(parsed.options.hostProfileId ? { hostProfileId: parsed.options.hostProfileId } : {}),
...(parsed.options.projectId ? { projectId: parsed.options.projectId } : {}),
runOutcomeObserver: (result) => {
if (result.sandboxBoundary === 'recovered') {
boundaryFailureInvocationIds.delete(result.outcomeId);
unclassifiedBoundaryFailure = false;
} else if (result.sandboxBoundary === 'unresolved') {
if (result.sandboxBoundary === 'unresolved') {
boundaryFailureInvocationIds.add(result.outcomeId);
unclassifiedBoundaryFailure = false;
}
Expand Down
34 changes: 6 additions & 28 deletions packages/cli/src/runtime-host-run-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -592,14 +592,10 @@ class TurnOutcomeClassifier {
>();
readonly #unresolvedSandboxFailures = new Map<
string,
{
readonly failedStepId: string | undefined;
readonly failedToolName: string | undefined;
}
{ readonly failedStepId: string | undefined; readonly failedToolName: string | undefined }
>();
#finalOutput: string | undefined;
#terminal: TerminalOutcomeObservation | undefined;
#sandboxBoundaryRecovered = false;

constructor(outcomeId: string) {
this.#outcomeId = outcomeId;
Expand All @@ -624,29 +620,16 @@ class TurnOutcomeClassifier {
});
return;
case 'tool_result': {
const call = this.#callByToolUseId.get(observation.toolUseId);
if (observation.outcome === 'sandbox_failure') {
const call = this.#callByToolUseId.get(observation.toolUseId);
this.#unresolvedSandboxFailures.set(observation.toolUseId, {
failedStepId: call?.stepId,
failedToolName: call?.toolName,
});
return;
}
const unresolved = [...this.#unresolvedSandboxFailures.values()];
// The wire has no retry identity. A later success can only prove recovery
// when there is exactly one unresolved candidate.
if (
observation.outcome === 'success' &&
call?.toolName !== 'request_sandbox_boundary' &&
unresolved.length === 1 &&
call?.stepId !== undefined &&
unresolved[0]?.failedStepId !== undefined &&
call.stepId !== unresolved[0].failedStepId &&
call.toolName === unresolved[0].failedToolName
) {
this.#unresolvedSandboxFailures.clear();
this.#sandboxBoundaryRecovered = true;
}
// No clearing path: `maka run` denies every widening request, so the
// boundary cannot move mid-Turn and a later success cannot prove that
// a blocked call recovered. The failure stays unresolved to the end.
return;
}
}
Expand All @@ -658,12 +641,7 @@ class TurnOutcomeClassifier {
const terminal = this.#terminal;
if (!terminal && incomplete === 'pending') return undefined;
const completed = terminal?.status === 'completed';
const sandboxBoundary =
this.#unresolvedSandboxFailures.size > 0
? 'unresolved'
: this.#sandboxBoundaryRecovered
? 'recovered'
: 'none';
const sandboxBoundary = this.#unresolvedSandboxFailures.size > 0 ? 'unresolved' : 'none';
const failure =
terminal?.status === 'failed'
? terminal.failure
Expand Down