From 21ed846b2767c5ad8fc981da2e64371040ad25be Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:54:45 -0700 Subject: [PATCH 1/4] test(claude): real-claude suite passes on any signed-in machine and current model aliases Model ids come from the CLI's own model list instead of dated ids, tests that need a SessionStart hook or an effort bring their own project settings, and the supervised-stop test launches and reads its transcript under one Claude home, as acquisition does. --- .../claude/claude-structured-real-cli.test.ts | 38 +++++++++++++------ .../claude-supervised-stop-real-cli.test.ts | 16 +++++--- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index f19358e1476a..c77bf5db4467 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -78,6 +78,23 @@ function realAdapter( return adapter } +/** A temp project with its own SessionStart hook, so the hook frame never depends on the machine's real settings. */ +async function projectWithStartHook( + prefix: string, + settings: Record = {} +): Promise { + const cwd = await mkdtemp(join(tmpdir(), prefix)) + await mkdir(join(cwd, '.claude'), { recursive: true }) + await writeFile( + join(cwd, '.claude', 'settings.json'), + JSON.stringify({ + ...settings, + hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] } + }) + ) + return cwd +} + function identity(providerSessionId: string): AgentSessionJournalIdentity { return { sessionId: 'real-cli-handshake', @@ -158,7 +175,7 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => { const providerSessionId = randomUUID() const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') const events: ClaudeStructuredSessionEvent[] = [] - const cwd = await mkdtemp(join(tmpdir(), 'orca-command-init-')) + const cwd = await projectWithStartHook('orca-command-init-') await mkdir(join(cwd, '.claude', 'commands'), { recursive: true }) await writeFile( join(cwd, '.claude', 'commands', 'orca-init-catalog-proof.md'), @@ -211,7 +228,8 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => { const providerSessionId = randomUUID() const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') const events: ClaudeStructuredSessionEvent[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + const cwd = await projectWithStartHook('orca-effort-readback-', { effortLevel: 'low' }) + const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd) try { await adapter.acquire({ @@ -233,9 +251,10 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => { expect(published.filter((frame) => 'effortLevel' in frame)).toEqual([]) // Goes red if `effective.effortLevel` is renamed or dropped, which no // fixture-backed test can see. - expect(options.current.effort).toEqual(expect.any(String)) + expect(options.current.effort).toBe('low') } finally { await adapter.closeAll() + await rm(cwd, { recursive: true, force: true }) } }, 15_000 @@ -347,7 +366,9 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => { expect(frames[0]?.model).toMatch(/^claude-haiku-/) await expect( adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) - ).resolves.toMatchObject({ current: { model: 'haiku' } }) + ).resolves.toMatchObject({ + current: { model: 'haiku', confirmed: expect.arrayContaining(['model']) } + }) } finally { await adapter.closeAll() } @@ -577,14 +598,7 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => { async () => { const providerSessionId = randomUUID() const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const cwd = await mkdtemp(join(tmpdir(), 'orca-queued-stop-')) - await mkdir(join(cwd, '.claude'), { recursive: true }) - await writeFile( - join(cwd, '.claude', 'settings.json'), - JSON.stringify({ - hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] } - }) - ) + const cwd = await projectWithStartHook('orca-queued-stop-') const events: ClaudeStructuredSessionEvent[] = [] const settlements: unknown[] = [] const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd, (settlement) => diff --git a/src/main/claude/claude-supervised-stop-real-cli.test.ts b/src/main/claude/claude-supervised-stop-real-cli.test.ts index 0f21a943caea..661624b69c67 100644 --- a/src/main/claude/claude-supervised-stop-real-cli.test.ts +++ b/src/main/claude/claude-supervised-stop-real-cli.test.ts @@ -1,11 +1,13 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' -import { homedir, tmpdir } from 'node:os' +import { tmpdir } from 'node:os' import { join } from 'node:path' import { randomUUID } from 'node:crypto' import { afterEach, describe, expect, it } from 'vitest' import { runProcess } from '@orca/process-host' import { resolveSessionFilePath } from '../native-chat/session-file-resolver' import { SUPERVISED_GRACEFUL_EXIT_MS } from './claude-child-exit-proof-ladder' +import { claudeConfigDirEnvPatch } from './claude-config-dir-pin' +import { realClaudeLaunchHome } from './claude-real-cli-availability-test-support' import { openClaudeStreamJsonConnection, type ClaudeStreamJsonConnection @@ -15,14 +17,16 @@ import { claudeStructuredPermissionOptions } from './claude-structured-launch-resolution' -// Opt-in only: spends a real (haiku) turn per case. Run it in an isolated HOME with -// ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS when auth lives in a settings file. +// Opt-in only: spends a real (haiku) turn per case. Needs ORCA_REAL_CLAUDE_BIN; set +// ORCA_REAL_CLAUDE_SETTINGS only when auth lives in a settings file outside the Claude home. const CLAUDE_BIN = process.env.ORCA_REAL_CLAUDE_BIN ?? '' const enabled = process.env.ORCA_REAL_CLAUDE_SUPERVISED_STOP === '1' && process.platform !== 'win32' && CLAUDE_BIN.length > 0 const FRAMES_OUT = process.env.ORCA_REAL_CLAUDE_FRAMES_OUT +// Launch and transcript lookup share one home, as production acquisition does. +const home = realClaudeLaunchHome() const TOOL_MARKER = 'orca_supervised_stop_probe' const TOOL_PROMPT = `Use the Bash tool to run exactly this command, with no timeout argument, and nothing else: python3 -c "import time; time.sleep(120) # ${TOOL_MARKER}"` @@ -94,7 +98,8 @@ async function open(sessionId: string, cwd: string, resume: boolean, frames: Fra }, ...(resume ? { resume: sessionId } : { sessionId }) }, - cwd + cwd, + env: { ...home.env, ...claudeConfigDirEnvPatch(home.claudeConfigDir, { env: home.env }) } }, { onMessage: (message) => frames.push({ at: Date.now(), message }) } ) @@ -113,11 +118,10 @@ function userMessage(text: string): Record { } async function transcriptLines(sessionId: string): Promise { - const configDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') const path = await until( () => resolveSessionFilePath('claude', sessionId, { - claudeProjectsDir: join(configDir, 'projects') + claudeProjectsDir: join(home.claudeConfigDir, 'projects') }), 'the transcript', 15_000 From 4d5f90a038ba7ae97d96fd520ca8e28b23c03688 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:57:11 -0700 Subject: [PATCH 2/4] test(claude): one real-claude flow for the model, effort and permission pickers and the context meter Drives the live CLI through the production send path (model list, set model, apply effort, read settings), checks the turn's context report, and relaunches into a new permission mode the way Orca changes it. CI only meets these requests through stand-ins. --- ...claude-structured-real-cli-options.test.ts | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 src/main/claude/claude-structured-real-cli-options.test.ts diff --git a/src/main/claude/claude-structured-real-cli-options.test.ts b/src/main/claude/claude-structured-real-cli-options.test.ts new file mode 100644 index 000000000000..2d23880d2847 --- /dev/null +++ b/src/main/claude/claude-structured-real-cli-options.test.ts @@ -0,0 +1,237 @@ +// The SDK requests behind the model, effort and permission pickers and the context meter, against +// the live CLI: CI meets them only through stand-ins, which pin the shape we read, not the one the +// CLI sends. Runs only where a signed-in Claude CLI exists, like the rest of the real-CLI suite. + +import { randomUUID } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import type { AgentSessionJournalIdentity } from '../../shared/agent-session-journal-types' +import type { AgentSessionExecutionOptions } from '../../shared/agent-session-execution-options' +import { readAgentJournalTurn } from '../../shared/agent-session-turn-record' +import { + claudeProviderHandle, + claudeProviderHandleLeafUuid +} from '../../shared/agent-session-provider-handle-encoding' +import { + CLAUDE_STRUCTURED_BASE_OPTIONS, + claudeStructuredPermissionOptions +} from './claude-structured-launch-resolution' +import { listedModels } from './claude-structured-model-catalog' +import { + ClaudeStructuredSessionAdapter, + type ClaudeStructuredSessionEvent +} from './claude-structured-session-adapter' +import { + realClaudeAuthenticated, + realClaudeAvailable, + realClaudeCommand, + realClaudeLaunchHome +} from './claude-real-cli-availability-test-support' +import { claudeStartupSettled } from './claude-structured-session-test-support' +import { journal } from './claude-context-usage-test-support' + +const SESSION_ID = 'real-cli-options' +// The SDK's effort scale, low to high; the CLI's list order is not a contract. +const EFFORT_SCALE = ['low', 'medium', 'high', 'xhigh', 'max'] + +function identity(providerSessionId: string, leafUuid: string | null): AgentSessionJournalIdentity { + return { + sessionId: SESSION_ID, + workspaceId: 'real-cli-options-workspace', + hostId: 'local', + agent: 'claude', + providerHandle: claudeProviderHandle(providerSessionId, leafUuid) + } +} + +async function until(check: () => boolean, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (check()) { + return true + } + await new Promise((resolve) => setTimeout(resolve, 200)) + } + return check() +} + +// Inits name a 1M-window model with `[1m]`; context reports and responses drop it. +const baseModelId = (model: string): string => + model + .replace(/\[[^\]]*\]$/u, '') + .trim() + .toLowerCase() + +describe.skipIf(!realClaudeAvailable)('Claude structured real CLI options', () => { + it.skipIf(!realClaudeAuthenticated)( + 'applies a message model and effort, reports context usage, and relaunches into a new permission mode', + async () => { + const providerSessionId = randomUUID() + const { claudeConfigDir, env } = realClaudeLaunchHome() + const cwd = await mkdtemp(join(tmpdir(), 'orca-real-options-')) + const events: ClaudeStructuredSessionEvent[] = [] + const record = journal() + const persisted: { leafUuid: string | null } = { leafUuid: null } + let launches = 0 + const adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async ({ identity: launchIdentity, permissionMode }) => { + const permission = claudeStructuredPermissionOptions(permissionMode ?? 'ask') + // A relaunch resumes the transcript the first launch wrote, as the host's resolver does. + const resumes = launches++ > 0 + return { + pathToClaudeCodeExecutable: realClaudeCommand, + options: { + ...CLAUDE_STRUCTURED_BASE_OPTIONS, + ...permission, + extraArgs: { ...CLAUDE_STRUCTURED_BASE_OPTIONS.extraArgs, ...permission.extraArgs }, + ...(resumes ? { resume: providerSessionId } : { sessionId: providerSessionId }) + }, + cwd, + env, + claudeConfigDir, + providerSessionId, + resumeLeafUuid: + resumes && launchIdentity.providerHandle + ? claudeProviderHandleLeafUuid(launchIdentity.providerHandle) + : null, + resumesTranscript: resumes, + continuesChain: resumes + } + }, + onEvent: (event) => events.push(event), + persistHandle: async (handle) => { + persisted.leafUuid = handle.leafUuid + }, + readProcessStartTime: async () => 1 + }) + const frames = (from: number) => + events.slice(from).flatMap((event) => (event.type === 'message' ? [event.message] : [])) + const inits = (from: number) => + frames(from).filter((message) => message.type === 'system' && message.subtype === 'init') + const results = (from: number) => frames(from).filter((message) => message.type === 'result') + const send = ( + clientMessageId: string, + text: string, + fence: number, + executionOptions?: AgentSessionExecutionOptions + ) => + adapter.dispatch({ + sessionId: SESSION_ID, + clientMessageId, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, + requestedAt: Date.now(), + fence, + ...(executionOptions ? { executionOptions } : {}) + }) + const contextReport = () => + record + .items() + .flatMap((row) => { + const used = readAgentJournalTurn(row.body)?.contextUsage?.used + return used?.kind === 'report' ? [used] : [] + }) + .at(-1) + + try { + await adapter.acquire({ + identity: identity(providerSessionId, null), + fence: 1, + spawnToken: 'real-cli-options-1', + permissionMode: 'ask', + events: record.sink + }) + await claudeStartupSettled(adapter, SESSION_ID) + + // A row without `value` or `resolvedModel` silently turns off the admit check, the effort + // guard and model confirmation. + const connection = adapter['sessions'].get(SESSION_ID)?.connection + if (!connection) { + throw new Error('no live Claude connection') + } + const listed = listedModels({ models: await connection.supportedModels() }) + const rows = ['haiku', 'sonnet'].map((alias) => listed.find(({ id }) => id === alias)) + for (const row of rows) { + expect(row?.resolvedModel).toEqual(expect.stringMatching(/\S/u)) + } + const opened = await adapter.readOptions({ sessionId: SESSION_ID, fence: 1 }) + expect(opened.catalogListing?.models.map(({ id }) => id)).toContain('haiku') + + const pick = rows.find((row) => row !== undefined && row.efforts.length > 0) + const level = pick?.efforts + .map(({ value }) => value) + .filter((value) => EFFORT_SCALE.includes(value)) + .sort((left, right) => EFFORT_SCALE.indexOf(left) - EFFORT_SCALE.indexOf(right))[0] + if (!pick?.resolvedModel || !level) { + throw new Error('neither haiku nor sonnet lists an effort level') + } + + // The production send path: list_models, set_model, apply_flag_settings and get_settings. + const firstTurn = events.length + await expect( + send('real-cli-options-1', 'Reply with exactly: OK. Do not use any tools.', 1, { + model: pick.id, + effort: level + }) + ).resolves.toEqual({ state: 'admitted' }) + expect(await until(() => results(firstTurn).length > 0, 90_000)).toBe(true) + expect(results(firstTurn)[0]).toMatchObject({ subtype: 'success', is_error: false }) + expect(inits(firstTurn)[0]?.model).toBe(pick.resolvedModel) + // Effort is read only through get_settings; no frame carries it, init included. + expect(frames(0).filter((frame) => 'effortLevel' in frame)).toEqual([]) + await expect( + adapter.readOptions({ sessionId: SESSION_ID, fence: 1 }) + ).resolves.toMatchObject({ + current: { + model: pick.id, + effort: level, + confirmed: expect.arrayContaining(['model', 'effort']) + } + }) + + // The turn end asks get_context_usage; an answer Orca cannot read blanks the meter silently. + expect(await until(() => contextReport() !== undefined, 10_000)).toBe(true) + const report = contextReport() + if (!report) { + throw new Error('no context report on any turn row') + } + expect(report.usedTokens).toBeGreaterThan(0) + expect(report.windowTokens).toBeGreaterThanOrEqual(report.usedTokens) + expect(report.model).toMatch(/\S/u) + expect(baseModelId(report.model)).toBe(baseModelId(pick.resolvedModel)) + + // Orca never sends set_permission_mode: it retires the child and resumes it under the new + // flag with the saved picks, so the CLI must read its mode at launch. + await adapter.closeSession(SESSION_ID) + const resumedTurn = events.length + await adapter.acquire({ + identity: identity(providerSessionId, persisted.leafUuid), + fence: 2, + spawnToken: 'real-cli-options-2', + permissionMode: 'accept-edits', + options: { model: pick.id, effort: level }, + events: record.sink + }) + await claudeStartupSettled(adapter, SESSION_ID) + await expect(send('real-cli-options-2', 'Reply with exactly: OK.', 2)).resolves.toEqual({ + state: 'admitted' + }) + expect(await until(() => results(resumedTurn).length > 0, 90_000)).toBe(true) + expect(results(resumedTurn)[0]).toMatchObject({ subtype: 'success', is_error: false }) + expect(inits(resumedTurn)[0]).toMatchObject({ + session_id: providerSessionId, + permissionMode: 'acceptEdits', + model: pick.resolvedModel + }) + await expect( + adapter.readOptions({ sessionId: SESSION_ID, fence: 2 }) + ).resolves.toMatchObject({ current: { effort: level } }) + } finally { + await adapter.closeAll() + await rm(cwd, { recursive: true, force: true }) + } + }, + 180_000 + ) +}) From 07576e1c779f00a9bae2efdd330a57563e987a7d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:01:44 -0700 Subject: [PATCH 3/4] test(claude): accept the ran options a send now reports Since #30561 a Claude send resolves with the options it ran under; the stop test only needs the send admitted, and the options flow checks the ran options are exactly the per-message pick. --- src/main/claude/claude-structured-real-cli-options.test.ts | 7 +++++-- src/main/claude/claude-structured-real-cli.test.ts | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/claude/claude-structured-real-cli-options.test.ts b/src/main/claude/claude-structured-real-cli-options.test.ts index 2d23880d2847..e6002b507c6c 100644 --- a/src/main/claude/claude-structured-real-cli-options.test.ts +++ b/src/main/claude/claude-structured-real-cli-options.test.ts @@ -168,13 +168,14 @@ describe.skipIf(!realClaudeAvailable)('Claude structured real CLI options', () = } // The production send path: list_models, set_model, apply_flag_settings and get_settings. + // `ranOptions` is what the chat records as the picks this message actually ran with. const firstTurn = events.length await expect( send('real-cli-options-1', 'Reply with exactly: OK. Do not use any tools.', 1, { model: pick.id, effort: level }) - ).resolves.toEqual({ state: 'admitted' }) + ).resolves.toEqual({ state: 'admitted', ranOptions: { model: pick.id, effort: level } }) expect(await until(() => results(firstTurn).length > 0, 90_000)).toBe(true) expect(results(firstTurn)[0]).toMatchObject({ subtype: 'success', is_error: false }) expect(inits(firstTurn)[0]?.model).toBe(pick.resolvedModel) @@ -214,7 +215,9 @@ describe.skipIf(!realClaudeAvailable)('Claude structured real CLI options', () = events: record.sink }) await claudeStartupSettled(adapter, SESSION_ID) - await expect(send('real-cli-options-2', 'Reply with exactly: OK.', 2)).resolves.toEqual({ + await expect( + send('real-cli-options-2', 'Reply with exactly: OK.', 2) + ).resolves.toMatchObject({ state: 'admitted' }) expect(await until(() => results(resumedTurn).length > 0, 90_000)).toBe(true) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index c77bf5db4467..42a02b46bdf3 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -580,9 +580,9 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => { expect(replyStarted(first)).toBe(false) const second = events.length - await expect(send('real-cli-stop-2', 'Reply with the single word ok.')).resolves.toEqual({ - state: 'admitted' - }) + await expect( + send('real-cli-stop-2', 'Reply with the single word ok.') + ).resolves.toMatchObject({ state: 'admitted' }) await expect(result(second)).resolves.toMatchObject({ subtype: 'success' }) } finally { await adapter.closeAll() From 8a6580a60e50d4ef9ed5e50ac8317e834ff3a11e Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:00:19 -0700 Subject: [PATCH 4/4] test(claude): an alias pick's dispatch report may omit the model the CLI resolved --- ...claude-structured-real-cli-options.test.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/claude/claude-structured-real-cli-options.test.ts b/src/main/claude/claude-structured-real-cli-options.test.ts index e6002b507c6c..decec04d8849 100644 --- a/src/main/claude/claude-structured-real-cli-options.test.ts +++ b/src/main/claude/claude-structured-real-cli-options.test.ts @@ -168,14 +168,19 @@ describe.skipIf(!realClaudeAvailable)('Claude structured real CLI options', () = } // The production send path: list_models, set_model, apply_flag_settings and get_settings. - // `ranOptions` is what the chat records as the picks this message actually ran with. + // `ranOptions` carries only what the CLI reported; it names the resolved id, so an alias + // pick reports no model and the recorder falls back to the message's own pick. const firstTurn = events.length - await expect( - send('real-cli-options-1', 'Reply with exactly: OK. Do not use any tools.', 1, { - model: pick.id, - effort: level - }) - ).resolves.toEqual({ state: 'admitted', ranOptions: { model: pick.id, effort: level } }) + const outcome = await send( + 'real-cli-options-1', + 'Reply with exactly: OK. Do not use any tools.', + 1, + { model: pick.id, effort: level } + ) + expect(outcome).toMatchObject({ state: 'admitted', ranOptions: { effort: level } }) + expect([undefined, pick.id]).toContain( + 'ranOptions' in outcome ? outcome.ranOptions?.model : undefined + ) expect(await until(() => results(firstTurn).length > 0, 90_000)).toBe(true) expect(results(firstTurn)[0]).toMatchObject({ subtype: 'success', is_error: false }) expect(inits(firstTurn)[0]?.model).toBe(pick.resolvedModel)