From 6ed04f16db50c38a6e28f14f23c679ed07684e53 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 14:05:08 -0700 Subject: [PATCH 1/4] test(claude): add an explicit opt-in gate for real Claude CLI tests An installed, signed-in claude is not consent to spend turns on that account. The gate skips before probing the binary unless ORCA_REAL_CLAUDE_CLI_TEST=1. --- .../claude/claude-real-cli-test-gate.test.ts | 85 +++++++++++++++++++ src/main/claude/claude-real-cli-test-gate.ts | 52 ++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 src/main/claude/claude-real-cli-test-gate.test.ts create mode 100644 src/main/claude/claude-real-cli-test-gate.ts diff --git a/src/main/claude/claude-real-cli-test-gate.test.ts b/src/main/claude/claude-real-cli-test-gate.test.ts new file mode 100644 index 000000000000..508fd6af08ed --- /dev/null +++ b/src/main/claude/claude-real-cli-test-gate.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it, vi } from 'vitest' +import { + REAL_CLAUDE_CLI_TEST_ENV, + resolveRealClaudeCliGate, + type ClaudeCliProbeResult +} from './claude-real-cli-test-gate' + +/** A claude that is installed and signed in, as far as the probes can tell. */ +function signedInClaude() { + return vi.fn((args: readonly string[]): ClaudeCliProbeResult => { + if (args[0] === '--version') { + return { status: 0, stdout: '2.1.0 (Claude Code)\n' } + } + return { + status: 0, + stdout: JSON.stringify({ loggedIn: true, projectsDirectory: '/home/dev/.claude/projects' }) + } + }) +} + +describe('resolveRealClaudeCliGate', () => { + it.each([undefined, '', '0', 'true'])( + 'skips without probing a signed-in claude when %s is the opt-in value', + (value) => { + const runClaude = signedInClaude() + + const gate = resolveRealClaudeCliGate({ [REAL_CLAUDE_CLI_TEST_ENV]: value }, runClaude) + + expect(gate).toEqual({ + skipReason: 'set ORCA_REAL_CLAUDE_CLI_TEST=1 to run against the real claude CLI', + authStatus: null + }) + expect(runClaude).not.toHaveBeenCalled() + } + ) + + it('runs with the CLI account report once opted in and signed in', () => { + const runClaude = signedInClaude() + + const gate = resolveRealClaudeCliGate({ [REAL_CLAUDE_CLI_TEST_ENV]: '1' }, runClaude) + + expect(gate).toEqual({ + skipReason: null, + authStatus: { loggedIn: true, projectsDirectory: '/home/dev/.claude/projects' } + }) + expect(runClaude.mock.calls).toEqual([[['--version']], [['auth', 'status', '--json']]]) + }) + + it('still skips when opted in but no claude binary answers', () => { + const runClaude = vi.fn((): ClaudeCliProbeResult => ({ status: null, stdout: '' })) + + const gate = resolveRealClaudeCliGate({ [REAL_CLAUDE_CLI_TEST_ENV]: '1' }, runClaude) + + expect(gate).toEqual({ skipReason: '`claude --version` failed', authStatus: null }) + expect(runClaude).toHaveBeenCalledTimes(1) + }) + + it.each([ + ['a failed auth probe', { status: 1, stdout: '' }], + ['unparseable auth output', { status: 0, stdout: 'not json' }] + ])('runs with no account report after %s', (_label, authResult) => { + const runClaude = vi.fn((args: readonly string[]): ClaudeCliProbeResult => + args[0] === '--version' ? { status: 0, stdout: '2.1.0\n' } : authResult + ) + + // The suite's signed-out case still runs; its signed-in cases skip on a null report. + expect(resolveRealClaudeCliGate({ [REAL_CLAUDE_CLI_TEST_ENV]: '1' }, runClaude)).toEqual({ + skipReason: null, + authStatus: null + }) + }) + + it('keeps only the account fields it understands', () => { + const runClaude = vi.fn((args: readonly string[]): ClaudeCliProbeResult => + args[0] === '--version' + ? { status: 0, stdout: '2.1.0\n' } + : { status: 0, stdout: JSON.stringify({ loggedIn: 'yes', projectsDirectory: 7 }) } + ) + + expect(resolveRealClaudeCliGate({ [REAL_CLAUDE_CLI_TEST_ENV]: '1' }, runClaude)).toEqual({ + skipReason: null, + authStatus: {} + }) + }) +}) diff --git a/src/main/claude/claude-real-cli-test-gate.ts b/src/main/claude/claude-real-cli-test-gate.ts new file mode 100644 index 000000000000..de25ef87bf47 --- /dev/null +++ b/src/main/claude/claude-real-cli-test-gate.ts @@ -0,0 +1,52 @@ +/** Opt-in for tests that drive the real `claude` CLI on this machine's signed-in account. */ +export const REAL_CLAUDE_CLI_TEST_ENV = 'ORCA_REAL_CLAUDE_CLI_TEST' + +export type ClaudeCliProbeResult = { status: number | null; stdout: string } + +export type RealClaudeAuthStatus = { loggedIn?: boolean; projectsDirectory?: string } + +export type RealClaudeCliGate = { + /** Null when the real-CLI suite may run; otherwise why it is skipped. */ + skipReason: string | null + /** The CLI's own account report; null when skipped or unreadable. */ + authStatus: RealClaudeAuthStatus | null +} + +function parseAuthStatus(stdout: string): RealClaudeAuthStatus | null { + let parsed: unknown + try { + parsed = JSON.parse(stdout) + } catch { + return null + } + if (!parsed || typeof parsed !== 'object') { + return null + } + const status: RealClaudeAuthStatus = {} + if ('loggedIn' in parsed && typeof parsed.loggedIn === 'boolean') { + status.loggedIn = parsed.loggedIn + } + if ('projectsDirectory' in parsed && typeof parsed.projectsDirectory === 'string') { + status.projectsDirectory = parsed.projectsDirectory + } + return status +} + +export function resolveRealClaudeCliGate( + env: NodeJS.ProcessEnv, + runClaude: (args: readonly string[]) => ClaudeCliProbeResult +): RealClaudeCliGate { + // Why before any probe: an installed, signed-in CLI is not consent to spend turns on that + // account and write transcripts into its home from a broad local test run. + if (env[REAL_CLAUDE_CLI_TEST_ENV] !== '1') { + return { + skipReason: `set ${REAL_CLAUDE_CLI_TEST_ENV}=1 to run against the real claude CLI`, + authStatus: null + } + } + if (runClaude(['--version']).status !== 0) { + return { skipReason: '`claude --version` failed', authStatus: null } + } + const auth = runClaude(['auth', 'status', '--json']) + return { skipReason: null, authStatus: auth.status === 0 ? parseAuthStatus(auth.stdout) : null } +} From 31c413f0cb78046eba375ebc4ee0612a30b8780d Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 14:05:21 -0700 Subject: [PATCH 2/4] test(claude): run the real Claude CLI suite only when opted in The suite used to run whenever `claude --version` succeeded and `claude auth status` reported a login, so any broad local vitest run spent real turns on the developer's account and wrote transcripts into their Claude home. It now also requires ORCA_REAL_CLAUDE_CLI_TEST=1, and the skipped suite's title names the variable. --- .../claude/claude-structured-real-cli.test.ts | 36 ++++++------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index cfd4c8c33eab..86d54683e882 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -8,6 +8,7 @@ import type { AgentSessionJournalIdentity } from '../../shared/agent-session-jou import { resolveClaudeCommand } from '../codex-cli/command' import { resolveSessionFilePath } from '../native-chat/session-file-resolver' import { getSpawnArgsForWindows } from '../win32-utils' +import { resolveRealClaudeCliGate } from './claude-real-cli-test-gate' import { CLAUDE_STRUCTURED_BASE_OPTIONS } from './claude-structured-launch-resolution' import { ClaudeStructuredSessionAdapter, @@ -16,35 +17,20 @@ import { import type { ClaudeStructuredSessionAdapterDeps } from './claude-structured-session-state' const command = resolveClaudeCommand() -const versionLaunch = getSpawnArgsForWindows(command, ['--version']) -const realClaudeAvailable = - spawnSync(versionLaunch.spawnCmd, versionLaunch.spawnArgs, { - stdio: 'ignore', - windowsHide: true, - timeout: 5_000 - }).status === 0 -const authStatusLaunch = getSpawnArgsForWindows(command, ['auth', 'status', '--json']) -/** The CLI's own account report — the only source of truth for where it writes that - * is not derived from Orca's own path expressions. */ -const realClaudeAuthStatus = (() => { - if (!realClaudeAvailable) { - return null - } - const result = spawnSync(authStatusLaunch.spawnCmd, authStatusLaunch.spawnArgs, { +const gate = resolveRealClaudeCliGate(process.env, (args) => { + const launch = getSpawnArgsForWindows(command, [...args]) + const result = spawnSync(launch.spawnCmd, launch.spawnArgs, { encoding: 'utf8', windowsHide: true, timeout: 5_000 }) - if (result.status !== 0) { - return null - } - try { - return JSON.parse(result.stdout) as { loggedIn?: boolean; projectsDirectory?: string } - } catch { - return null - } -})() + return { status: result.status, stdout: result.stdout ?? '' } +}) +/** The CLI's own account report — the only source of truth for where it writes that + * is not derived from Orca's own path expressions. */ +const realClaudeAuthStatus = gate.authStatus const realClaudeAuthenticated = realClaudeAuthStatus?.loggedIn === true +const suiteTitle = `Claude structured real CLI handshake${gate.skipReason ? ` (skipped: ${gate.skipReason})` : ''}` function realAdapter( providerSessionId: string, @@ -105,7 +91,7 @@ async function waitForResolvedTranscript( } } -describe.skipIf(!realClaudeAvailable)('Claude structured real CLI handshake', () => { +describe.skipIf(gate.skipReason !== null)(suiteTitle, () => { it.skipIf(!realClaudeAuthenticated)( 'proves a pre-minted session before the first user message', async () => { From 79bc8f618ecf53dc44b69ca39bd5b4c965249586 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:15:18 -0700 Subject: [PATCH 3/4] docs(agents): say when to run the opt-in real Claude CLI suite --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 8ac5cd365cc3..5df9ee33a8f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,6 +47,7 @@ Avoid type assertions except `as const`. Unavoidable casts need a line-specific - **Test**: `pnpm test [path/to/file.test.ts]` - **Lint**: `oxlint`, or `pnpm run check:code-quality:changed` for changed files (full `pnpm lint` is slow); format with `pnpm format` - **Design system**: `pnpm run lint:design-system` for the full renderer report (not a gate); the changed-lines gate above is what CI enforces +- **Real Claude CLI**: when you change Claude structured-session code (`src/main/claude/claude-structured-*`), run `ORCA_REAL_CLAUDE_CLI_TEST=1 pnpm test src/main/claude/claude-structured-real-cli.test.ts`; it uses your real Claude login # Writing Pull Requests From 1d2ce27f7652d80333446ee709ed58ed0f858ade Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:45:09 -0700 Subject: [PATCH 4/4] test(claude): name the skipped real-CLI suite without re-indenting it --- .../claude/claude-structured-real-cli.test.ts | 694 +++++++++--------- 1 file changed, 346 insertions(+), 348 deletions(-) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index 2c7daafae946..677d2de4cb9e 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -20,6 +20,7 @@ import { import type { ClaudeStructuredSessionAdapterDeps } from './claude-structured-session-state' const command = realClaudeCommand +const suiteTitle = `Claude structured real CLI handshake${realClaudeCliGate.skipReason ? ` (skipped: ${realClaudeCliGate.skipReason})` : ''}` function realAdapter( providerSessionId: string, @@ -80,384 +81,381 @@ async function waitForResolvedTranscript( } } -describe.skipIf(!realClaudeAvailable)( - `Claude structured real CLI handshake${realClaudeCliGate.skipReason ? ` (skipped: ${realClaudeCliGate.skipReason})` : ''}`, - () => { - it.skipIf(!realClaudeAuthenticated)( - 'proves a pre-minted session before the first user message', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const cwd = await mkdtemp(join(tmpdir(), 'orca-command-init-')) - await mkdir(join(cwd, '.claude', 'commands'), { recursive: true }) - await writeFile( - join(cwd, '.claude', 'commands', 'orca-init-catalog-proof.md'), - '---\ndescription: Initialization catalog proof\n---\nReply with OK.\n' - ) - const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd) - - try { - const acquisition = await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli' - }) - const observedSubtypes = events.flatMap((event) => - event.type === 'message' ? [event.message.subtype] : [] - ) - - expect(acquisition.link.handle).toMatchObject({ - provider: 'claude', - sessionId: providerSessionId, - // Init/SessionStart UUIDs are protocol frames, not resumable - // main-transcript leaves; no cursor exists before the first user turn. - leafUuid: null - }) - expect(observedSubtypes).toContain('hook_started') - expect(adapter.readCommands('real-cli-handshake')).toContainEqual({ - name: 'orca-init-catalog-proof', - kind: 'command', - kindUnspecified: true - }) - expect( - adapter.readCommands('real-cli-handshake')?.some(({ name }) => name === 'help') - ).toBe(false) - } finally { - await adapter.closeAll() - await rm(cwd, { recursive: true, force: true }) - } - }, - 10_000 - ) +describe.skipIf(!realClaudeAvailable)(suiteTitle, () => { + it.skipIf(!realClaudeAuthenticated)( + 'proves a pre-minted session before the first user message', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const cwd = await mkdtemp(join(tmpdir(), 'orca-command-init-')) + await mkdir(join(cwd, '.claude', 'commands'), { recursive: true }) + await writeFile( + join(cwd, '.claude', 'commands', 'orca-init-catalog-proof.md'), + '---\ndescription: Initialization catalog proof\n---\nReply with OK.\n' + ) + const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd) - // Unit tests can only pin the shape we read, which is exactly how the blank - // Effort pill survived every gate: the fixture invented an `effortLevel` on a - // frame the CLI does not send. This asserts both halves against the live - // binary — that get_settings reports the effort, and that init does not. - it.skipIf(!realClaudeAuthenticated)( - 'reports the current effort through get_settings and never on the init frame', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + try { + const acquisition = await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli' + }) + const observedSubtypes = events.flatMap((event) => + event.type === 'message' ? [event.message.subtype] : [] + ) - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-effort' - }) - const published = events.flatMap((event) => - event.type === 'message' ? [event.message] : [] - ) - const options = await adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) + expect(acquisition.link.handle).toMatchObject({ + provider: 'claude', + sessionId: providerSessionId, + // Init/SessionStart UUIDs are protocol frames, not resumable + // main-transcript leaves; no cursor exists before the first user turn. + leafUuid: null + }) + expect(observedSubtypes).toContain('hook_started') + expect(adapter.readCommands('real-cli-handshake')).toContainEqual({ + name: 'orca-init-catalog-proof', + kind: 'command', + kindUnspecified: true + }) + expect( + adapter.readCommands('real-cli-handshake')?.some(({ name }) => name === 'help') + ).toBe(false) + } finally { + await adapter.closeAll() + await rm(cwd, { recursive: true, force: true }) + } + }, + 10_000 + ) - expect(published.length).toBeGreaterThan(0) - // Not just the init frame: no frame the CLI publishes carries an effort - // at all. Goes red the day one does, which is when the simpler fix - // becomes available. Which frame proves the session varies by host, so - // this asserts over all of them rather than picking one. - expect(published.filter((frame) => 'effortLevel' in frame)).toEqual([]) - // Goes red if `effective.effortLevel` is renamed or dropped, which no - // fixture-backed test can see. - expect(options.current.effort).toEqual(expect.any(String)) - } finally { - await adapter.closeAll() - } - }, - 15_000 - ) + // Unit tests can only pin the shape we read, which is exactly how the blank + // Effort pill survived every gate: the fixture invented an `effortLevel` on a + // frame the CLI does not send. This asserts both halves against the live + // binary — that get_settings reports the effort, and that init does not. + it.skipIf(!realClaudeAuthenticated)( + 'reports the current effort through get_settings and never on the init frame', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events) - // Mobile native chat never reads the structured journal — it reads the CLI's own - // transcript through native-chat/session-file-resolver.ts. So this resolves the way - // transcript-read-cache.ts:104 does, with NO root override, and checks the answer - // against the root the CLI itself reports. Deriving the expected root from Orca's own - // `CLAUDE_CONFIG_DIR || ~/.claude` expression — the same one the code under test uses — - // would move both sides together and stay green in exactly the environment that - // blacks mobile out. - // The turn is what creates the file: an init-only handshake writes nothing. - it.skipIf(!realClaudeAuthenticated || !realClaudeAuthStatus?.projectsDirectory)( - 'writes its transcript where the mobile session-file resolver looks for it', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const adapter = realAdapter(providerSessionId, claudeConfigDir) - const cliProjectsDir = realClaudeAuthStatus?.projectsDirectory as string + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-effort' + }) + const published = events.flatMap((event) => + event.type === 'message' ? [event.message] : [] + ) + const options = await adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) - let transcriptPath: string | null = null - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-transcript' - }) - await adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId: 'real-cli-transcript-1', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, - fence: 1 - }) - transcriptPath = await waitForResolvedTranscript(providerSessionId) - } finally { - await adapter.closeAll() - } + expect(published.length).toBeGreaterThan(0) + // Not just the init frame: no frame the CLI publishes carries an effort + // at all. Goes red the day one does, which is when the simpler fix + // becomes available. Which frame proves the session varies by host, so + // this asserts over all of them rather than picking one. + expect(published.filter((frame) => 'effortLevel' in frame)).toEqual([]) + // Goes red if `effective.effortLevel` is renamed or dropped, which no + // fixture-backed test can see. + expect(options.current.effort).toEqual(expect.any(String)) + } finally { + await adapter.closeAll() + } + }, + 15_000 + ) - expect(transcriptPath).not.toBeNull() - expect(basename(transcriptPath ?? '')).toBe(`${providerSessionId}.jsonl`) - // `//.jsonl` - expect(relative(cliProjectsDir, transcriptPath ?? '').split(/[\\/]/)).toHaveLength(2) - // And the pinned account home is that same root, so the host-side leaf recovery - // (structured-claude-runtime-adapter.ts:64) and mobile agree. - expect(join(claudeConfigDir, 'projects')).toBe(cliProjectsDir) - }, - 45_000 - ) + // Mobile native chat never reads the structured journal — it reads the CLI's own + // transcript through native-chat/session-file-resolver.ts. So this resolves the way + // transcript-read-cache.ts:104 does, with NO root override, and checks the answer + // against the root the CLI itself reports. Deriving the expected root from Orca's own + // `CLAUDE_CONFIG_DIR || ~/.claude` expression — the same one the code under test uses — + // would move both sides together and stay green in exactly the environment that + // blacks mobile out. + // The turn is what creates the file: an init-only handshake writes nothing. + it.skipIf(!realClaudeAuthenticated || !realClaudeAuthStatus?.projectsDirectory)( + 'writes its transcript where the mobile session-file resolver looks for it', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const adapter = realAdapter(providerSessionId, claudeConfigDir) + const cliProjectsDir = realClaudeAuthStatus?.projectsDirectory as string - // The model half of the same lesson: a fixture can only pin the shape we read. - // set_model answers success for a model it never resolves — a nonexistent id is - // accepted and only fails once a turn runs — so the CLI's own report is the only - // adoption evidence, and it arrives on the init frame that opens each turn. This - // asserts that frame carries the resolved model against the live binary; it goes - // red the day the CLI stops reporting it, which is the day the confirmation - // silently degrades to echoing back whatever Orca sent. - it.skipIf(!realClaudeAuthenticated)( - 'reports the model it adopted on the init frame that opens each turn', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + let transcriptPath: string | null = null + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-transcript' + }) + await adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId: 'real-cli-transcript-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, + fence: 1 + }) + transcriptPath = await waitForResolvedTranscript(providerSessionId) + } finally { + await adapter.closeAll() + } - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-model' - }) - await adapter.setOption({ - sessionId: 'real-cli-handshake', - key: 'model', - value: 'haiku', - fence: 1 - }) - const before = events.length - await adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId: 'real-cli-model-1', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Say ok' }] }, - fence: 1 - }) - const deadline = Date.now() + 60_000 - let frames: Record[] = [] - for (;;) { - frames = events - .slice(before) - .flatMap((event) => - event.type === 'message' && - event.message.type === 'system' && - event.message.subtype === 'init' - ? [event.message] - : [] - ) - if (frames.length > 0 || Date.now() >= deadline) { - break - } - await new Promise((resolve) => setTimeout(resolve, 250)) - } + expect(transcriptPath).not.toBeNull() + expect(basename(transcriptPath ?? '')).toBe(`${providerSessionId}.jsonl`) + // `//.jsonl` + expect(relative(cliProjectsDir, transcriptPath ?? '').split(/[\\/]/)).toHaveLength(2) + // And the pinned account home is that same root, so the host-side leaf recovery + // (structured-claude-runtime-adapter.ts:64) and mobile agree. + expect(join(claudeConfigDir, 'projects')).toBe(cliProjectsDir) + }, + 45_000 + ) - expect(frames).not.toHaveLength(0) - // Both halves: the field exists, and it names the model the picker asked - // for in the catalog's resolved shape rather than the id Orca sent. - expect(frames[0]?.model).toEqual(expect.any(String)) - expect(frames[0]?.model).toBe('claude-haiku-4-5-20251001') - await expect( - adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) - ).resolves.toMatchObject({ current: { model: 'haiku' } }) - } finally { - await adapter.closeAll() - } - }, - 90_000 - ) + // The model half of the same lesson: a fixture can only pin the shape we read. + // set_model answers success for a model it never resolves — a nonexistent id is + // accepted and only fails once a turn runs — so the CLI's own report is the only + // adoption evidence, and it arrives on the init frame that opens each turn. This + // asserts that frame carries the resolved model against the live binary; it goes + // red the day the CLI stops reporting it, which is the day the confirmation + // silently degrades to echoing back whatever Orca sent. + it.skipIf(!realClaudeAuthenticated)( + 'reports the model it adopted on the init frame that opens each turn', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events) - // The window a Stop naming no turn exists for: Orca has written the message, and Claude has not - // started its reply, so no turn id exists. Only the live binary can say the interrupt lands there. - it.skipIf(!realClaudeAuthenticated)( - 'stops a message interrupted before its reply starts, and takes the next one', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events) - const frames = (from: number) => - events.slice(from).flatMap((event) => (event.type === 'message' ? [event.message] : [])) - const replyStarted = (from: number) => - frames(from).some( - (frame) => - frame.type === 'stream_event' && - typeof frame.event === 'object' && - frame.event !== null && - 'type' in frame.event && - frame.event.type === 'message_start' - ) - const result = async (from: number) => { - const deadline = Date.now() + 60_000 - for (;;) { - const found = frames(from).find((frame) => frame.type === 'result') - if (found || Date.now() >= deadline) { - return found - } - await new Promise((resolve) => setTimeout(resolve, 100)) + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-model' + }) + await adapter.setOption({ + sessionId: 'real-cli-handshake', + key: 'model', + value: 'haiku', + fence: 1 + }) + const before = events.length + await adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId: 'real-cli-model-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Say ok' }] }, + fence: 1 + }) + const deadline = Date.now() + 60_000 + let frames: Record[] = [] + for (;;) { + frames = events + .slice(before) + .flatMap((event) => + event.type === 'message' && + event.message.type === 'system' && + event.message.subtype === 'init' + ? [event.message] + : [] + ) + if (frames.length > 0 || Date.now() >= deadline) { + break } + await new Promise((resolve) => setTimeout(resolve, 250)) } - const send = (clientMessageId: string, text: string) => - adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId, - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, - fence: 1 - }) - - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-stop' - }) - const first = events.length - await expect( - send('real-cli-stop-1', 'Count from 1 to 300, one number per line, and nothing else.') - ).resolves.toEqual({ state: 'admitted' }) - const startedBeforeStop = replyStarted(first) - - await expect( - adapter.cancelTurn({ sessionId: 'real-cli-handshake', fence: 1 }) - ).resolves.toEqual({ - cancelled: true - }) - const stopped = await result(first) - - expect(startedBeforeStop).toBe(false) - // Claude 2.1.280 echoes the message, then ends that turn as interrupted without replying. - expect(stopped).toMatchObject({ subtype: 'error_during_execution' }) - expect(replyStarted(first)).toBe(false) - const second = events.length - await expect(send('real-cli-stop-2', 'Reply with the single word ok.')).resolves.toEqual({ - state: 'admitted' - }) - await expect(result(second)).resolves.toMatchObject({ subtype: 'success' }) - } finally { - await adapter.closeAll() - } - }, - 150_000 - ) + expect(frames).not.toHaveLength(0) + // Both halves: the field exists, and it names the model the picker asked + // for in the catalog's resolved shape rather than the id Orca sent. + expect(frames[0]?.model).toEqual(expect.any(String)) + expect(frames[0]?.model).toBe('claude-haiku-4-5-20251001') + await expect( + adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) + ).resolves.toMatchObject({ current: { model: 'haiku' } }) + } finally { + await adapter.closeAll() + } + }, + 90_000 + ) - // Orca installs a SessionStart hook, so its frame proves most real starts before the turn's - // system/init, the only frame that says this CLI can cancel what it queued. - it.skipIf(!realClaudeAuthenticated)( - 'withdraws a follow-up queued behind a turn that is stopped, behind a SessionStart hook', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const cwd = await mkdtemp(join(tmpdir(), 'orca-queued-stop-')) - await mkdir(join(cwd, '.claude'), { recursive: true }) - await writeFile( - join(cwd, '.claude', 'settings.json'), - JSON.stringify({ - hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] } - }) - ) - const events: ClaudeStructuredSessionEvent[] = [] - const settlements: unknown[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd, (settlement) => - settlements.push(settlement) + // The window a Stop naming no turn exists for: Orca has written the message, and Claude has not + // started its reply, so no turn id exists. Only the live binary can say the interrupt lands there. + it.skipIf(!realClaudeAuthenticated)( + 'stops a message interrupted before its reply starts, and takes the next one', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + const frames = (from: number) => + events.slice(from).flatMap((event) => (event.type === 'message' ? [event.message] : [])) + const replyStarted = (from: number) => + frames(from).some( + (frame) => + frame.type === 'stream_event' && + typeof frame.event === 'object' && + frame.event !== null && + 'type' in frame.event && + frame.event.type === 'message_start' ) - const send = (clientMessageId: string, text: string) => - adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId, - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, - fence: 1 - }) - const waitFor = async (found: () => boolean): Promise => { - const deadline = Date.now() + 60_000 - while (!found() && Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, 100)) + const result = async (from: number) => { + const deadline = Date.now() + 60_000 + for (;;) { + const found = frames(from).find((frame) => frame.type === 'result') + if (found || Date.now() >= deadline) { + return found } - return found() + await new Promise((resolve) => setTimeout(resolve, 100)) } + } + const send = (clientMessageId: string, text: string) => + adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, + fence: 1 + }) - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-queued-stop' - }) - await send('real-cli-queued-stop-a', 'Count from 1 to 400, one number per line.') - // A's reply is streaming, so the next send queues behind its turn. - await expect( - waitFor(() => - events.some( - (event) => - event.type === 'message' && - event.message.type === 'stream_event' && - JSON.stringify(event.message).includes('text_delta') - ) - ) - ).resolves.toBe(true) - await send('real-cli-queued-stop-b', 'Say the word banana.') + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-stop' + }) + const first = events.length + await expect( + send('real-cli-stop-1', 'Count from 1 to 300, one number per line, and nothing else.') + ).resolves.toEqual({ state: 'admitted' }) + const startedBeforeStop = replyStarted(first) - await expect( - adapter.cancelTurn({ - sessionId: 'real-cli-handshake', - turnId: 'turn-a', - fence: 1, - resolveLiveTurnId: () => 'turn-a', - // What the host reads for B: handed over, not yet answered. - dispatchStatus: { state: 'pending', recovered: false } - }) - ).resolves.toEqual({ cancelled: true }) + await expect( + adapter.cancelTurn({ sessionId: 'real-cli-handshake', fence: 1 }) + ).resolves.toEqual({ + cancelled: true + }) + const stopped = await result(first) - expect(settlements).toContainEqual({ - sessionId: 'real-cli-handshake', - clientMessageId: 'real-cli-queued-stop-b', - state: 'rejected', - reason: 'provider_cancelled_before_start', - rejection: { kind: 'cancelled' } - }) - } finally { - await adapter.closeAll() - await rm(cwd, { recursive: true, force: true }) - } - }, - 150_000 - ) + expect(startedBeforeStop).toBe(false) + // Claude 2.1.280 echoes the message, then ends that turn as interrupted without replying. + expect(stopped).toMatchObject({ subtype: 'error_during_execution' }) + expect(replyStarted(first)).toBe(false) + + const second = events.length + await expect(send('real-cli-stop-2', 'Reply with the single word ok.')).resolves.toEqual({ + state: 'admitted' + }) + await expect(result(second)).resolves.toMatchObject({ subtype: 'success' }) + } finally { + await adapter.closeAll() + } + }, + 150_000 + ) - it('turns a real silent unauthenticated startup into sign-in guidance', async () => { - const claudeConfigDir = await mkdtemp(join(tmpdir(), 'orca-claude-no-auth-')) + // Orca installs a SessionStart hook, so its frame proves most real starts before the turn's + // system/init, the only frame that says this CLI can cancel what it queued. + it.skipIf(!realClaudeAuthenticated)( + 'withdraws a follow-up queued behind a turn that is stopped, behind a SessionStart hook', + async () => { const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const cwd = await mkdtemp(join(tmpdir(), 'orca-queued-stop-')) + await mkdir(join(cwd, '.claude'), { recursive: true }) + await writeFile( + join(cwd, '.claude', 'settings.json'), + JSON.stringify({ + hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] } + }) + ) const events: ClaudeStructuredSessionEvent[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + const settlements: unknown[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd, (settlement) => + settlements.push(settlement) + ) + const send = (clientMessageId: string, text: string) => + adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, + fence: 1 + }) + const waitFor = async (found: () => boolean): Promise => { + const deadline = Date.now() + 60_000 + while (!found() && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)) + } + return found() + } try { await adapter.acquire({ identity: identity(providerSessionId), fence: 1, - spawnToken: 'real-cli-no-auth' + spawnToken: 'real-cli-queued-stop' }) - await adapter.drainObservedExits() - expect(events.find((event) => event.type === 'ended')).toMatchObject({ - reason: expect.stringMatching(/not signed in.*Claude CLI.*CLAUDE_CONFIG_DIR/s), - startupUnproven: true + await send('real-cli-queued-stop-a', 'Count from 1 to 400, one number per line.') + // A's reply is streaming, so the next send queues behind its turn. + await expect( + waitFor(() => + events.some( + (event) => + event.type === 'message' && + event.message.type === 'stream_event' && + JSON.stringify(event.message).includes('text_delta') + ) + ) + ).resolves.toBe(true) + await send('real-cli-queued-stop-b', 'Say the word banana.') + + await expect( + adapter.cancelTurn({ + sessionId: 'real-cli-handshake', + turnId: 'turn-a', + fence: 1, + resolveLiveTurnId: () => 'turn-a', + // What the host reads for B: handed over, not yet answered. + dispatchStatus: { state: 'pending', recovered: false } + }) + ).resolves.toEqual({ cancelled: true }) + + expect(settlements).toContainEqual({ + sessionId: 'real-cli-handshake', + clientMessageId: 'real-cli-queued-stop-b', + state: 'rejected', + reason: 'provider_cancelled_before_start', + rejection: { kind: 'cancelled' } }) } finally { await adapter.closeAll() - await rm(claudeConfigDir, { recursive: true, force: true }) + await rm(cwd, { recursive: true, force: true }) } - }, 10_000) - } -) + }, + 150_000 + ) + + it('turns a real silent unauthenticated startup into sign-in guidance', async () => { + const claudeConfigDir = await mkdtemp(join(tmpdir(), 'orca-claude-no-auth-')) + const providerSessionId = randomUUID() + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-no-auth' + }) + await adapter.drainObservedExits() + expect(events.find((event) => event.type === 'ended')).toMatchObject({ + reason: expect.stringMatching(/not signed in.*Claude CLI.*CLAUDE_CONFIG_DIR/s), + startupUnproven: true + }) + } finally { + await adapter.closeAll() + await rm(claudeConfigDir, { recursive: true, force: true }) + } + }, 10_000) +})