test(claude): isolate real CLI fixtures and cover chat option requests

src/main/claude/claude-structured-real-cli-options.test.ts

@@ -0,0 +1,245 @@
+// The SDK requests behind the model, effort and permission pickers and the context meter, against
+// the live CLI: CI meets them only through stand-ins, which pin the shape we read, not the one the
+// CLI sends. Runs only where a signed-in Claude CLI exists, like the rest of the real-CLI suite.
+
+import { randomUUID } from 'node:crypto'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, expect, it } from 'vitest'
+import type { AgentSessionJournalIdentity } from '../../shared/agent-session-journal-types'
+import type { AgentSessionExecutionOptions } from '../../shared/agent-session-execution-options'
+import { readAgentJournalTurn } from '../../shared/agent-session-turn-record'
+import {
+  claudeProviderHandle,
+  claudeProviderHandleLeafUuid
+} from '../../shared/agent-session-provider-handle-encoding'
+import {
+  CLAUDE_STRUCTURED_BASE_OPTIONS,
+  claudeStructuredPermissionOptions
+} from './claude-structured-launch-resolution'
+import { listedModels } from './claude-structured-model-catalog'
+import {
+  ClaudeStructuredSessionAdapter,
+  type ClaudeStructuredSessionEvent
+} from './claude-structured-session-adapter'
+import {
+  realClaudeAuthenticated,
+  realClaudeAvailable,
+  realClaudeCommand,
+  realClaudeLaunchHome
+} from './claude-real-cli-availability-test-support'
+import { claudeStartupSettled } from './claude-structured-session-test-support'
+import { journal } from './claude-context-usage-test-support'
+
+const SESSION_ID = 'real-cli-options'
+// The SDK's effort scale, low to high; the CLI's list order is not a contract.
+const EFFORT_SCALE = ['low', 'medium', 'high', 'xhigh', 'max']
+
+function identity(providerSessionId: string, leafUuid: string | null): AgentSessionJournalIdentity {
+  return {
+    sessionId: SESSION_ID,
+    workspaceId: 'real-cli-options-workspace',
+    hostId: 'local',
+    agent: 'claude',
+    providerHandle: claudeProviderHandle(providerSessionId, leafUuid)
+  }
+}
+
+async function until(check: () => boolean, timeoutMs: number): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    if (check()) {
+      return true
+    }
+    await new Promise((resolve) => setTimeout(resolve, 200))
+  }
+  return check()
+}
+
+// Inits name a 1M-window model with `[1m]`; context reports and responses drop it.
+const baseModelId = (model: string): string =>
+  model
+    .replace(/\[[^\]]*\]$/u, '')
+    .trim()
+    .toLowerCase()
+
+describe.skipIf(!realClaudeAvailable)('Claude structured real CLI options', () => {
+  it.skipIf(!realClaudeAuthenticated)(
+    'applies a message model and effort, reports context usage, and relaunches into a new permission mode',
+    async () => {
+      const providerSessionId = randomUUID()
+      const { claudeConfigDir, env } = realClaudeLaunchHome()
+      const cwd = await mkdtemp(join(tmpdir(), 'orca-real-options-'))
+      const events: ClaudeStructuredSessionEvent[] = []
+      const record = journal()
+      const persisted: { leafUuid: string | null } = { leafUuid: null }
+      let launches = 0
+      const adapter = new ClaudeStructuredSessionAdapter({
+        resolveLaunch: async ({ identity: launchIdentity, permissionMode }) => {
+          const permission = claudeStructuredPermissionOptions(permissionMode ?? 'ask')
+          // A relaunch resumes the transcript the first launch wrote, as the host's resolver does.
+          const resumes = launches++ > 0
+          return {
+            pathToClaudeCodeExecutable: realClaudeCommand,
+            options: {
+              ...CLAUDE_STRUCTURED_BASE_OPTIONS,
+              ...permission,
+              extraArgs: { ...CLAUDE_STRUCTURED_BASE_OPTIONS.extraArgs, ...permission.extraArgs },
+              ...(resumes ? { resume: providerSessionId } : { sessionId: providerSessionId })
+            },
+            cwd,
+            env,
+            claudeConfigDir,
+            providerSessionId,
+            resumeLeafUuid:
+              resumes && launchIdentity.providerHandle
+                ? claudeProviderHandleLeafUuid(launchIdentity.providerHandle)
+                : null,
+            resumesTranscript: resumes,
+            continuesChain: resumes
+          }
+        },
+        onEvent: (event) => events.push(event),
+        persistHandle: async (handle) => {
+          persisted.leafUuid = handle.leafUuid
+        },
+        readProcessStartTime: async () => 1
+      })
+      const frames = (from: number) =>
+        events.slice(from).flatMap((event) => (event.type === 'message' ? [event.message] : []))
+      const inits = (from: number) =>
+        frames(from).filter((message) => message.type === 'system' && message.subtype === 'init')
+      const results = (from: number) => frames(from).filter((message) => message.type === 'result')
+      const send = (
+        clientMessageId: string,
+        text: string,
+        fence: number,
+        executionOptions?: AgentSessionExecutionOptions
+      ) =>
+        adapter.dispatch({
+          sessionId: SESSION_ID,
+          clientMessageId,
+          body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] },
+          requestedAt: Date.now(),
+          fence,
+          ...(executionOptions ? { executionOptions } : {})
+        })
+      const contextReport = () =>
+        record
+          .items()
+          .flatMap((row) => {
+            const used = readAgentJournalTurn(row.body)?.contextUsage?.used
+            return used?.kind === 'report' ? [used] : []
+          })
+          .at(-1)
+
+      try {
+        await adapter.acquire({
+          identity: identity(providerSessionId, null),
+          fence: 1,
+          spawnToken: 'real-cli-options-1',
+          permissionMode: 'ask',
+          events: record.sink
+        })
+        await claudeStartupSettled(adapter, SESSION_ID)
+
+        // A row without `value` or `resolvedModel` silently turns off the admit check, the effort
+        // guard and model confirmation.
+        const connection = adapter['sessions'].get(SESSION_ID)?.connection
+        if (!connection) {
+          throw new Error('no live Claude connection')
+        }
+        const listed = listedModels({ models: await connection.supportedModels() })
+        const rows = ['haiku', 'sonnet'].map((alias) => listed.find(({ id }) => id === alias))
+        for (const row of rows) {
+          expect(row?.resolvedModel).toEqual(expect.stringMatching(/\S/u))
+        }
+        const opened = await adapter.readOptions({ sessionId: SESSION_ID, fence: 1 })
+        expect(opened.catalogListing?.models.map(({ id }) => id)).toContain('haiku')
+
+        const pick = rows.find((row) => row !== undefined && row.efforts.length > 0)
+        const level = pick?.efforts
+          .map(({ value }) => value)
+          .filter((value) => EFFORT_SCALE.includes(value))
+          .sort((left, right) => EFFORT_SCALE.indexOf(left) - EFFORT_SCALE.indexOf(right))[0]
+        if (!pick?.resolvedModel || !level) {
+          throw new Error('neither haiku nor sonnet lists an effort level')
+        }
+
+        // The production send path: list_models, set_model, apply_flag_settings and get_settings.
+        // `ranOptions` carries only what the CLI reported; it names the resolved id, so an alias
+        // pick reports no model and the recorder falls back to the message's own pick.
+        const firstTurn = events.length
+        const outcome = await send(
+          'real-cli-options-1',
+          'Reply with exactly: OK. Do not use any tools.',
+          1,
+          { model: pick.id, effort: level }
+        )
+        expect(outcome).toMatchObject({ state: 'admitted', ranOptions: { effort: level } })
+        expect([undefined, pick.id]).toContain(
+          'ranOptions' in outcome ? outcome.ranOptions?.model : undefined
+        )
+        expect(await until(() => results(firstTurn).length > 0, 90_000)).toBe(true)
+        expect(results(firstTurn)[0]).toMatchObject({ subtype: 'success', is_error: false })
+        expect(inits(firstTurn)[0]?.model).toBe(pick.resolvedModel)
+        // Effort is read only through get_settings; no frame carries it, init included.
+        expect(frames(0).filter((frame) => 'effortLevel' in frame)).toEqual([])
+        await expect(
+          adapter.readOptions({ sessionId: SESSION_ID, fence: 1 })
+        ).resolves.toMatchObject({
+          current: {
+            model: pick.id,
+            effort: level,
+            confirmed: expect.arrayContaining(['model', 'effort'])
+          }
+        })
+
+        // The turn end asks get_context_usage; an answer Orca cannot read blanks the meter silently.
+        expect(await until(() => contextReport() !== undefined, 10_000)).toBe(true)
+        const report = contextReport()
+        if (!report) {
+          throw new Error('no context report on any turn row')
+        }
+        expect(report.usedTokens).toBeGreaterThan(0)
+        expect(report.windowTokens).toBeGreaterThanOrEqual(report.usedTokens)
+        expect(report.model).toMatch(/\S/u)
+        expect(baseModelId(report.model)).toBe(baseModelId(pick.resolvedModel))
+
+        // Orca never sends set_permission_mode: it retires the child and resumes it under the new
+        // flag with the saved picks, so the CLI must read its mode at launch.
+        await adapter.closeSession(SESSION_ID)
+        const resumedTurn = events.length
+        await adapter.acquire({
+          identity: identity(providerSessionId, persisted.leafUuid),
+          fence: 2,
+          spawnToken: 'real-cli-options-2',
+          permissionMode: 'accept-edits',
+          options: { model: pick.id, effort: level },
+          events: record.sink
+        })
+        await claudeStartupSettled(adapter, SESSION_ID)
+        await expect(
+          send('real-cli-options-2', 'Reply with exactly: OK.', 2)
+        ).resolves.toMatchObject({
+          state: 'admitted'
+        })
+        expect(await until(() => results(resumedTurn).length > 0, 90_000)).toBe(true)
+        expect(results(resumedTurn)[0]).toMatchObject({ subtype: 'success', is_error: false })
+        expect(inits(resumedTurn)[0]).toMatchObject({
+          session_id: providerSessionId,
+          permissionMode: 'acceptEdits',
+          model: pick.resolvedModel
+        })
+        await expect(
+          adapter.readOptions({ sessionId: SESSION_ID, fence: 2 })
+        ).resolves.toMatchObject({ current: { effort: level } })
+      } finally {
+        await adapter.closeAll()
+        await rm(cwd, { recursive: true, force: true })
+      }
+    },
+    180_000
+  )
+})

src/main/claude/claude-structured-real-cli.test.ts

@@ -78,6 +78,23 @@ function realAdapter(
   return adapter
 }
 
+/** A temp project with its own SessionStart hook, so the hook frame never depends on the machine's real settings. */
+async function projectWithStartHook(
+  prefix: string,
+  settings: Record<string, unknown> = {}
+): Promise<string> {
+  const cwd = await mkdtemp(join(tmpdir(), prefix))
+  await mkdir(join(cwd, '.claude'), { recursive: true })
+  await writeFile(
+    join(cwd, '.claude', 'settings.json'),
+    JSON.stringify({
+      ...settings,
+      hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] }
+    })
+  )
+  return cwd
+}
+
 function identity(providerSessionId: string): AgentSessionJournalIdentity {
   return {
     sessionId: 'real-cli-handshake',
@@ -158,7 +175,7 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => {
       const providerSessionId = randomUUID()
       const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude')
       const events: ClaudeStructuredSessionEvent[] = []
-      const cwd = await mkdtemp(join(tmpdir(), 'orca-command-init-'))
+      const cwd = await projectWithStartHook('orca-command-init-')
       await mkdir(join(cwd, '.claude', 'commands'), { recursive: true })
       await writeFile(
         join(cwd, '.claude', 'commands', 'orca-init-catalog-proof.md'),
@@ -211,7 +228,8 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => {
       const providerSessionId = randomUUID()
       const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude')
       const events: ClaudeStructuredSessionEvent[] = []
-      const adapter = realAdapter(providerSessionId, claudeConfigDir, events)
+      const cwd = await projectWithStartHook('orca-effort-readback-', { effortLevel: 'low' })
+      const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd)
 
       try {
         await adapter.acquire({
@@ -233,9 +251,10 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => {
         expect(published.filter((frame) => 'effortLevel' in frame)).toEqual([])
         // Goes red if `effective.effortLevel` is renamed or dropped, which no
         // fixture-backed test can see.
-        expect(options.current.effort).toEqual(expect.any(String))
+        expect(options.current.effort).toBe('low')
       } finally {
         await adapter.closeAll()
+        await rm(cwd, { recursive: true, force: true })
       }
     },
     15_000
@@ -347,7 +366,9 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => {
         expect(frames[0]?.model).toMatch(/^claude-haiku-/)
         await expect(
           adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 })
-        ).resolves.toMatchObject({ current: { model: 'haiku' } })
+        ).resolves.toMatchObject({
+          current: { model: 'haiku', confirmed: expect.arrayContaining(['model']) }
+        })
       } finally {
         await adapter.closeAll()
       }
@@ -579,14 +600,7 @@ describe.skipIf(!realClaudeAvailable)(suiteTitle, () => {
     async () => {
       const providerSessionId = randomUUID()
       const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude')
-      const cwd = await mkdtemp(join(tmpdir(), 'orca-queued-stop-'))
-      await mkdir(join(cwd, '.claude'), { recursive: true })
-      await writeFile(
-        join(cwd, '.claude', 'settings.json'),
-        JSON.stringify({
-          hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] }
-        })
-      )
+      const cwd = await projectWithStartHook('orca-queued-stop-')
       const events: ClaudeStructuredSessionEvent[] = []
       const settlements: unknown[] = []
       const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd, (settlement) =>

src/main/claude/claude-supervised-stop-real-cli.test.ts

@@ -1,11 +1,13 @@
 import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
-import { homedir, tmpdir } from 'node:os'
+import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { randomUUID } from 'node:crypto'
 import { afterEach, describe, expect, it } from 'vitest'
 import { runProcess } from '@orca/process-host'
 import { resolveSessionFilePath } from '../native-chat/session-file-resolver'
 import { SUPERVISED_GRACEFUL_EXIT_MS } from './claude-child-exit-proof-ladder'
+import { claudeConfigDirEnvPatch } from './claude-config-dir-pin'
+import { realClaudeLaunchHome } from './claude-real-cli-availability-test-support'
 import {
   openClaudeStreamJsonConnection,
   type ClaudeStreamJsonConnection
@@ -15,14 +17,16 @@ import {
   claudeStructuredPermissionOptions
 } from './claude-structured-launch-resolution'
 
-// Opt-in only: spends a real (haiku) turn per case. Run it in an isolated HOME with
-// ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS when auth lives in a settings file.
+// Opt-in only: spends a real (haiku) turn per case. Needs ORCA_REAL_CLAUDE_BIN; set
+// ORCA_REAL_CLAUDE_SETTINGS only when auth lives in a settings file outside the Claude home.
 const CLAUDE_BIN = process.env.ORCA_REAL_CLAUDE_BIN ?? ''
 const enabled =
   process.env.ORCA_REAL_CLAUDE_SUPERVISED_STOP === '1' &&
   process.platform !== 'win32' &&
   CLAUDE_BIN.length > 0
 const FRAMES_OUT = process.env.ORCA_REAL_CLAUDE_FRAMES_OUT
+// Launch and transcript lookup share one home, as production acquisition does.
+const home = realClaudeLaunchHome()
 const TOOL_MARKER = 'orca_supervised_stop_probe'
 const TOOL_PROMPT = `Use the Bash tool to run exactly this command, with no timeout argument, and nothing else: python3 -c "import time; time.sleep(120)  # ${TOOL_MARKER}"`
 
@@ -94,7 +98,8 @@ async function open(sessionId: string, cwd: string, resume: boolean, frames: Fra
         },
         ...(resume ? { resume: sessionId } : { sessionId })
       },
-      cwd
+      cwd,
+      env: { ...home.env, ...claudeConfigDirEnvPatch(home.claudeConfigDir, { env: home.env }) }
     },
     { onMessage: (message) => frames.push({ at: Date.now(), message }) }
   )
@@ -113,11 +118,10 @@ function userMessage(text: string): Record<string, unknown> {
 }
 
 async function transcriptLines(sessionId: string): Promise<string> {
-  const configDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude')
   const path = await until(
     () =>
       resolveSessionFilePath('claude', sessionId, {
-        claudeProjectsDir: join(configDir, 'projects')
+        claudeProjectsDir: join(home.claudeConfigDir, 'projects')
       }),
     'the transcript',
     15_000