From 21f4015c65ac0400487030be7b09440582cea2af Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:10:59 -0700 Subject: [PATCH 1/4] test: make tests that start a real agent CLI opt-in Tests that start the real Claude, Codex or Hermes CLI now skip unless ORCA_RUN_REAL_AGENT_TESTS=1, checked before any availability probe runs, so a broad local vitest run no longer starts the developer's real agent against their own config and account. The Codex contract CI job sets the variable and turns a missing opt-in into a red test. --- .github/workflows/pr.yml | 2 + ...real-cli-availability-test-support.test.ts | 72 +++ ...aude-real-cli-availability-test-support.ts | 19 +- .../claude-structured-real-cli-fold.test.ts | 265 +++++----- .../claude/claude-structured-real-cli.test.ts | 475 +++++++++--------- .../claude-supervised-stop-real-cli.test.ts | 12 +- .../codex-index-heal-binary-contract.test.ts | 40 +- src/main/hermes/hook-service.test.ts | 7 +- .../real-agent-tests-opt-in-test-support.ts | 10 + ...tration-codex-real-pty.integration.test.ts | 10 +- 10 files changed, 517 insertions(+), 395 deletions(-) create mode 100644 src/main/claude/claude-real-cli-availability-test-support.test.ts create mode 100644 src/main/real-agent-tests-opt-in-test-support.ts diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 795beca131e1..f2bf98881149 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -460,6 +460,8 @@ jobs: # reports success. This turns a failed or missing install into a red test # instead of a green no-op. ORCA_CODEX_CONTRACT_REQUIRED: '1' + # Why: real-agent tests skip unless opted in; this job exists to run one. + ORCA_RUN_REAL_AGENT_TESTS: '1' ORCA_CODEX_CONTRACT_VERSION: ${{ env.CODEX_CLI_VERSION }} run: | set -euo pipefail diff --git a/src/main/claude/claude-real-cli-availability-test-support.test.ts b/src/main/claude/claude-real-cli-availability-test-support.test.ts new file mode 100644 index 000000000000..a9964cb221e0 --- /dev/null +++ b/src/main/claude/claude-real-cli-availability-test-support.test.ts @@ -0,0 +1,72 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + REAL_AGENT_TESTS_ENV, + realAgentTestsEnabled +} from '../real-agent-tests-opt-in-test-support' + +const { spawnMock, spawnSyncMock } = vi.hoisted(() => ({ + spawnMock: vi.fn(), + spawnSyncMock: vi.fn(() => ({ status: 1, signal: null, stdout: '', stderr: '', pid: 0 })) +})) + +// Why the lowest layer: nothing below this helper may start a process, whichever wrapper it uses. +vi.mock('node:child_process', async (importOriginal) => ({ + ...(await importOriginal>()), + spawn: spawnMock, + spawnSync: spawnSyncMock +})) + +// Why a missing path: even a mock that failed to apply could not reach a real CLI. +vi.mock('../codex-cli/command', () => ({ + resolveClaudeCommand: () => '/nonexistent/orca-test-claude' +})) + +async function loadAvailability() { + vi.resetModules() + return import('./claude-real-cli-availability-test-support') +} + +describe('real Claude CLI availability', () => { + beforeEach(() => { + spawnMock.mockClear() + spawnSyncMock.mockClear() + }) + + afterEach(() => { + vi.unstubAllEnvs() + }) + + it('skips without spawning anything unless real-agent tests are opted into', async () => { + vi.stubEnv(REAL_AGENT_TESTS_ENV, undefined) + + const { realClaudeSkipReason, realClaudeAuthStatus } = await loadAvailability() + + expect(realClaudeSkipReason).toContain(`${REAL_AGENT_TESTS_ENV}=1`) + expect(realClaudeAuthStatus).toBeNull() + expect(spawnSyncMock).not.toHaveBeenCalled() + expect(spawnMock).not.toHaveBeenCalled() + }) + + it('probes the CLI once opted into', async () => { + vi.stubEnv(REAL_AGENT_TESTS_ENV, '1') + + const { realClaudeSkipReason } = await loadAvailability() + + // The mocked probe fails, so this proves the gate, not a real CLI. + expect(realClaudeSkipReason).toMatch(/no runnable Claude CLI/) + expect(spawnSyncMock).toHaveBeenCalledTimes(1) + expect(spawnSyncMock.mock.calls[0]?.[1]).toContain('--version') + }) +}) + +describe('realAgentTestsEnabled', () => { + it.each([ + [undefined, false], + ['', false], + ['0', false], + ['true', false], + ['1', true] + ])('%j -> %s', (value, expected) => { + expect(realAgentTestsEnabled({ [REAL_AGENT_TESTS_ENV]: value })).toBe(expected) + }) +}) diff --git a/src/main/claude/claude-real-cli-availability-test-support.ts b/src/main/claude/claude-real-cli-availability-test-support.ts index faa322c456b1..074198fe5a6b 100644 --- a/src/main/claude/claude-real-cli-availability-test-support.ts +++ b/src/main/claude/claude-real-cli-availability-test-support.ts @@ -1,5 +1,5 @@ // Whether a real, signed-in Claude CLI is present — the gate every real-CLI -// suite skips on. Probed once per test process. +// suite skips on. Probed once per test process, and only when real-agent tests are opted into. import { homedir } from 'node:os' import { join } from 'node:path' @@ -7,6 +7,10 @@ import { runProcessSync, type ProcessResult } from '../../shared/child-process/r import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution' import { CLAUDE_AUTH_ENV_VARS } from '../claude-accounts/environment' import { resolveClaudeCommand } from '../codex-cli/command' +import { + REAL_AGENT_TESTS_SKIP_NOTE, + realAgentTestsEnabled +} from '../real-agent-tests-opt-in-test-support' export const realClaudeCommand = resolveClaudeCommand() @@ -25,7 +29,9 @@ function probeRealClaude(args: string[]): ProcessResult | null { } } -export const realClaudeAvailable = probeRealClaude(['--version'])?.code === 0 +const optedIn = realAgentTestsEnabled() + +const realClaudeAvailable = optedIn && probeRealClaude(['--version'])?.code === 0 /** The CLI's own account report — the only source of truth for where it writes that * is not derived from Orca's own path expressions. */ @@ -44,7 +50,14 @@ export const realClaudeAuthStatus = (() => { } })() -export const realClaudeAuthenticated = realClaudeAuthStatus?.loggedIn === true +/** Why every real-CLI test in this process skips, or null when they may run. */ +export const realClaudeSkipReason: string | null = !optedIn + ? REAL_AGENT_TESTS_SKIP_NOTE + : !realClaudeAvailable + ? `no runnable Claude CLI at ${realClaudeCommand}` + : realClaudeAuthStatus?.loggedIn !== true + ? 'the Claude CLI is not signed in' + : null /** The config dir and env auth the availability probe above saw, for a real-CLI launch. * The connection strips an inherited CLAUDE_CONFIG_DIR and inherited auth vars, so diff --git a/src/main/claude/claude-structured-real-cli-fold.test.ts b/src/main/claude/claude-structured-real-cli-fold.test.ts index 5ce21d83f170..1aa0e646fb65 100644 --- a/src/main/claude/claude-structured-real-cli-fold.test.ts +++ b/src/main/claude/claude-structured-real-cli-fold.test.ts @@ -1,6 +1,6 @@ // The fold receipt against the real CLI: a message sent while a turn runs is // folded by the CLI into the running turn, and Orca must keep ONE turn row — -// no interrupted marking, no second bar. Runs only where a signed-in Claude CLI +// no interrupted marking, no second bar. Runs only when opted in and a signed-in Claude CLI // exists, like the rest of the real-CLI suite. Live sessions prove the session // from a SessionStart hook frame BEFORE system/init arrives, which is exactly // the path the fixture harness cannot fake end to end. @@ -9,7 +9,7 @@ import { randomUUID } from 'node:crypto' import { mkdtemp } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentJournalItemBody, AgentSessionJournalIdentity @@ -22,10 +22,9 @@ import { type ClaudeStructuredSessionEvent } from './claude-structured-session-adapter' import { - realClaudeAuthenticated, - realClaudeAvailable, realClaudeCommand, - realClaudeLaunchHome + realClaudeLaunchHome, + realClaudeSkipReason } from './claude-real-cli-availability-test-support' const SESSION_ID = 'real-cli-fold' @@ -57,138 +56,140 @@ async function until(check: () => boolean, timeoutMs: number): Promise return check() } -describe.skipIf(!realClaudeAvailable)('Claude structured real CLI fold', () => { - it.skipIf(!realClaudeAuthenticated)( - 'keeps one turn row when the CLI folds a mid-turn send', - async () => { - const providerSessionId = randomUUID() - const { claudeConfigDir, env } = realClaudeLaunchHome() - const cwd = await mkdtemp(join(tmpdir(), 'orca-real-fold-')) - const events: ClaudeStructuredSessionEvent[] = [] - const turnRows: NonNullable>[] = [] - const sink: StructuredAgentSessionEventSink = { - appendItem: (_identity, body: AgentJournalItemBody) => { - const turn = readAgentJournalTurn(body) - if (turn) { - turnRows.push(turn) - } - }, - appendTombstone: () => {}, - publish: () => {} - } - const settled = vi.fn() - const adapter = new ClaudeStructuredSessionAdapter({ - resolveLaunch: async () => ({ - pathToClaudeCodeExecutable: realClaudeCommand, - options: { - ...CLAUDE_STRUCTURED_BASE_OPTIONS, - extraArgs: { - ...CLAUDE_STRUCTURED_BASE_OPTIONS.extraArgs, - settings: FOLD_SESSION_SETTINGS - }, - sessionId: providerSessionId +describe('Claude structured real CLI fold', () => { + beforeEach((ctx) => { + if (realClaudeSkipReason) { + ctx.skip(realClaudeSkipReason) + } + }) + + it('keeps one turn row when the CLI folds a mid-turn send', async () => { + const providerSessionId = randomUUID() + const { claudeConfigDir, env } = realClaudeLaunchHome() + const cwd = await mkdtemp(join(tmpdir(), 'orca-real-fold-')) + const events: ClaudeStructuredSessionEvent[] = [] + const turnRows: NonNullable>[] = [] + const sink: StructuredAgentSessionEventSink = { + appendItem: (_identity, body: AgentJournalItemBody) => { + const turn = readAgentJournalTurn(body) + if (turn) { + turnRows.push(turn) + } + }, + appendTombstone: () => {}, + publish: () => {} + } + const settled = vi.fn() + const adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: realClaudeCommand, + options: { + ...CLAUDE_STRUCTURED_BASE_OPTIONS, + extraArgs: { + ...CLAUDE_STRUCTURED_BASE_OPTIONS.extraArgs, + settings: FOLD_SESSION_SETTINGS }, - cwd, - env, - claudeConfigDir, - providerSessionId, - resumeLeafUuid: null, - resumesTranscript: false, - continuesChain: false - }), - onEvent: (event) => events.push(event), - onDispatchSettledLate: settled, - readProcessStartTime: async () => 1 + sessionId: providerSessionId + }, + cwd, + env, + claudeConfigDir, + providerSessionId, + resumeLeafUuid: null, + resumesTranscript: false, + continuesChain: false + }), + onEvent: (event) => events.push(event), + onDispatchSettledLate: settled, + readProcessStartTime: async () => 1 + }) + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-fold', + events: sink }) - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-fold', - events: sink - }) - await adapter.awaitStarted(SESSION_ID) - // Startup proved from the SessionStart hook frame, with no init yet. - expect( - events.some( - (event) => - event.type === 'message' && - event.message.type === 'system' && - event.message.subtype === 'init' - ) - ).toBe(false) - - await expect( - adapter.dispatch({ - sessionId: SESSION_ID, - clientMessageId: 'client-A', - body: { - kind: 'message', - role: 'user', - blocks: [ - { - type: 'text', - text: 'Use the Bash tool to run `sleep 8` two separate times, one call at a time, waiting for each. Then reply exactly: FIRST DONE' - } - ] - }, - requestedAt: Date.now(), - fence: 1 - }) - ).resolves.toEqual({ state: 'admitted' }) - - // Wait for A's replay to open the turn, then send B mid-turn: the first - // `sleep 8` guarantees the CLI is still inside A's request cycle. - expect( - await until( - () => events.some((event) => event.type === 'message' && event.startsTurn === true), - 30_000 - ) - ).toBe(true) - await new Promise((resolve) => setTimeout(resolve, 2_000)) - await expect( - adapter.dispatch({ - sessionId: SESSION_ID, - clientMessageId: 'client-B', - body: { - kind: 'message', - role: 'user', - blocks: [{ type: 'text', text: 'Also say the word banana at the end of your reply.' }] - }, - requestedAt: Date.now(), - fence: 1 - }) - ).resolves.toEqual({ state: 'admitted' }) + await adapter.awaitStarted(SESSION_ID) + // Startup proved from the SessionStart hook frame, with no init yet. + expect( + events.some( + (event) => + event.type === 'message' && + event.message.type === 'system' && + event.message.subtype === 'init' + ) + ).toBe(false) - const results = () => - events.flatMap((event) => - event.type === 'message' && event.message.type === 'result' ? [event.message] : [] - ) - expect(await until(() => results().length > 0, 90_000)).toBe(true) + await expect( + adapter.dispatch({ + sessionId: SESSION_ID, + clientMessageId: 'client-A', + body: { + kind: 'message', + role: 'user', + blocks: [ + { + type: 'text', + text: 'Use the Bash tool to run `sleep 8` two separate times, one call at a time, waiting for each. Then reply exactly: FIRST DONE' + } + ] + }, + requestedAt: Date.now(), + fence: 1 + }) + ).resolves.toEqual({ state: 'admitted' }) - // The CLI folded: one result names both sends. (If this ever reports a - // lone send, the steer raced past the fold window — a miss, not a fold.) - const named = results().flatMap((message) => - Array.isArray(message.user_message_uuids) ? [message.user_message_uuids] : [] + // Wait for A's replay to open the turn, then send B mid-turn: the first + // `sleep 8` guarantees the CLI is still inside A's request cycle. + expect( + await until( + () => events.some((event) => event.type === 'message' && event.startsTurn === true), + 30_000 ) - expect(named[0]).toHaveLength(2) + ).toBe(true) + await new Promise((resolve) => setTimeout(resolve, 2_000)) + await expect( + adapter.dispatch({ + sessionId: SESSION_ID, + clientMessageId: 'client-B', + body: { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: 'Also say the word banana at the end of your reply.' }] + }, + requestedAt: Date.now(), + fence: 1 + }) + ).resolves.toEqual({ state: 'admitted' }) - // ONE turn for the whole run: never interrupted, and B settled accepted - // into it under its own adopted uuid. - expect(turnRows.every((turn) => turn.state !== 'interrupted')).toBe(true) - expect([...new Set(turnRows.map((turn) => turn.turnId))]).toHaveLength(1) - expect(turnRows.at(-1)).toMatchObject({ state: 'completed' }) - expect(settled).toHaveBeenCalledWith( - expect.objectContaining({ - sessionId: SESSION_ID, - clientMessageId: 'client-B', - providerIdentity: expect.objectContaining({ provider: 'claude' }) - }) + const results = () => + events.flatMap((event) => + event.type === 'message' && event.message.type === 'result' ? [event.message] : [] ) - } finally { - await adapter.closeAll() - } - }, - 150_000 - ) + expect(await until(() => results().length > 0, 90_000)).toBe(true) + + // The CLI folded: one result names both sends. (If this ever reports a + // lone send, the steer raced past the fold window — a miss, not a fold.) + const named = results().flatMap((message) => + Array.isArray(message.user_message_uuids) ? [message.user_message_uuids] : [] + ) + expect(named[0]).toHaveLength(2) + + // ONE turn for the whole run: never interrupted, and B settled accepted + // into it under its own adopted uuid. + expect(turnRows.every((turn) => turn.state !== 'interrupted')).toBe(true) + expect([...new Set(turnRows.map((turn) => turn.turnId))]).toHaveLength(1) + expect(turnRows.at(-1)).toMatchObject({ state: 'completed' }) + expect(settled).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + clientMessageId: 'client-B', + providerIdentity: expect.objectContaining({ provider: 'claude' }) + }) + ) + } finally { + await adapter.closeAll() + } + }, 150_000) }) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index 20670147ca0d..c3a6b9b1880e 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -2,15 +2,14 @@ import { randomUUID } from 'node:crypto' import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { homedir, tmpdir } from 'node:os' import { basename, join, relative } from 'node:path' -import { describe, expect, it } from 'vitest' +import { beforeEach, describe, expect, it } from 'vitest' import type { AgentSessionJournalIdentity } from '../../shared/agent-session-journal-types' import { resolveSessionFilePath } from '../native-chat/session-file-resolver' import { CLAUDE_STRUCTURED_BASE_OPTIONS } from './claude-structured-launch-resolution' import { - realClaudeAuthenticated, realClaudeAuthStatus, - realClaudeAvailable, - realClaudeCommand + realClaudeCommand, + realClaudeSkipReason } from './claude-real-cli-availability-test-support' import { ClaudeStructuredSessionAdapter, @@ -79,93 +78,89 @@ async function waitForResolvedTranscript( } } -describe.skipIf(!realClaudeAvailable)('Claude structured real CLI handshake', () => { - it.skipIf(!realClaudeAuthenticated)( - 'proves a pre-minted session before the first user message', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const cwd = await mkdtemp(join(tmpdir(), 'orca-command-init-')) - await mkdir(join(cwd, '.claude', 'commands'), { recursive: true }) - await writeFile( - join(cwd, '.claude', 'commands', 'orca-init-catalog-proof.md'), - '---\ndescription: Initialization catalog proof\n---\nReply with OK.\n' - ) - const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd) +describe('Claude structured real CLI handshake', () => { + beforeEach((ctx) => { + if (realClaudeSkipReason) { + ctx.skip(realClaudeSkipReason) + } + }) - try { - const acquisition = await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli' - }) - const observedSubtypes = events.flatMap((event) => - event.type === 'message' ? [event.message.subtype] : [] - ) + it('proves a pre-minted session before the first user message', async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const cwd = await mkdtemp(join(tmpdir(), 'orca-command-init-')) + await mkdir(join(cwd, '.claude', 'commands'), { recursive: true }) + await writeFile( + join(cwd, '.claude', 'commands', 'orca-init-catalog-proof.md'), + '---\ndescription: Initialization catalog proof\n---\nReply with OK.\n' + ) + const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd) - expect(acquisition.link.handle).toMatchObject({ - provider: 'claude', - sessionId: providerSessionId, - // Init/SessionStart UUIDs are protocol frames, not resumable - // main-transcript leaves; no cursor exists before the first user turn. - leafUuid: null - }) - expect(observedSubtypes).toContain('hook_started') - expect(adapter.readCommands('real-cli-handshake')).toContainEqual({ - name: 'orca-init-catalog-proof', - kind: 'command', - kindUnspecified: true - }) - expect( - adapter.readCommands('real-cli-handshake')?.some(({ name }) => name === 'help') - ).toBe(false) - } finally { - await adapter.closeAll() - await rm(cwd, { recursive: true, force: true }) - } - }, - 10_000 - ) + try { + const acquisition = await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli' + }) + const observedSubtypes = events.flatMap((event) => + event.type === 'message' ? [event.message.subtype] : [] + ) + + expect(acquisition.link.handle).toMatchObject({ + provider: 'claude', + sessionId: providerSessionId, + // Init/SessionStart UUIDs are protocol frames, not resumable + // main-transcript leaves; no cursor exists before the first user turn. + leafUuid: null + }) + expect(observedSubtypes).toContain('hook_started') + expect(adapter.readCommands('real-cli-handshake')).toContainEqual({ + name: 'orca-init-catalog-proof', + kind: 'command', + kindUnspecified: true + }) + expect(adapter.readCommands('real-cli-handshake')?.some(({ name }) => name === 'help')).toBe( + false + ) + } finally { + await adapter.closeAll() + await rm(cwd, { recursive: true, force: true }) + } + }, 10_000) // Unit tests can only pin the shape we read, which is exactly how the blank // Effort pill survived every gate: the fixture invented an `effortLevel` on a // frame the CLI does not send. This asserts both halves against the live // binary — that get_settings reports the effort, and that init does not. - it.skipIf(!realClaudeAuthenticated)( - 'reports the current effort through get_settings and never on the init frame', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + it('reports the current effort through get_settings and never on the init frame', async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events) - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-effort' - }) - const published = events.flatMap((event) => - event.type === 'message' ? [event.message] : [] - ) - const options = await adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-effort' + }) + const published = events.flatMap((event) => (event.type === 'message' ? [event.message] : [])) + const options = await adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) - expect(published.length).toBeGreaterThan(0) - // Not just the init frame: no frame the CLI publishes carries an effort - // at all. Goes red the day one does, which is when the simpler fix - // becomes available. Which frame proves the session varies by host, so - // this asserts over all of them rather than picking one. - expect(published.filter((frame) => 'effortLevel' in frame)).toEqual([]) - // Goes red if `effective.effortLevel` is renamed or dropped, which no - // fixture-backed test can see. - expect(options.current.effort).toEqual(expect.any(String)) - } finally { - await adapter.closeAll() - } - }, - 15_000 - ) + expect(published.length).toBeGreaterThan(0) + // Not just the init frame: no frame the CLI publishes carries an effort + // at all. Goes red the day one does, which is when the simpler fix + // becomes available. Which frame proves the session varies by host, so + // this asserts over all of them rather than picking one. + expect(published.filter((frame) => 'effortLevel' in frame)).toEqual([]) + // Goes red if `effective.effortLevel` is renamed or dropped, which no + // fixture-backed test can see. + expect(options.current.effort).toEqual(expect.any(String)) + } finally { + await adapter.closeAll() + } + }, 15_000) // Mobile native chat never reads the structured journal — it reads the CLI's own // transcript through native-chat/session-file-resolver.ts. So this resolves the way @@ -175,42 +170,42 @@ describe.skipIf(!realClaudeAvailable)('Claude structured real CLI handshake', () // would move both sides together and stay green in exactly the environment that // blacks mobile out. // The turn is what creates the file: an init-only handshake writes nothing. - it.skipIf(!realClaudeAuthenticated || !realClaudeAuthStatus?.projectsDirectory)( - 'writes its transcript where the mobile session-file resolver looks for it', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const adapter = realAdapter(providerSessionId, claudeConfigDir) - const cliProjectsDir = realClaudeAuthStatus?.projectsDirectory as string + it('writes its transcript where the mobile session-file resolver looks for it', async (ctx) => { + ctx.skip( + !realClaudeAuthStatus?.projectsDirectory, + 'claude auth status reports no projectsDirectory' + ) + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const adapter = realAdapter(providerSessionId, claudeConfigDir) + const cliProjectsDir = realClaudeAuthStatus?.projectsDirectory as string - let transcriptPath: string | null = null - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-transcript' - }) - await adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId: 'real-cli-transcript-1', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, - fence: 1 - }) - transcriptPath = await waitForResolvedTranscript(providerSessionId) - } finally { - await adapter.closeAll() - } + let transcriptPath: string | null = null + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-transcript' + }) + await adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId: 'real-cli-transcript-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, + fence: 1 + }) + transcriptPath = await waitForResolvedTranscript(providerSessionId) + } finally { + await adapter.closeAll() + } - expect(transcriptPath).not.toBeNull() - expect(basename(transcriptPath ?? '')).toBe(`${providerSessionId}.jsonl`) - // `//.jsonl` - expect(relative(cliProjectsDir, transcriptPath ?? '').split(/[\\/]/)).toHaveLength(2) - // And the pinned account home is that same root, so the host-side leaf recovery - // (structured-claude-runtime-adapter.ts:64) and mobile agree. - expect(join(claudeConfigDir, 'projects')).toBe(cliProjectsDir) - }, - 45_000 - ) + expect(transcriptPath).not.toBeNull() + expect(basename(transcriptPath ?? '')).toBe(`${providerSessionId}.jsonl`) + // `//.jsonl` + expect(relative(cliProjectsDir, transcriptPath ?? '').split(/[\\/]/)).toHaveLength(2) + // And the pinned account home is that same root, so the host-side leaf recovery + // (structured-claude-runtime-adapter.ts:64) and mobile agree. + expect(join(claudeConfigDir, 'projects')).toBe(cliProjectsDir) + }, 45_000) // The model half of the same lesson: a fixture can only pin the shape we read. // set_model answers success for a model it never resolves — a nonexistent id is @@ -219,146 +214,138 @@ describe.skipIf(!realClaudeAvailable)('Claude structured real CLI handshake', () // asserts that frame carries the resolved model against the live binary; it goes // red the day the CLI stops reporting it, which is the day the confirmation // silently degrades to echoing back whatever Orca sent. - it.skipIf(!realClaudeAuthenticated)( - 'reports the model it adopted on the init frame that opens each turn', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + it('reports the model it adopted on the init frame that opens each turn', async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events) - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-model' - }) - await adapter.setOption({ - sessionId: 'real-cli-handshake', - key: 'model', - value: 'haiku', - fence: 1 - }) - const before = events.length - await adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId: 'real-cli-model-1', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Say ok' }] }, - fence: 1 - }) - const deadline = Date.now() + 60_000 - let frames: Record[] = [] - for (;;) { - frames = events - .slice(before) - .flatMap((event) => - event.type === 'message' && - event.message.type === 'system' && - event.message.subtype === 'init' - ? [event.message] - : [] - ) - if (frames.length > 0 || Date.now() >= deadline) { - break - } - await new Promise((resolve) => setTimeout(resolve, 250)) + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-model' + }) + await adapter.setOption({ + sessionId: 'real-cli-handshake', + key: 'model', + value: 'haiku', + fence: 1 + }) + const before = events.length + await adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId: 'real-cli-model-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Say ok' }] }, + fence: 1 + }) + const deadline = Date.now() + 60_000 + let frames: Record[] = [] + for (;;) { + frames = events + .slice(before) + .flatMap((event) => + event.type === 'message' && + event.message.type === 'system' && + event.message.subtype === 'init' + ? [event.message] + : [] + ) + if (frames.length > 0 || Date.now() >= deadline) { + break } - - expect(frames).not.toHaveLength(0) - // Both halves: the field exists, and it names the model the picker asked - // for in the catalog's resolved shape rather than the id Orca sent. - expect(frames[0]?.model).toEqual(expect.any(String)) - expect(frames[0]?.model).toBe('claude-haiku-4-5-20251001') - await expect( - adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) - ).resolves.toMatchObject({ current: { model: 'haiku' } }) - } finally { - await adapter.closeAll() + await new Promise((resolve) => setTimeout(resolve, 250)) } - }, - 90_000 - ) + + expect(frames).not.toHaveLength(0) + // Both halves: the field exists, and it names the model the picker asked + // for in the catalog's resolved shape rather than the id Orca sent. + expect(frames[0]?.model).toEqual(expect.any(String)) + expect(frames[0]?.model).toBe('claude-haiku-4-5-20251001') + await expect( + adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) + ).resolves.toMatchObject({ current: { model: 'haiku' } }) + } finally { + await adapter.closeAll() + } + }, 90_000) // Orca installs a SessionStart hook, so its frame proves most real starts before the turn's // system/init, the only frame that says this CLI can cancel what it queued. - it.skipIf(!realClaudeAuthenticated)( - 'withdraws a follow-up queued behind a turn that is stopped, behind a SessionStart hook', - async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const cwd = await mkdtemp(join(tmpdir(), 'orca-queued-stop-')) - await mkdir(join(cwd, '.claude'), { recursive: true }) - await writeFile( - join(cwd, '.claude', 'settings.json'), - JSON.stringify({ - hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] } - }) - ) - const events: ClaudeStructuredSessionEvent[] = [] - const settlements: unknown[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd, (settlement) => - settlements.push(settlement) - ) - const send = (clientMessageId: string, text: string) => - adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId, - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, - fence: 1 - }) - const waitFor = async (found: () => boolean): Promise => { - const deadline = Date.now() + 60_000 - while (!found() && Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, 100)) - } - return found() + it('withdraws a follow-up queued behind a turn that is stopped, behind a SessionStart hook', async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const cwd = await mkdtemp(join(tmpdir(), 'orca-queued-stop-')) + await mkdir(join(cwd, '.claude'), { recursive: true }) + await writeFile( + join(cwd, '.claude', 'settings.json'), + JSON.stringify({ + hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] } + }) + ) + const events: ClaudeStructuredSessionEvent[] = [] + const settlements: unknown[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd, (settlement) => + settlements.push(settlement) + ) + const send = (clientMessageId: string, text: string) => + adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, + fence: 1 + }) + const waitFor = async (found: () => boolean): Promise => { + const deadline = Date.now() + 60_000 + while (!found() && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)) } + return found() + } - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-queued-stop' - }) - await send('real-cli-queued-stop-a', 'Count from 1 to 400, one number per line.') - // A's reply is streaming, so the next send queues behind its turn. - await expect( - waitFor(() => - events.some( - (event) => - event.type === 'message' && - event.message.type === 'stream_event' && - JSON.stringify(event.message).includes('text_delta') - ) + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-queued-stop' + }) + await send('real-cli-queued-stop-a', 'Count from 1 to 400, one number per line.') + // A's reply is streaming, so the next send queues behind its turn. + await expect( + waitFor(() => + events.some( + (event) => + event.type === 'message' && + event.message.type === 'stream_event' && + JSON.stringify(event.message).includes('text_delta') ) - ).resolves.toBe(true) - await send('real-cli-queued-stop-b', 'Say the word banana.') - - await expect( - adapter.cancelTurn({ - sessionId: 'real-cli-handshake', - turnId: 'turn-a', - fence: 1, - resolveLiveTurnId: () => 'turn-a', - // What the host reads for B: handed over, not yet answered. - dispatchStatus: { state: 'pending', recovered: false } - }) - ).resolves.toEqual({ cancelled: true }) + ) + ).resolves.toBe(true) + await send('real-cli-queued-stop-b', 'Say the word banana.') - expect(settlements).toContainEqual({ + await expect( + adapter.cancelTurn({ sessionId: 'real-cli-handshake', - clientMessageId: 'real-cli-queued-stop-b', - state: 'rejected', - reason: 'provider_cancelled_before_start', - rejection: { kind: 'cancelled' } + turnId: 'turn-a', + fence: 1, + resolveLiveTurnId: () => 'turn-a', + // What the host reads for B: handed over, not yet answered. + dispatchStatus: { state: 'pending', recovered: false } }) - } finally { - await adapter.closeAll() - await rm(cwd, { recursive: true, force: true }) - } - }, - 150_000 - ) + ).resolves.toEqual({ cancelled: true }) + + expect(settlements).toContainEqual({ + sessionId: 'real-cli-handshake', + clientMessageId: 'real-cli-queued-stop-b', + state: 'rejected', + reason: 'provider_cancelled_before_start', + rejection: { kind: 'cancelled' } + }) + } finally { + await adapter.closeAll() + await rm(cwd, { recursive: true, force: true }) + } + }, 150_000) it('turns a real silent unauthenticated startup into sign-in guidance', async () => { const claudeConfigDir = await mkdtemp(join(tmpdir(), 'orca-claude-no-auth-')) diff --git a/src/main/claude/claude-supervised-stop-real-cli.test.ts b/src/main/claude/claude-supervised-stop-real-cli.test.ts index 82432e3fd6e6..089b9ddbc5d9 100644 --- a/src/main/claude/claude-supervised-stop-real-cli.test.ts +++ b/src/main/claude/claude-supervised-stop-real-cli.test.ts @@ -2,9 +2,13 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { homedir, tmpdir } from 'node:os' import { join } from 'node:path' import { randomUUID } from 'node:crypto' -import { afterEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { runProcess } from '../../shared/child-process/run-process' import { resolveSessionFilePath } from '../native-chat/session-file-resolver' +import { + REAL_AGENT_TESTS_SKIP_NOTE, + realAgentTestsEnabled +} from '../real-agent-tests-opt-in-test-support' import { SUPERVISED_GRACEFUL_EXIT_MS } from './claude-child-exit-proof-ladder' import { openClaudeStreamJsonConnection, @@ -16,7 +20,7 @@ import { } from './claude-structured-launch-resolution' // Opt-in only: spends a real (haiku) turn per case. Run it in an isolated HOME with -// ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS when auth lives in a settings file. +// ORCA_RUN_REAL_AGENT_TESTS=1 and ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS when auth lives in a settings file. const CLAUDE_BIN = process.env.ORCA_REAL_CLAUDE_BIN ?? '' const enabled = process.env.ORCA_REAL_CLAUDE_SUPERVISED_STOP === '1' && @@ -183,6 +187,10 @@ afterEach(async () => { }) describe.runIf(enabled)('real Claude stopped through the POSIX supervisor', () => { + beforeEach((ctx) => { + ctx.skip(!realAgentTestsEnabled(), REAL_AGENT_TESTS_SKIP_NOTE) + }) + it.each(['close mid-tool', 'SIGTERM to the supervisor mid-tool', 'close while idle'])( '%s: stops Claude and its tool, keeps the transcript line-atomic, and resumes', async (stopCase) => { diff --git a/src/main/codex/codex-index-heal-binary-contract.test.ts b/src/main/codex/codex-index-heal-binary-contract.test.ts index 5580fc02d1d9..97878ead3191 100644 --- a/src/main/codex/codex-index-heal-binary-contract.test.ts +++ b/src/main/codex/codex-index-heal-binary-contract.test.ts @@ -3,7 +3,12 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { promisify } from 'node:util' -import { afterEach, beforeAll, describe, expect, it } from 'vitest' +import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { + REAL_AGENT_TESTS_ENV, + REAL_AGENT_TESTS_SKIP_NOTE, + realAgentTestsEnabled +} from '../real-agent-tests-opt-in-test-support' import SyncDatabase from '../sqlite/sync-database' import { runCodexAppServerSession, type CodexAppServerRpc } from './codex-app-server-session' import { findNewestCodexStateDbPath } from './codex-state-db' @@ -28,17 +33,21 @@ const describeCodexContract = binary ? describe : describe.skip // Why this guard: skipping is the right local-dev default, but a CI job whose whole // purpose is the real binary must not pass by reporting zero assertions. The job sets -// ORCA_CODEX_CONTRACT_REQUIRED=1, which turns a missing binary into a red test. -describe.runIf(process.env.ORCA_CODEX_CONTRACT_REQUIRED === '1' && !binary)( - 'codex binary index-heal contract prerequisites', - () => { - it('was given a Codex binary to run against', () => { - expect.fail( - 'ORCA_CODEX_CONTRACT_REQUIRED=1 but ORCA_CODEX_CONTRACT_BINARY is unset, so the contract would have silently skipped' - ) - }) - } -) +// ORCA_CODEX_CONTRACT_REQUIRED=1, which turns a missing binary or opt-in into a red test. +describe.runIf( + process.env.ORCA_CODEX_CONTRACT_REQUIRED === '1' && (!binary || !realAgentTestsEnabled()) +)('codex binary index-heal contract prerequisites', () => { + it.runIf(!binary)('was given a Codex binary to run against', () => { + expect.fail( + 'ORCA_CODEX_CONTRACT_REQUIRED=1 but ORCA_CODEX_CONTRACT_BINARY is unset, so the contract would have silently skipped' + ) + }) + it.runIf(!realAgentTestsEnabled())('was allowed to start a real agent CLI', () => { + expect.fail( + `ORCA_CODEX_CONTRACT_REQUIRED=1 but ${REAL_AGENT_TESTS_ENV} is not 1, so the contract would have silently skipped` + ) + }) +}) // Why fixed: `thread/read` never reaches the network, and a whole session is // spawn + initialize + one RPC. A generous ceiling still fails fast on a wedged child. @@ -56,6 +65,9 @@ describeCodexContract( const disposableHomes: string[] = [] beforeAll(async () => { + if (!realAgentTestsEnabled()) { + return + } // Why assert the version: the whole point of a real-binary check is that the // binary drifts. A job that quietly ran some other Codex would report a // contract this repo never verified. @@ -65,6 +77,10 @@ describeCodexContract( expect(stdout.trim()).toBe(`codex-cli ${expectedVersion}`) }) + beforeEach((ctx) => { + ctx.skip(!realAgentTestsEnabled(), REAL_AGENT_TESTS_SKIP_NOTE) + }) + afterEach(() => { while (disposableHomes.length > 0) { rmSync(disposableHomes.pop() as string, { recursive: true, force: true }) diff --git a/src/main/hermes/hook-service.test.ts b/src/main/hermes/hook-service.test.ts index 3158b551480a..a7df55e7b219 100644 --- a/src/main/hermes/hook-service.test.ts +++ b/src/main/hermes/hook-service.test.ts @@ -7,6 +7,10 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { parse } from 'yaml' import { makePaneKey } from '../../shared/stable-pane-id' +import { + REAL_AGENT_TESTS_SKIP_NOTE, + realAgentTestsEnabled +} from '../real-agent-tests-opt-in-test-support' import { HermesHookService, _internals } from './hook-service' const PANE_KEY = makePaneKey('tab-1', '11111111-1111-4111-8111-111111111111') @@ -106,7 +110,8 @@ describe('HermesHookService', () => { expect(status.detail).toContain('not enabled') }) - it('is visible to the real hermes CLI when hermes is installed', () => { + it('is visible to the real hermes CLI when hermes is installed', (ctx) => { + ctx.skip(!realAgentTestsEnabled(), REAL_AGENT_TESTS_SKIP_NOTE) const hermesAvailable = spawnSync('hermes', ['--version'], { encoding: 'utf-8' }).status === 0 if (!hermesAvailable) { return diff --git a/src/main/real-agent-tests-opt-in-test-support.ts b/src/main/real-agent-tests-opt-in-test-support.ts new file mode 100644 index 000000000000..bf1c15ca467a --- /dev/null +++ b/src/main/real-agent-tests-opt-in-test-support.ts @@ -0,0 +1,10 @@ +// The one switch every test that starts a real agent CLI checks first. Those tests run the +// CLI against the developer's own config and account, so a broad local run must never start one. + +export const REAL_AGENT_TESTS_ENV = 'ORCA_RUN_REAL_AGENT_TESTS' + +export const REAL_AGENT_TESTS_SKIP_NOTE = `starts a real agent CLI; set ${REAL_AGENT_TESTS_ENV}=1 to run it` + +export function realAgentTestsEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return env[REAL_AGENT_TESTS_ENV] === '1' +} diff --git a/src/main/runtime/orchestration-codex-real-pty.integration.test.ts b/src/main/runtime/orchestration-codex-real-pty.integration.test.ts index 28be6b513a75..3b9c69a6e726 100644 --- a/src/main/runtime/orchestration-codex-real-pty.integration.test.ts +++ b/src/main/runtime/orchestration-codex-real-pty.integration.test.ts @@ -3,13 +3,17 @@ import { mkdtempSync, mkdirSync, realpathSync, rmSync, writeFileSync } from 'nod import { tmpdir } from 'node:os' import { join } from 'node:path' import * as pty from 'node-pty' -import { expect, it, vi } from 'vitest' +import { beforeEach, expect, it, vi } from 'vitest' import { AgentHookServer } from '../agent-hooks/server' import { getManagedScript } from '../codex/codex-hook-script' import { getSyntheticAgentTerminalTitle } from '../../shared/synthetic-agent-title' import { extractAllOscTitles } from '../../shared/osc-title-extraction' import { extractOscTitleScanTail } from '../../shared/osc-title-scan-tail' import { settledWriteStub } from '../providers/settled-pty-write-stub' +import { + REAL_AGENT_TESTS_SKIP_NOTE, + realAgentTestsEnabled +} from '../real-agent-tests-opt-in-test-support' import { createBoundRun, createDatabase, @@ -36,6 +40,10 @@ const trials = (['before', 'after'] as const).flatMap((arrival) => ) const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) +beforeEach((ctx) => { + ctx.skip(!realAgentTestsEnabled(), REAL_AGENT_TESTS_SKIP_NOTE) +}) + it.skipIf(!binary || process.platform === 'win32').each(trials)( 'submits mail arriving $arrival a real Codex completion (trial $trial)', async ({ arrival }) => { From 3019888e4c8344f04070b445faf87b1a1ba0c841 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:13:08 -0700 Subject: [PATCH 2/4] test: type the spawnSync mock's arguments --- .../claude-real-cli-availability-test-support.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/claude/claude-real-cli-availability-test-support.test.ts b/src/main/claude/claude-real-cli-availability-test-support.test.ts index a9964cb221e0..54a57e91c1be 100644 --- a/src/main/claude/claude-real-cli-availability-test-support.test.ts +++ b/src/main/claude/claude-real-cli-availability-test-support.test.ts @@ -6,7 +6,13 @@ import { const { spawnMock, spawnSyncMock } = vi.hoisted(() => ({ spawnMock: vi.fn(), - spawnSyncMock: vi.fn(() => ({ status: 1, signal: null, stdout: '', stderr: '', pid: 0 })) + spawnSyncMock: vi.fn<(program: string, args: readonly string[]) => unknown>(() => ({ + status: 1, + signal: null, + stdout: '', + stderr: '', + pid: 0 + })) })) // Why the lowest layer: nothing below this helper may start a process, whichever wrapper it uses. From 991721ab174e1fdba50c240cd34f58c05c764b28 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:26:53 -0700 Subject: [PATCH 3/4] test: gate real-agent tests through their existing skip conditions Revert the Claude real-CLI suites to main and make the helper's existing realClaudeAvailable/realClaudeAuthenticated false without the opt-in, so their gates skip with zero edits. The other suites get a one-condition change instead of new hooks. --- ...real-cli-availability-test-support.test.ts | 10 +- ...aude-real-cli-availability-test-support.ts | 21 +- .../claude-structured-real-cli-fold.test.ts | 265 +++++----- .../claude/claude-structured-real-cli.test.ts | 475 +++++++++--------- .../claude-supervised-stop-real-cli.test.ts | 15 +- .../codex-index-heal-binary-contract.test.ts | 42 +- src/main/hermes/hook-service.test.ts | 12 +- .../real-agent-tests-opt-in-test-support.ts | 2 - ...tration-codex-real-pty.integration.test.ts | 13 +- 9 files changed, 414 insertions(+), 441 deletions(-) diff --git a/src/main/claude/claude-real-cli-availability-test-support.test.ts b/src/main/claude/claude-real-cli-availability-test-support.test.ts index 54a57e91c1be..afb1752d94a9 100644 --- a/src/main/claude/claude-real-cli-availability-test-support.test.ts +++ b/src/main/claude/claude-real-cli-availability-test-support.test.ts @@ -45,9 +45,11 @@ describe('real Claude CLI availability', () => { it('skips without spawning anything unless real-agent tests are opted into', async () => { vi.stubEnv(REAL_AGENT_TESTS_ENV, undefined) - const { realClaudeSkipReason, realClaudeAuthStatus } = await loadAvailability() + const { realClaudeAvailable, realClaudeAuthenticated, realClaudeAuthStatus } = + await loadAvailability() - expect(realClaudeSkipReason).toContain(`${REAL_AGENT_TESTS_ENV}=1`) + expect(realClaudeAvailable).toBe(false) + expect(realClaudeAuthenticated).toBe(false) expect(realClaudeAuthStatus).toBeNull() expect(spawnSyncMock).not.toHaveBeenCalled() expect(spawnMock).not.toHaveBeenCalled() @@ -56,10 +58,10 @@ describe('real Claude CLI availability', () => { it('probes the CLI once opted into', async () => { vi.stubEnv(REAL_AGENT_TESTS_ENV, '1') - const { realClaudeSkipReason } = await loadAvailability() + const { realClaudeAvailable } = await loadAvailability() // The mocked probe fails, so this proves the gate, not a real CLI. - expect(realClaudeSkipReason).toMatch(/no runnable Claude CLI/) + expect(realClaudeAvailable).toBe(false) expect(spawnSyncMock).toHaveBeenCalledTimes(1) expect(spawnSyncMock.mock.calls[0]?.[1]).toContain('--version') }) diff --git a/src/main/claude/claude-real-cli-availability-test-support.ts b/src/main/claude/claude-real-cli-availability-test-support.ts index 074198fe5a6b..bf61a1ba0abe 100644 --- a/src/main/claude/claude-real-cli-availability-test-support.ts +++ b/src/main/claude/claude-real-cli-availability-test-support.ts @@ -1,5 +1,5 @@ // Whether a real, signed-in Claude CLI is present — the gate every real-CLI -// suite skips on. Probed once per test process, and only when real-agent tests are opted into. +// suite skips on. Probed once per test process, and only with ORCA_RUN_REAL_AGENT_TESTS=1. import { homedir } from 'node:os' import { join } from 'node:path' @@ -7,10 +7,7 @@ import { runProcessSync, type ProcessResult } from '../../shared/child-process/r import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution' import { CLAUDE_AUTH_ENV_VARS } from '../claude-accounts/environment' import { resolveClaudeCommand } from '../codex-cli/command' -import { - REAL_AGENT_TESTS_SKIP_NOTE, - realAgentTestsEnabled -} from '../real-agent-tests-opt-in-test-support' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' export const realClaudeCommand = resolveClaudeCommand() @@ -29,9 +26,8 @@ function probeRealClaude(args: string[]): ProcessResult | null { } } -const optedIn = realAgentTestsEnabled() - -const realClaudeAvailable = optedIn && probeRealClaude(['--version'])?.code === 0 +export const realClaudeAvailable = + realAgentTestsEnabled() && probeRealClaude(['--version'])?.code === 0 /** The CLI's own account report — the only source of truth for where it writes that * is not derived from Orca's own path expressions. */ @@ -50,14 +46,7 @@ export const realClaudeAuthStatus = (() => { } })() -/** Why every real-CLI test in this process skips, or null when they may run. */ -export const realClaudeSkipReason: string | null = !optedIn - ? REAL_AGENT_TESTS_SKIP_NOTE - : !realClaudeAvailable - ? `no runnable Claude CLI at ${realClaudeCommand}` - : realClaudeAuthStatus?.loggedIn !== true - ? 'the Claude CLI is not signed in' - : null +export const realClaudeAuthenticated = realClaudeAuthStatus?.loggedIn === true /** The config dir and env auth the availability probe above saw, for a real-CLI launch. * The connection strips an inherited CLAUDE_CONFIG_DIR and inherited auth vars, so diff --git a/src/main/claude/claude-structured-real-cli-fold.test.ts b/src/main/claude/claude-structured-real-cli-fold.test.ts index 1aa0e646fb65..5ce21d83f170 100644 --- a/src/main/claude/claude-structured-real-cli-fold.test.ts +++ b/src/main/claude/claude-structured-real-cli-fold.test.ts @@ -1,6 +1,6 @@ // The fold receipt against the real CLI: a message sent while a turn runs is // folded by the CLI into the running turn, and Orca must keep ONE turn row — -// no interrupted marking, no second bar. Runs only when opted in and a signed-in Claude CLI +// no interrupted marking, no second bar. Runs only where a signed-in Claude CLI // exists, like the rest of the real-CLI suite. Live sessions prove the session // from a SessionStart hook frame BEFORE system/init arrives, which is exactly // the path the fixture harness cannot fake end to end. @@ -9,7 +9,7 @@ import { randomUUID } from 'node:crypto' import { mkdtemp } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import type { AgentJournalItemBody, AgentSessionJournalIdentity @@ -22,9 +22,10 @@ import { type ClaudeStructuredSessionEvent } from './claude-structured-session-adapter' import { + realClaudeAuthenticated, + realClaudeAvailable, realClaudeCommand, - realClaudeLaunchHome, - realClaudeSkipReason + realClaudeLaunchHome } from './claude-real-cli-availability-test-support' const SESSION_ID = 'real-cli-fold' @@ -56,140 +57,138 @@ async function until(check: () => boolean, timeoutMs: number): Promise return check() } -describe('Claude structured real CLI fold', () => { - beforeEach((ctx) => { - if (realClaudeSkipReason) { - ctx.skip(realClaudeSkipReason) - } - }) - - it('keeps one turn row when the CLI folds a mid-turn send', async () => { - const providerSessionId = randomUUID() - const { claudeConfigDir, env } = realClaudeLaunchHome() - const cwd = await mkdtemp(join(tmpdir(), 'orca-real-fold-')) - const events: ClaudeStructuredSessionEvent[] = [] - const turnRows: NonNullable>[] = [] - const sink: StructuredAgentSessionEventSink = { - appendItem: (_identity, body: AgentJournalItemBody) => { - const turn = readAgentJournalTurn(body) - if (turn) { - turnRows.push(turn) - } - }, - appendTombstone: () => {}, - publish: () => {} - } - const settled = vi.fn() - const adapter = new ClaudeStructuredSessionAdapter({ - resolveLaunch: async () => ({ - pathToClaudeCodeExecutable: realClaudeCommand, - options: { - ...CLAUDE_STRUCTURED_BASE_OPTIONS, - extraArgs: { - ...CLAUDE_STRUCTURED_BASE_OPTIONS.extraArgs, - settings: FOLD_SESSION_SETTINGS - }, - sessionId: providerSessionId +describe.skipIf(!realClaudeAvailable)('Claude structured real CLI fold', () => { + it.skipIf(!realClaudeAuthenticated)( + 'keeps one turn row when the CLI folds a mid-turn send', + async () => { + const providerSessionId = randomUUID() + const { claudeConfigDir, env } = realClaudeLaunchHome() + const cwd = await mkdtemp(join(tmpdir(), 'orca-real-fold-')) + const events: ClaudeStructuredSessionEvent[] = [] + const turnRows: NonNullable>[] = [] + const sink: StructuredAgentSessionEventSink = { + appendItem: (_identity, body: AgentJournalItemBody) => { + const turn = readAgentJournalTurn(body) + if (turn) { + turnRows.push(turn) + } }, - cwd, - env, - claudeConfigDir, - providerSessionId, - resumeLeafUuid: null, - resumesTranscript: false, - continuesChain: false - }), - onEvent: (event) => events.push(event), - onDispatchSettledLate: settled, - readProcessStartTime: async () => 1 - }) - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-fold', - events: sink - }) - await adapter.awaitStarted(SESSION_ID) - // Startup proved from the SessionStart hook frame, with no init yet. - expect( - events.some( - (event) => - event.type === 'message' && - event.message.type === 'system' && - event.message.subtype === 'init' - ) - ).toBe(false) - - await expect( - adapter.dispatch({ - sessionId: SESSION_ID, - clientMessageId: 'client-A', - body: { - kind: 'message', - role: 'user', - blocks: [ - { - type: 'text', - text: 'Use the Bash tool to run `sleep 8` two separate times, one call at a time, waiting for each. Then reply exactly: FIRST DONE' - } - ] + appendTombstone: () => {}, + publish: () => {} + } + const settled = vi.fn() + const adapter = new ClaudeStructuredSessionAdapter({ + resolveLaunch: async () => ({ + pathToClaudeCodeExecutable: realClaudeCommand, + options: { + ...CLAUDE_STRUCTURED_BASE_OPTIONS, + extraArgs: { + ...CLAUDE_STRUCTURED_BASE_OPTIONS.extraArgs, + settings: FOLD_SESSION_SETTINGS + }, + sessionId: providerSessionId }, - requestedAt: Date.now(), - fence: 1 + cwd, + env, + claudeConfigDir, + providerSessionId, + resumeLeafUuid: null, + resumesTranscript: false, + continuesChain: false + }), + onEvent: (event) => events.push(event), + onDispatchSettledLate: settled, + readProcessStartTime: async () => 1 + }) + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-fold', + events: sink }) - ).resolves.toEqual({ state: 'admitted' }) + await adapter.awaitStarted(SESSION_ID) + // Startup proved from the SessionStart hook frame, with no init yet. + expect( + events.some( + (event) => + event.type === 'message' && + event.message.type === 'system' && + event.message.subtype === 'init' + ) + ).toBe(false) - // Wait for A's replay to open the turn, then send B mid-turn: the first - // `sleep 8` guarantees the CLI is still inside A's request cycle. - expect( - await until( - () => events.some((event) => event.type === 'message' && event.startsTurn === true), - 30_000 - ) - ).toBe(true) - await new Promise((resolve) => setTimeout(resolve, 2_000)) - await expect( - adapter.dispatch({ - sessionId: SESSION_ID, - clientMessageId: 'client-B', - body: { - kind: 'message', - role: 'user', - blocks: [{ type: 'text', text: 'Also say the word banana at the end of your reply.' }] - }, - requestedAt: Date.now(), - fence: 1 - }) - ).resolves.toEqual({ state: 'admitted' }) + await expect( + adapter.dispatch({ + sessionId: SESSION_ID, + clientMessageId: 'client-A', + body: { + kind: 'message', + role: 'user', + blocks: [ + { + type: 'text', + text: 'Use the Bash tool to run `sleep 8` two separate times, one call at a time, waiting for each. Then reply exactly: FIRST DONE' + } + ] + }, + requestedAt: Date.now(), + fence: 1 + }) + ).resolves.toEqual({ state: 'admitted' }) - const results = () => - events.flatMap((event) => - event.type === 'message' && event.message.type === 'result' ? [event.message] : [] - ) - expect(await until(() => results().length > 0, 90_000)).toBe(true) + // Wait for A's replay to open the turn, then send B mid-turn: the first + // `sleep 8` guarantees the CLI is still inside A's request cycle. + expect( + await until( + () => events.some((event) => event.type === 'message' && event.startsTurn === true), + 30_000 + ) + ).toBe(true) + await new Promise((resolve) => setTimeout(resolve, 2_000)) + await expect( + adapter.dispatch({ + sessionId: SESSION_ID, + clientMessageId: 'client-B', + body: { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: 'Also say the word banana at the end of your reply.' }] + }, + requestedAt: Date.now(), + fence: 1 + }) + ).resolves.toEqual({ state: 'admitted' }) - // The CLI folded: one result names both sends. (If this ever reports a - // lone send, the steer raced past the fold window — a miss, not a fold.) - const named = results().flatMap((message) => - Array.isArray(message.user_message_uuids) ? [message.user_message_uuids] : [] - ) - expect(named[0]).toHaveLength(2) + const results = () => + events.flatMap((event) => + event.type === 'message' && event.message.type === 'result' ? [event.message] : [] + ) + expect(await until(() => results().length > 0, 90_000)).toBe(true) - // ONE turn for the whole run: never interrupted, and B settled accepted - // into it under its own adopted uuid. - expect(turnRows.every((turn) => turn.state !== 'interrupted')).toBe(true) - expect([...new Set(turnRows.map((turn) => turn.turnId))]).toHaveLength(1) - expect(turnRows.at(-1)).toMatchObject({ state: 'completed' }) - expect(settled).toHaveBeenCalledWith( - expect.objectContaining({ - sessionId: SESSION_ID, - clientMessageId: 'client-B', - providerIdentity: expect.objectContaining({ provider: 'claude' }) - }) - ) - } finally { - await adapter.closeAll() - } - }, 150_000) + // The CLI folded: one result names both sends. (If this ever reports a + // lone send, the steer raced past the fold window — a miss, not a fold.) + const named = results().flatMap((message) => + Array.isArray(message.user_message_uuids) ? [message.user_message_uuids] : [] + ) + expect(named[0]).toHaveLength(2) + + // ONE turn for the whole run: never interrupted, and B settled accepted + // into it under its own adopted uuid. + expect(turnRows.every((turn) => turn.state !== 'interrupted')).toBe(true) + expect([...new Set(turnRows.map((turn) => turn.turnId))]).toHaveLength(1) + expect(turnRows.at(-1)).toMatchObject({ state: 'completed' }) + expect(settled).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + clientMessageId: 'client-B', + providerIdentity: expect.objectContaining({ provider: 'claude' }) + }) + ) + } finally { + await adapter.closeAll() + } + }, + 150_000 + ) }) diff --git a/src/main/claude/claude-structured-real-cli.test.ts b/src/main/claude/claude-structured-real-cli.test.ts index c3a6b9b1880e..20670147ca0d 100644 --- a/src/main/claude/claude-structured-real-cli.test.ts +++ b/src/main/claude/claude-structured-real-cli.test.ts @@ -2,14 +2,15 @@ import { randomUUID } from 'node:crypto' import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' import { homedir, tmpdir } from 'node:os' import { basename, join, relative } from 'node:path' -import { beforeEach, describe, expect, it } from 'vitest' +import { describe, expect, it } from 'vitest' import type { AgentSessionJournalIdentity } from '../../shared/agent-session-journal-types' import { resolveSessionFilePath } from '../native-chat/session-file-resolver' import { CLAUDE_STRUCTURED_BASE_OPTIONS } from './claude-structured-launch-resolution' import { + realClaudeAuthenticated, realClaudeAuthStatus, - realClaudeCommand, - realClaudeSkipReason + realClaudeAvailable, + realClaudeCommand } from './claude-real-cli-availability-test-support' import { ClaudeStructuredSessionAdapter, @@ -78,89 +79,93 @@ async function waitForResolvedTranscript( } } -describe('Claude structured real CLI handshake', () => { - beforeEach((ctx) => { - if (realClaudeSkipReason) { - ctx.skip(realClaudeSkipReason) - } - }) - - it('proves a pre-minted session before the first user message', async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const cwd = await mkdtemp(join(tmpdir(), 'orca-command-init-')) - await mkdir(join(cwd, '.claude', 'commands'), { recursive: true }) - await writeFile( - join(cwd, '.claude', 'commands', 'orca-init-catalog-proof.md'), - '---\ndescription: Initialization catalog proof\n---\nReply with OK.\n' - ) - const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd) - - try { - const acquisition = await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli' - }) - const observedSubtypes = events.flatMap((event) => - event.type === 'message' ? [event.message.subtype] : [] +describe.skipIf(!realClaudeAvailable)('Claude structured real CLI handshake', () => { + it.skipIf(!realClaudeAuthenticated)( + 'proves a pre-minted session before the first user message', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const cwd = await mkdtemp(join(tmpdir(), 'orca-command-init-')) + await mkdir(join(cwd, '.claude', 'commands'), { recursive: true }) + await writeFile( + join(cwd, '.claude', 'commands', 'orca-init-catalog-proof.md'), + '---\ndescription: Initialization catalog proof\n---\nReply with OK.\n' ) + const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd) - expect(acquisition.link.handle).toMatchObject({ - provider: 'claude', - sessionId: providerSessionId, - // Init/SessionStart UUIDs are protocol frames, not resumable - // main-transcript leaves; no cursor exists before the first user turn. - leafUuid: null - }) - expect(observedSubtypes).toContain('hook_started') - expect(adapter.readCommands('real-cli-handshake')).toContainEqual({ - name: 'orca-init-catalog-proof', - kind: 'command', - kindUnspecified: true - }) - expect(adapter.readCommands('real-cli-handshake')?.some(({ name }) => name === 'help')).toBe( - false - ) - } finally { - await adapter.closeAll() - await rm(cwd, { recursive: true, force: true }) - } - }, 10_000) + try { + const acquisition = await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli' + }) + const observedSubtypes = events.flatMap((event) => + event.type === 'message' ? [event.message.subtype] : [] + ) + + expect(acquisition.link.handle).toMatchObject({ + provider: 'claude', + sessionId: providerSessionId, + // Init/SessionStart UUIDs are protocol frames, not resumable + // main-transcript leaves; no cursor exists before the first user turn. + leafUuid: null + }) + expect(observedSubtypes).toContain('hook_started') + expect(adapter.readCommands('real-cli-handshake')).toContainEqual({ + name: 'orca-init-catalog-proof', + kind: 'command', + kindUnspecified: true + }) + expect( + adapter.readCommands('real-cli-handshake')?.some(({ name }) => name === 'help') + ).toBe(false) + } finally { + await adapter.closeAll() + await rm(cwd, { recursive: true, force: true }) + } + }, + 10_000 + ) // Unit tests can only pin the shape we read, which is exactly how the blank // Effort pill survived every gate: the fixture invented an `effortLevel` on a // frame the CLI does not send. This asserts both halves against the live // binary — that get_settings reports the effort, and that init does not. - it('reports the current effort through get_settings and never on the init frame', async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + it.skipIf(!realClaudeAuthenticated)( + 'reports the current effort through get_settings and never on the init frame', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events) - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-effort' - }) - const published = events.flatMap((event) => (event.type === 'message' ? [event.message] : [])) - const options = await adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-effort' + }) + const published = events.flatMap((event) => + event.type === 'message' ? [event.message] : [] + ) + const options = await adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) - expect(published.length).toBeGreaterThan(0) - // Not just the init frame: no frame the CLI publishes carries an effort - // at all. Goes red the day one does, which is when the simpler fix - // becomes available. Which frame proves the session varies by host, so - // this asserts over all of them rather than picking one. - expect(published.filter((frame) => 'effortLevel' in frame)).toEqual([]) - // Goes red if `effective.effortLevel` is renamed or dropped, which no - // fixture-backed test can see. - expect(options.current.effort).toEqual(expect.any(String)) - } finally { - await adapter.closeAll() - } - }, 15_000) + expect(published.length).toBeGreaterThan(0) + // Not just the init frame: no frame the CLI publishes carries an effort + // at all. Goes red the day one does, which is when the simpler fix + // becomes available. Which frame proves the session varies by host, so + // this asserts over all of them rather than picking one. + expect(published.filter((frame) => 'effortLevel' in frame)).toEqual([]) + // Goes red if `effective.effortLevel` is renamed or dropped, which no + // fixture-backed test can see. + expect(options.current.effort).toEqual(expect.any(String)) + } finally { + await adapter.closeAll() + } + }, + 15_000 + ) // Mobile native chat never reads the structured journal — it reads the CLI's own // transcript through native-chat/session-file-resolver.ts. So this resolves the way @@ -170,42 +175,42 @@ describe('Claude structured real CLI handshake', () => { // would move both sides together and stay green in exactly the environment that // blacks mobile out. // The turn is what creates the file: an init-only handshake writes nothing. - it('writes its transcript where the mobile session-file resolver looks for it', async (ctx) => { - ctx.skip( - !realClaudeAuthStatus?.projectsDirectory, - 'claude auth status reports no projectsDirectory' - ) - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const adapter = realAdapter(providerSessionId, claudeConfigDir) - const cliProjectsDir = realClaudeAuthStatus?.projectsDirectory as string + it.skipIf(!realClaudeAuthenticated || !realClaudeAuthStatus?.projectsDirectory)( + 'writes its transcript where the mobile session-file resolver looks for it', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const adapter = realAdapter(providerSessionId, claudeConfigDir) + const cliProjectsDir = realClaudeAuthStatus?.projectsDirectory as string - let transcriptPath: string | null = null - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-transcript' - }) - await adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId: 'real-cli-transcript-1', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, - fence: 1 - }) - transcriptPath = await waitForResolvedTranscript(providerSessionId) - } finally { - await adapter.closeAll() - } + let transcriptPath: string | null = null + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-transcript' + }) + await adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId: 'real-cli-transcript-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'hi' }] }, + fence: 1 + }) + transcriptPath = await waitForResolvedTranscript(providerSessionId) + } finally { + await adapter.closeAll() + } - expect(transcriptPath).not.toBeNull() - expect(basename(transcriptPath ?? '')).toBe(`${providerSessionId}.jsonl`) - // `//.jsonl` - expect(relative(cliProjectsDir, transcriptPath ?? '').split(/[\\/]/)).toHaveLength(2) - // And the pinned account home is that same root, so the host-side leaf recovery - // (structured-claude-runtime-adapter.ts:64) and mobile agree. - expect(join(claudeConfigDir, 'projects')).toBe(cliProjectsDir) - }, 45_000) + expect(transcriptPath).not.toBeNull() + expect(basename(transcriptPath ?? '')).toBe(`${providerSessionId}.jsonl`) + // `//.jsonl` + expect(relative(cliProjectsDir, transcriptPath ?? '').split(/[\\/]/)).toHaveLength(2) + // And the pinned account home is that same root, so the host-side leaf recovery + // (structured-claude-runtime-adapter.ts:64) and mobile agree. + expect(join(claudeConfigDir, 'projects')).toBe(cliProjectsDir) + }, + 45_000 + ) // The model half of the same lesson: a fixture can only pin the shape we read. // set_model answers success for a model it never resolves — a nonexistent id is @@ -214,138 +219,146 @@ describe('Claude structured real CLI handshake', () => { // asserts that frame carries the resolved model against the live binary; it goes // red the day the CLI stops reporting it, which is the day the confirmation // silently degrades to echoing back whatever Orca sent. - it('reports the model it adopted on the init frame that opens each turn', async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const events: ClaudeStructuredSessionEvent[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events) + it.skipIf(!realClaudeAuthenticated)( + 'reports the model it adopted on the init frame that opens each turn', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const events: ClaudeStructuredSessionEvent[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events) - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-model' - }) - await adapter.setOption({ - sessionId: 'real-cli-handshake', - key: 'model', - value: 'haiku', - fence: 1 - }) - const before = events.length - await adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId: 'real-cli-model-1', - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Say ok' }] }, - fence: 1 - }) - const deadline = Date.now() + 60_000 - let frames: Record[] = [] - for (;;) { - frames = events - .slice(before) - .flatMap((event) => - event.type === 'message' && - event.message.type === 'system' && - event.message.subtype === 'init' - ? [event.message] - : [] - ) - if (frames.length > 0 || Date.now() >= deadline) { - break + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-model' + }) + await adapter.setOption({ + sessionId: 'real-cli-handshake', + key: 'model', + value: 'haiku', + fence: 1 + }) + const before = events.length + await adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId: 'real-cli-model-1', + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text: 'Say ok' }] }, + fence: 1 + }) + const deadline = Date.now() + 60_000 + let frames: Record[] = [] + for (;;) { + frames = events + .slice(before) + .flatMap((event) => + event.type === 'message' && + event.message.type === 'system' && + event.message.subtype === 'init' + ? [event.message] + : [] + ) + if (frames.length > 0 || Date.now() >= deadline) { + break + } + await new Promise((resolve) => setTimeout(resolve, 250)) } - await new Promise((resolve) => setTimeout(resolve, 250)) - } - expect(frames).not.toHaveLength(0) - // Both halves: the field exists, and it names the model the picker asked - // for in the catalog's resolved shape rather than the id Orca sent. - expect(frames[0]?.model).toEqual(expect.any(String)) - expect(frames[0]?.model).toBe('claude-haiku-4-5-20251001') - await expect( - adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) - ).resolves.toMatchObject({ current: { model: 'haiku' } }) - } finally { - await adapter.closeAll() - } - }, 90_000) + expect(frames).not.toHaveLength(0) + // Both halves: the field exists, and it names the model the picker asked + // for in the catalog's resolved shape rather than the id Orca sent. + expect(frames[0]?.model).toEqual(expect.any(String)) + expect(frames[0]?.model).toBe('claude-haiku-4-5-20251001') + await expect( + adapter.readOptions({ sessionId: 'real-cli-handshake', fence: 1 }) + ).resolves.toMatchObject({ current: { model: 'haiku' } }) + } finally { + await adapter.closeAll() + } + }, + 90_000 + ) // Orca installs a SessionStart hook, so its frame proves most real starts before the turn's // system/init, the only frame that says this CLI can cancel what it queued. - it('withdraws a follow-up queued behind a turn that is stopped, behind a SessionStart hook', async () => { - const providerSessionId = randomUUID() - const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') - const cwd = await mkdtemp(join(tmpdir(), 'orca-queued-stop-')) - await mkdir(join(cwd, '.claude'), { recursive: true }) - await writeFile( - join(cwd, '.claude', 'settings.json'), - JSON.stringify({ - hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] } - }) - ) - const events: ClaudeStructuredSessionEvent[] = [] - const settlements: unknown[] = [] - const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd, (settlement) => - settlements.push(settlement) - ) - const send = (clientMessageId: string, text: string) => - adapter.dispatch({ - sessionId: 'real-cli-handshake', - clientMessageId, - body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, - fence: 1 - }) - const waitFor = async (found: () => boolean): Promise => { - const deadline = Date.now() + 60_000 - while (!found() && Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, 100)) + it.skipIf(!realClaudeAuthenticated)( + 'withdraws a follow-up queued behind a turn that is stopped, behind a SessionStart hook', + async () => { + const providerSessionId = randomUUID() + const claudeConfigDir = process.env.CLAUDE_CONFIG_DIR?.trim() || join(homedir(), '.claude') + const cwd = await mkdtemp(join(tmpdir(), 'orca-queued-stop-')) + await mkdir(join(cwd, '.claude'), { recursive: true }) + await writeFile( + join(cwd, '.claude', 'settings.json'), + JSON.stringify({ + hooks: { SessionStart: [{ hooks: [{ type: 'command', command: 'true' }] }] } + }) + ) + const events: ClaudeStructuredSessionEvent[] = [] + const settlements: unknown[] = [] + const adapter = realAdapter(providerSessionId, claudeConfigDir, events, cwd, (settlement) => + settlements.push(settlement) + ) + const send = (clientMessageId: string, text: string) => + adapter.dispatch({ + sessionId: 'real-cli-handshake', + clientMessageId, + body: { kind: 'message', role: 'user', blocks: [{ type: 'text', text }] }, + fence: 1 + }) + const waitFor = async (found: () => boolean): Promise => { + const deadline = Date.now() + 60_000 + while (!found() && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)) + } + return found() } - return found() - } - try { - await adapter.acquire({ - identity: identity(providerSessionId), - fence: 1, - spawnToken: 'real-cli-queued-stop' - }) - await send('real-cli-queued-stop-a', 'Count from 1 to 400, one number per line.') - // A's reply is streaming, so the next send queues behind its turn. - await expect( - waitFor(() => - events.some( - (event) => - event.type === 'message' && - event.message.type === 'stream_event' && - JSON.stringify(event.message).includes('text_delta') + try { + await adapter.acquire({ + identity: identity(providerSessionId), + fence: 1, + spawnToken: 'real-cli-queued-stop' + }) + await send('real-cli-queued-stop-a', 'Count from 1 to 400, one number per line.') + // A's reply is streaming, so the next send queues behind its turn. + await expect( + waitFor(() => + events.some( + (event) => + event.type === 'message' && + event.message.type === 'stream_event' && + JSON.stringify(event.message).includes('text_delta') + ) ) - ) - ).resolves.toBe(true) - await send('real-cli-queued-stop-b', 'Say the word banana.') + ).resolves.toBe(true) + await send('real-cli-queued-stop-b', 'Say the word banana.') + + await expect( + adapter.cancelTurn({ + sessionId: 'real-cli-handshake', + turnId: 'turn-a', + fence: 1, + resolveLiveTurnId: () => 'turn-a', + // What the host reads for B: handed over, not yet answered. + dispatchStatus: { state: 'pending', recovered: false } + }) + ).resolves.toEqual({ cancelled: true }) - await expect( - adapter.cancelTurn({ + expect(settlements).toContainEqual({ sessionId: 'real-cli-handshake', - turnId: 'turn-a', - fence: 1, - resolveLiveTurnId: () => 'turn-a', - // What the host reads for B: handed over, not yet answered. - dispatchStatus: { state: 'pending', recovered: false } + clientMessageId: 'real-cli-queued-stop-b', + state: 'rejected', + reason: 'provider_cancelled_before_start', + rejection: { kind: 'cancelled' } }) - ).resolves.toEqual({ cancelled: true }) - - expect(settlements).toContainEqual({ - sessionId: 'real-cli-handshake', - clientMessageId: 'real-cli-queued-stop-b', - state: 'rejected', - reason: 'provider_cancelled_before_start', - rejection: { kind: 'cancelled' } - }) - } finally { - await adapter.closeAll() - await rm(cwd, { recursive: true, force: true }) - } - }, 150_000) + } finally { + await adapter.closeAll() + await rm(cwd, { recursive: true, force: true }) + } + }, + 150_000 + ) it('turns a real silent unauthenticated startup into sign-in guidance', async () => { const claudeConfigDir = await mkdtemp(join(tmpdir(), 'orca-claude-no-auth-')) diff --git a/src/main/claude/claude-supervised-stop-real-cli.test.ts b/src/main/claude/claude-supervised-stop-real-cli.test.ts index 089b9ddbc5d9..aa25217b89bd 100644 --- a/src/main/claude/claude-supervised-stop-real-cli.test.ts +++ b/src/main/claude/claude-supervised-stop-real-cli.test.ts @@ -2,13 +2,10 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { homedir, tmpdir } from 'node:os' import { join } from 'node:path' import { randomUUID } from 'node:crypto' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it } from 'vitest' import { runProcess } from '../../shared/child-process/run-process' import { resolveSessionFilePath } from '../native-chat/session-file-resolver' -import { - REAL_AGENT_TESTS_SKIP_NOTE, - realAgentTestsEnabled -} from '../real-agent-tests-opt-in-test-support' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' import { SUPERVISED_GRACEFUL_EXIT_MS } from './claude-child-exit-proof-ladder' import { openClaudeStreamJsonConnection, @@ -20,9 +17,11 @@ import { } from './claude-structured-launch-resolution' // Opt-in only: spends a real (haiku) turn per case. Run it in an isolated HOME with -// ORCA_RUN_REAL_AGENT_TESTS=1 and ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS when auth lives in a settings file. +// ORCA_RUN_REAL_AGENT_TESTS=1 and ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS +// when auth lives in a settings file. const CLAUDE_BIN = process.env.ORCA_REAL_CLAUDE_BIN ?? '' const enabled = + realAgentTestsEnabled() && process.env.ORCA_REAL_CLAUDE_SUPERVISED_STOP === '1' && process.platform !== 'win32' && CLAUDE_BIN.length > 0 @@ -187,10 +186,6 @@ afterEach(async () => { }) describe.runIf(enabled)('real Claude stopped through the POSIX supervisor', () => { - beforeEach((ctx) => { - ctx.skip(!realAgentTestsEnabled(), REAL_AGENT_TESTS_SKIP_NOTE) - }) - it.each(['close mid-tool', 'SIGTERM to the supervisor mid-tool', 'close while idle'])( '%s: stops Claude and its tool, keeps the transcript line-atomic, and resumes', async (stopCase) => { diff --git a/src/main/codex/codex-index-heal-binary-contract.test.ts b/src/main/codex/codex-index-heal-binary-contract.test.ts index 69888b300a52..ca06ee7dc048 100644 --- a/src/main/codex/codex-index-heal-binary-contract.test.ts +++ b/src/main/codex/codex-index-heal-binary-contract.test.ts @@ -4,13 +4,9 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { promisify } from 'node:util' import { zstdCompressSync } from 'node:zlib' -import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeAll, describe, expect, it } from 'vitest' import { CODEX_SHORT_LIVED_PROBE_APP_SERVER_ARGS } from '../codex-cli/codex-read-only-app-server-args' -import { - REAL_AGENT_TESTS_ENV, - REAL_AGENT_TESTS_SKIP_NOTE, - realAgentTestsEnabled -} from '../real-agent-tests-opt-in-test-support' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' import SyncDatabase from '../sqlite/sync-database' import { runCodexAppServerSession, type CodexAppServerRpc } from './codex-app-server-session' import { findNewestCodexStateDbPath, readCodexStateDbBackfillStatus } from './codex-state-db' @@ -31,25 +27,22 @@ import { findNewestCodexStateDbPath, readCodexStateDbBackfillStatus } from './co const execFileAsync = promisify(execFile) const binary = process.env.ORCA_CODEX_CONTRACT_BINARY const expectedVersion = process.env.ORCA_CODEX_CONTRACT_VERSION -const describeCodexContract = binary ? describe : describe.skip +const contractRunnable = Boolean(binary) && realAgentTestsEnabled() +const describeCodexContract = contractRunnable ? describe : describe.skip // Why this guard: skipping is the right local-dev default, but a CI job whose whole // purpose is the real binary must not pass by reporting zero assertions. The job sets // ORCA_CODEX_CONTRACT_REQUIRED=1, which turns a missing binary or opt-in into a red test. -describe.runIf( - process.env.ORCA_CODEX_CONTRACT_REQUIRED === '1' && (!binary || !realAgentTestsEnabled()) -)('codex binary index-heal contract prerequisites', () => { - it.runIf(!binary)('was given a Codex binary to run against', () => { - expect.fail( - 'ORCA_CODEX_CONTRACT_REQUIRED=1 but ORCA_CODEX_CONTRACT_BINARY is unset, so the contract would have silently skipped' - ) - }) - it.runIf(!realAgentTestsEnabled())('was allowed to start a real agent CLI', () => { - expect.fail( - `ORCA_CODEX_CONTRACT_REQUIRED=1 but ${REAL_AGENT_TESTS_ENV} is not 1, so the contract would have silently skipped` - ) - }) -}) +describe.runIf(process.env.ORCA_CODEX_CONTRACT_REQUIRED === '1' && !contractRunnable)( + 'codex binary index-heal contract prerequisites', + () => { + it('was given a Codex binary to run against', () => { + expect.fail( + 'ORCA_CODEX_CONTRACT_REQUIRED=1 needs ORCA_CODEX_CONTRACT_BINARY and ORCA_RUN_REAL_AGENT_TESTS=1, or the contract silently skips' + ) + }) + } +) // Why fixed: `thread/read` never reaches the network, and a whole session is // spawn + initialize + one RPC. A generous ceiling still fails fast on a wedged child. @@ -67,9 +60,6 @@ describeCodexContract( const disposableHomes: string[] = [] beforeAll(async () => { - if (!realAgentTestsEnabled()) { - return - } // Why assert the version: the whole point of a real-binary check is that the // binary drifts. A job that quietly ran some other Codex would report a // contract this repo never verified. @@ -79,10 +69,6 @@ describeCodexContract( expect(stdout.trim()).toBe(`codex-cli ${expectedVersion}`) }) - beforeEach((ctx) => { - ctx.skip(!realAgentTestsEnabled(), REAL_AGENT_TESTS_SKIP_NOTE) - }) - afterEach(() => { while (disposableHomes.length > 0) { rmSync(disposableHomes.pop() as string, { recursive: true, force: true }) diff --git a/src/main/hermes/hook-service.test.ts b/src/main/hermes/hook-service.test.ts index a7df55e7b219..945a54d090cf 100644 --- a/src/main/hermes/hook-service.test.ts +++ b/src/main/hermes/hook-service.test.ts @@ -7,10 +7,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { parse } from 'yaml' import { makePaneKey } from '../../shared/stable-pane-id' -import { - REAL_AGENT_TESTS_SKIP_NOTE, - realAgentTestsEnabled -} from '../real-agent-tests-opt-in-test-support' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' import { HermesHookService, _internals } from './hook-service' const PANE_KEY = makePaneKey('tab-1', '11111111-1111-4111-8111-111111111111') @@ -110,9 +107,10 @@ describe('HermesHookService', () => { expect(status.detail).toContain('not enabled') }) - it('is visible to the real hermes CLI when hermes is installed', (ctx) => { - ctx.skip(!realAgentTestsEnabled(), REAL_AGENT_TESTS_SKIP_NOTE) - const hermesAvailable = spawnSync('hermes', ['--version'], { encoding: 'utf-8' }).status === 0 + it('is visible to the real hermes CLI when hermes is installed', () => { + const hermesAvailable = + realAgentTestsEnabled() && + spawnSync('hermes', ['--version'], { encoding: 'utf-8' }).status === 0 if (!hermesAvailable) { return } diff --git a/src/main/real-agent-tests-opt-in-test-support.ts b/src/main/real-agent-tests-opt-in-test-support.ts index bf1c15ca467a..37be86cf779e 100644 --- a/src/main/real-agent-tests-opt-in-test-support.ts +++ b/src/main/real-agent-tests-opt-in-test-support.ts @@ -3,8 +3,6 @@ export const REAL_AGENT_TESTS_ENV = 'ORCA_RUN_REAL_AGENT_TESTS' -export const REAL_AGENT_TESTS_SKIP_NOTE = `starts a real agent CLI; set ${REAL_AGENT_TESTS_ENV}=1 to run it` - export function realAgentTestsEnabled(env: NodeJS.ProcessEnv = process.env): boolean { return env[REAL_AGENT_TESTS_ENV] === '1' } diff --git a/src/main/runtime/orchestration-codex-real-pty.integration.test.ts b/src/main/runtime/orchestration-codex-real-pty.integration.test.ts index 3b9c69a6e726..d72481a2f70f 100644 --- a/src/main/runtime/orchestration-codex-real-pty.integration.test.ts +++ b/src/main/runtime/orchestration-codex-real-pty.integration.test.ts @@ -3,17 +3,14 @@ import { mkdtempSync, mkdirSync, realpathSync, rmSync, writeFileSync } from 'nod import { tmpdir } from 'node:os' import { join } from 'node:path' import * as pty from 'node-pty' -import { beforeEach, expect, it, vi } from 'vitest' +import { expect, it, vi } from 'vitest' import { AgentHookServer } from '../agent-hooks/server' import { getManagedScript } from '../codex/codex-hook-script' import { getSyntheticAgentTerminalTitle } from '../../shared/synthetic-agent-title' import { extractAllOscTitles } from '../../shared/osc-title-extraction' import { extractOscTitleScanTail } from '../../shared/osc-title-scan-tail' import { settledWriteStub } from '../providers/settled-pty-write-stub' -import { - REAL_AGENT_TESTS_SKIP_NOTE, - realAgentTestsEnabled -} from '../real-agent-tests-opt-in-test-support' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' import { createBoundRun, createDatabase, @@ -40,11 +37,7 @@ const trials = (['before', 'after'] as const).flatMap((arrival) => ) const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) -beforeEach((ctx) => { - ctx.skip(!realAgentTestsEnabled(), REAL_AGENT_TESTS_SKIP_NOTE) -}) - -it.skipIf(!binary || process.platform === 'win32').each(trials)( +it.skipIf(!binary || !realAgentTestsEnabled() || process.platform === 'win32').each(trials)( 'submits mail arriving $arrival a real Codex completion (trial $trial)', async ({ arrival }) => { const directory = realpathSync(mkdtempSync(join(tmpdir(), 'orca-codex-mailbox-'))) From d2a924c30df7eddfe210b030c7cd5cb62ab13755 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:45:07 -0700 Subject: [PATCH 4/4] test: gate the real OMP close probes behind the real-agent opt-in The two tests/tools OMP probes start an installed OMP binary and are collected by the unit Vitest config, but skipped only on ORCA_OMP_PROBE_BINARY. Require ORCA_RUN_REAL_AGENT_TESTS=1 as well, like the other binary-gated real-agent tests, and add it to their documented run commands. --- tests/tools/omp-close-lifecycle.md | 5 +++-- tests/tools/omp-close-lifecycle.test.mjs | 3 ++- tests/tools/omp-relay-close-lifecycle.md | 2 +- tests/tools/omp-relay-close-lifecycle.test.mjs | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/tools/omp-close-lifecycle.md b/tests/tools/omp-close-lifecycle.md index 03e2dd6fc31d..a7f4e79e4a61 100644 --- a/tests/tools/omp-close-lifecycle.md +++ b/tests/tools/omp-close-lifecycle.md @@ -9,7 +9,7 @@ to represent OMP typed into a shell. This isolates termination policy; it does n exercise Agent button delivery or terminal-tab/handle routing. ```sh -ORCA_BACKGROUND_LAUNCH=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ +ORCA_BACKGROUND_LAUNCH=1 ORCA_RUN_REAL_AGENT_TESTS=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts \ tests/tools/omp-close-lifecycle.test.mjs ``` @@ -17,7 +17,8 @@ ORCA_BACKGROUND_LAUNCH=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ The probe defaults to zsh on macOS and bash on other POSIX hosts. Set `ORCA_OMP_PROBE_SHELL` to the absolute path of either shell to override. Windows is skipped. It requires the existing node-pty native dependency for the current -Node runtime. The normal unit suite skips the test unless a binary is supplied. +Node runtime. The normal unit suite skips the test unless a binary is supplied +and `ORCA_RUN_REAL_AGENT_TESTS=1` is set. Each case waits five seconds for OMP startup, captures the owned process tree, requests explicit close or local quit cleanup, and verifies those exact process diff --git a/tests/tools/omp-close-lifecycle.test.mjs b/tests/tools/omp-close-lifecycle.test.mjs index 47df8325c563..1a3162ee5e93 100644 --- a/tests/tools/omp-close-lifecycle.test.mjs +++ b/tests/tools/omp-close-lifecycle.test.mjs @@ -7,6 +7,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs' import { join } from 'node:path' import { tmpdir } from 'node:os' import { runProcess } from '../../src/shared/child-process/run-process.ts' +import { realAgentTestsEnabled } from '../../src/main/real-agent-tests-opt-in-test-support.ts' import { captureDescendantSnapshot, readProcessTable @@ -44,7 +45,7 @@ const ownedPidRows = async (pids) => { } return result.stdout.trim() } -it.skipIf(!binary || process.platform === 'win32')( +it.skipIf(!binary || !realAgentTestsEnabled() || process.platform === 'win32')( 'observes actual OMP under production owned-PTY closure policy', async () => { const fixtures = join(process.cwd(), '.bench-fixtures') diff --git a/tests/tools/omp-relay-close-lifecycle.md b/tests/tools/omp-relay-close-lifecycle.md index d0ef5052ba39..bb359e09d414 100644 --- a/tests/tools/omp-relay-close-lifecycle.md +++ b/tests/tools/omp-relay-close-lifecycle.md @@ -5,7 +5,7 @@ This opt-in probe uses a real installed OMP binary and native PTYs behind produc transport; no SSH connection or rendered client is exercised. OMP source is read-only. ```sh -ORCA_BACKGROUND_LAUNCH=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ +ORCA_BACKGROUND_LAUNCH=1 ORCA_RUN_REAL_AGENT_TESTS=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ ORCA_OMP_PROBE_SHELL=/bin/bash \ node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts \ tests/tools/omp-relay-close-lifecycle.test.mjs diff --git a/tests/tools/omp-relay-close-lifecycle.test.mjs b/tests/tools/omp-relay-close-lifecycle.test.mjs index 2730356ebd1e..86af651b09ea 100644 --- a/tests/tools/omp-relay-close-lifecycle.test.mjs +++ b/tests/tools/omp-relay-close-lifecycle.test.mjs @@ -11,12 +11,13 @@ import { readProcessTable } from '../../src/main/pty-descendant-termination.ts' import { runProcess } from '../../src/shared/child-process/run-process.ts' +import { realAgentTestsEnabled } from '../../src/main/real-agent-tests-opt-in-test-support.ts' const binary = process.env.ORCA_OMP_PROBE_BINARY const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) const quote = (value) => `'${value.replaceAll("'", "'\\''")}'` -it.skipIf(!binary || process.platform === 'win32')( +it.skipIf(!binary || !realAgentTestsEnabled() || process.platform === 'win32')( 'closes actual OMP detached tools through the relay host', async () => { const fixtures = join(process.cwd(), '.bench-fixtures')