Stop tests from starting a real agent CLI unless asked

.github/workflows/pr.yml

@@ -460,6 +460,8 @@ jobs:
           # reports success. This turns a failed or missing install into a red test
           # instead of a green no-op.
           ORCA_CODEX_CONTRACT_REQUIRED: '1'
+          # Why: real-agent tests skip unless opted in; this job exists to run one.
+          ORCA_RUN_REAL_AGENT_TESTS: '1'
           ORCA_CODEX_CONTRACT_VERSION: ${{ env.CODEX_CLI_VERSION }}
         run: |
           set -euo pipefail

src/main/claude/claude-real-cli-availability-test-support.test.ts

@@ -0,0 +1,80 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+  REAL_AGENT_TESTS_ENV,
+  realAgentTestsEnabled
+} from '../real-agent-tests-opt-in-test-support'
+
+const { spawnMock, spawnSyncMock } = vi.hoisted(() => ({
+  spawnMock: vi.fn(),
+  spawnSyncMock: vi.fn<(program: string, args: readonly string[]) => unknown>(() => ({
+    status: 1,
+    signal: null,
+    stdout: '',
+    stderr: '',
+    pid: 0
+  }))
+}))
+
+// Why the lowest layer: nothing below this helper may start a process, whichever wrapper it uses.
+vi.mock('node:child_process', async (importOriginal) => ({
+  ...(await importOriginal<Record<string, unknown>>()),
+  spawn: spawnMock,
+  spawnSync: spawnSyncMock
+}))
+
+// Why a missing path: even a mock that failed to apply could not reach a real CLI.
+vi.mock('../codex-cli/command', () => ({
+  resolveClaudeCommand: () => '/nonexistent/orca-test-claude'
+}))
+
+async function loadAvailability() {
+  vi.resetModules()
+  return import('./claude-real-cli-availability-test-support')
+}
+
+describe('real Claude CLI availability', () => {
+  beforeEach(() => {
+    spawnMock.mockClear()
+    spawnSyncMock.mockClear()
+  })
+
+  afterEach(() => {
+    vi.unstubAllEnvs()
+  })
+
+  it('skips without spawning anything unless real-agent tests are opted into', async () => {
+    vi.stubEnv(REAL_AGENT_TESTS_ENV, undefined)
+
+    const { realClaudeAvailable, realClaudeAuthenticated, realClaudeAuthStatus } =
+      await loadAvailability()
+
+    expect(realClaudeAvailable).toBe(false)
+    expect(realClaudeAuthenticated).toBe(false)
+    expect(realClaudeAuthStatus).toBeNull()
+    expect(spawnSyncMock).not.toHaveBeenCalled()
+    expect(spawnMock).not.toHaveBeenCalled()
+  })
+
+  it('probes the CLI once opted into', async () => {
+    vi.stubEnv(REAL_AGENT_TESTS_ENV, '1')
+
+    const { realClaudeAvailable } = await loadAvailability()
+
+    // The mocked probe fails, so this proves the gate, not a real CLI.
+    expect(realClaudeAvailable).toBe(false)
+    expect(spawnSyncMock).toHaveBeenCalledTimes(1)
+    expect(spawnSyncMock.mock.calls[0]?.[1]).toContain('--version')
+  })
+})
+
+describe('realAgentTestsEnabled', () => {
+  it.each([
+    [undefined, false],
+    ['', false],
+    ['0', false],
+    ['true', false],
+    ['1', true]
+  ])('%j -> %s', (value, expected) => {
+    expect(realAgentTestsEnabled({ [REAL_AGENT_TESTS_ENV]: value })).toBe(expected)
+  })
+})

src/main/claude/claude-real-cli-availability-test-support.ts

@@ -1,12 +1,13 @@
 // Whether a real, signed-in Claude CLI is present — the gate every real-CLI
-// suite skips on. Probed once per test process.
+// suite skips on. Probed once per test process, and only with ORCA_RUN_REAL_AGENT_TESTS=1.
 
 import { homedir } from 'node:os'
 import { join } from 'node:path'
 import { runProcessSync, type ProcessResult } from '../../shared/child-process/run-process'
 import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution'
 import { CLAUDE_AUTH_ENV_VARS } from '../claude-accounts/environment'
 import { resolveClaudeCommand } from '../codex-cli/command'
+import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support'
 
 export const realClaudeCommand = resolveClaudeCommand()
 
@@ -25,7 +26,8 @@ function probeRealClaude(args: string[]): ProcessResult | null {
   }
 }
 
-export const realClaudeAvailable = probeRealClaude(['--version'])?.code === 0
+export const realClaudeAvailable =
+  realAgentTestsEnabled() && probeRealClaude(['--version'])?.code === 0
 
 /** The CLI's own account report — the only source of truth for where it writes that
  *  is not derived from Orca's own path expressions. */

src/main/claude/claude-supervised-stop-real-cli.test.ts

@@ -5,6 +5,7 @@ import { randomUUID } from 'node:crypto'
 import { afterEach, describe, expect, it } from 'vitest'
 import { runProcess } from '../../shared/child-process/run-process'
 import { resolveSessionFilePath } from '../native-chat/session-file-resolver'
+import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support'
 import { SUPERVISED_GRACEFUL_EXIT_MS } from './claude-child-exit-proof-ladder'
 import {
   openClaudeStreamJsonConnection,
@@ -16,9 +17,11 @@ import {
 } from './claude-structured-launch-resolution'
 
 // Opt-in only: spends a real (haiku) turn per case. Run it in an isolated HOME with
-// ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS when auth lives in a settings file.
+// ORCA_RUN_REAL_AGENT_TESTS=1 and ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS
+// when auth lives in a settings file.
 const CLAUDE_BIN = process.env.ORCA_REAL_CLAUDE_BIN ?? ''
 const enabled =
+  realAgentTestsEnabled() &&
   process.env.ORCA_REAL_CLAUDE_SUPERVISED_STOP === '1' &&
   process.platform !== 'win32' &&
   CLAUDE_BIN.length > 0

src/main/codex/codex-index-heal-binary-contract.test.ts

@@ -6,6 +6,7 @@ import { promisify } from 'node:util'
 import { zstdCompressSync } from 'node:zlib'
 import { afterEach, beforeAll, describe, expect, it } from 'vitest'
 import { CODEX_SHORT_LIVED_PROBE_APP_SERVER_ARGS } from '../codex-cli/codex-read-only-app-server-args'
+import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support'
 import SyncDatabase from '../sqlite/sync-database'
 import { runCodexAppServerSession, type CodexAppServerRpc } from './codex-app-server-session'
 import { findNewestCodexStateDbPath, readCodexStateDbBackfillStatus } from './codex-state-db'
@@ -26,17 +27,18 @@ import { findNewestCodexStateDbPath, readCodexStateDbBackfillStatus } from './co
 const execFileAsync = promisify(execFile)
 const binary = process.env.ORCA_CODEX_CONTRACT_BINARY
 const expectedVersion = process.env.ORCA_CODEX_CONTRACT_VERSION
-const describeCodexContract = binary ? describe : describe.skip
+const contractRunnable = Boolean(binary) && realAgentTestsEnabled()
+const describeCodexContract = contractRunnable ? describe : describe.skip
 
 // Why this guard: skipping is the right local-dev default, but a CI job whose whole
 // purpose is the real binary must not pass by reporting zero assertions. The job sets
-// ORCA_CODEX_CONTRACT_REQUIRED=1, which turns a missing binary into a red test.
-describe.runIf(process.env.ORCA_CODEX_CONTRACT_REQUIRED === '1' && !binary)(
+// ORCA_CODEX_CONTRACT_REQUIRED=1, which turns a missing binary or opt-in into a red test.
+describe.runIf(process.env.ORCA_CODEX_CONTRACT_REQUIRED === '1' && !contractRunnable)(
   'codex binary index-heal contract prerequisites',
   () => {
     it('was given a Codex binary to run against', () => {
       expect.fail(
-        'ORCA_CODEX_CONTRACT_REQUIRED=1 but ORCA_CODEX_CONTRACT_BINARY is unset, so the contract would have silently skipped'
+        'ORCA_CODEX_CONTRACT_REQUIRED=1 needs ORCA_CODEX_CONTRACT_BINARY and ORCA_RUN_REAL_AGENT_TESTS=1, or the contract silently skips'
       )
     })
   }

src/main/hermes/hook-service.test.ts

@@ -7,6 +7,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 import { parse } from 'yaml'
 
 import { makePaneKey } from '../../shared/stable-pane-id'
+import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support'
 import { HermesHookService, _internals } from './hook-service'
 
 const PANE_KEY = makePaneKey('tab-1', '11111111-1111-4111-8111-111111111111')
@@ -107,7 +108,9 @@ describe('HermesHookService', () => {
   })
 
   it('is visible to the real hermes CLI when hermes is installed', () => {
-    const hermesAvailable = spawnSync('hermes', ['--version'], { encoding: 'utf-8' }).status === 0
+    const hermesAvailable =
+      realAgentTestsEnabled() &&
+      spawnSync('hermes', ['--version'], { encoding: 'utf-8' }).status === 0
     if (!hermesAvailable) {
       return
     }

src/main/real-agent-tests-opt-in-test-support.ts

@@ -0,0 +1,8 @@
+// The one switch every test that starts a real agent CLI checks first. Those tests run the
+// CLI against the developer's own config and account, so a broad local run must never start one.
+
+export const REAL_AGENT_TESTS_ENV = 'ORCA_RUN_REAL_AGENT_TESTS'
+
+export function realAgentTestsEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
+  return env[REAL_AGENT_TESTS_ENV] === '1'
+}

src/main/runtime/orchestration-codex-real-pty.integration.test.ts

@@ -10,6 +10,7 @@ import { getSyntheticAgentTerminalTitle } from '../../shared/synthetic-agent-tit
 import { extractAllOscTitles } from '../../shared/osc-title-extraction'
 import { extractOscTitleScanTail } from '../../shared/osc-title-scan-tail'
 import { settledWriteStub } from '../providers/settled-pty-write-stub'
+import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support'
 import {
   createBoundRun,
   createDatabase,
@@ -36,7 +37,7 @@ const trials = (['before', 'after'] as const).flatMap((arrival) =>
 )
 const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms))
 
-it.skipIf(!binary || process.platform === 'win32').each(trials)(
+it.skipIf(!binary || !realAgentTestsEnabled() || process.platform === 'win32').each(trials)(
   'submits mail arriving $arrival a real Codex completion (trial $trial)',
   async ({ arrival }) => {
     const directory = realpathSync(mkdtempSync(join(tmpdir(), 'orca-codex-mailbox-')))

tests/tools/omp-close-lifecycle.md

@@ -9,15 +9,16 @@ to represent OMP typed into a shell. This isolates termination policy; it does n
 exercise Agent button delivery or terminal-tab/handle routing.
 
 ```sh
-ORCA_BACKGROUND_LAUNCH=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \
+ORCA_BACKGROUND_LAUNCH=1 ORCA_RUN_REAL_AGENT_TESTS=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \
   node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts \
   tests/tools/omp-close-lifecycle.test.mjs
 ```
 
 The probe defaults to zsh on macOS and bash on other POSIX hosts. Set
 `ORCA_OMP_PROBE_SHELL` to the absolute path of either shell to override. Windows
 is skipped. It requires the existing node-pty native dependency for the current
-Node runtime. The normal unit suite skips the test unless a binary is supplied.
+Node runtime. The normal unit suite skips the test unless a binary is supplied
+and `ORCA_RUN_REAL_AGENT_TESTS=1` is set.
 
 Each case waits five seconds for OMP startup, captures the owned process tree,
 requests explicit close or local quit cleanup, and verifies those exact process

tests/tools/omp-close-lifecycle.test.mjs

@@ -7,6 +7,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'
 import { join } from 'node:path'
 import { tmpdir } from 'node:os'
 import { runProcess } from '../../src/shared/child-process/run-process.ts'
+import { realAgentTestsEnabled } from '../../src/main/real-agent-tests-opt-in-test-support.ts'
 import {
   captureDescendantSnapshot,
   readProcessTable
@@ -44,7 +45,7 @@ const ownedPidRows = async (pids) => {
   }
   return result.stdout.trim()
 }
-it.skipIf(!binary || process.platform === 'win32')(
+it.skipIf(!binary || !realAgentTestsEnabled() || process.platform === 'win32')(
   'observes actual OMP under production owned-PTY closure policy',
   async () => {
     const fixtures = join(process.cwd(), '.bench-fixtures')

tests/tools/omp-relay-close-lifecycle.md

@@ -5,7 +5,7 @@ This opt-in probe uses a real installed OMP binary and native PTYs behind produc
 transport; no SSH connection or rendered client is exercised. OMP source is read-only.
 
 ```sh
-ORCA_BACKGROUND_LAUNCH=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \
+ORCA_BACKGROUND_LAUNCH=1 ORCA_RUN_REAL_AGENT_TESTS=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \
   ORCA_OMP_PROBE_SHELL=/bin/bash \
   node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts \
   tests/tools/omp-relay-close-lifecycle.test.mjs

tests/tools/omp-relay-close-lifecycle.test.mjs

@@ -11,12 +11,13 @@ import {
   readProcessTable
 } from '../../src/main/pty-descendant-termination.ts'
 import { runProcess } from '../../src/shared/child-process/run-process.ts'
+import { realAgentTestsEnabled } from '../../src/main/real-agent-tests-opt-in-test-support.ts'
 
 const binary = process.env.ORCA_OMP_PROBE_BINARY
 const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
 const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`
 
-it.skipIf(!binary || process.platform === 'win32')(
+it.skipIf(!binary || !realAgentTestsEnabled() || process.platform === 'win32')(
   'closes actual OMP detached tools through the relay host',
   async () => {
     const fixtures = join(process.cwd(), '.bench-fixtures')