diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 795beca131e1..f2bf98881149 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -460,6 +460,8 @@ jobs: # reports success. This turns a failed or missing install into a red test # instead of a green no-op. ORCA_CODEX_CONTRACT_REQUIRED: '1' + # Why: real-agent tests skip unless opted in; this job exists to run one. + ORCA_RUN_REAL_AGENT_TESTS: '1' ORCA_CODEX_CONTRACT_VERSION: ${{ env.CODEX_CLI_VERSION }} run: | set -euo pipefail diff --git a/src/main/claude/claude-real-cli-availability-test-support.test.ts b/src/main/claude/claude-real-cli-availability-test-support.test.ts new file mode 100644 index 000000000000..afb1752d94a9 --- /dev/null +++ b/src/main/claude/claude-real-cli-availability-test-support.test.ts @@ -0,0 +1,80 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + REAL_AGENT_TESTS_ENV, + realAgentTestsEnabled +} from '../real-agent-tests-opt-in-test-support' + +const { spawnMock, spawnSyncMock } = vi.hoisted(() => ({ + spawnMock: vi.fn(), + spawnSyncMock: vi.fn<(program: string, args: readonly string[]) => unknown>(() => ({ + status: 1, + signal: null, + stdout: '', + stderr: '', + pid: 0 + })) +})) + +// Why the lowest layer: nothing below this helper may start a process, whichever wrapper it uses. +vi.mock('node:child_process', async (importOriginal) => ({ + ...(await importOriginal>()), + spawn: spawnMock, + spawnSync: spawnSyncMock +})) + +// Why a missing path: even a mock that failed to apply could not reach a real CLI. +vi.mock('../codex-cli/command', () => ({ + resolveClaudeCommand: () => '/nonexistent/orca-test-claude' +})) + +async function loadAvailability() { + vi.resetModules() + return import('./claude-real-cli-availability-test-support') +} + +describe('real Claude CLI availability', () => { + beforeEach(() => { + spawnMock.mockClear() + spawnSyncMock.mockClear() + }) + + afterEach(() => { + vi.unstubAllEnvs() + }) + + it('skips without spawning anything unless real-agent tests are opted into', async () => { + vi.stubEnv(REAL_AGENT_TESTS_ENV, undefined) + + const { realClaudeAvailable, realClaudeAuthenticated, realClaudeAuthStatus } = + await loadAvailability() + + expect(realClaudeAvailable).toBe(false) + expect(realClaudeAuthenticated).toBe(false) + expect(realClaudeAuthStatus).toBeNull() + expect(spawnSyncMock).not.toHaveBeenCalled() + expect(spawnMock).not.toHaveBeenCalled() + }) + + it('probes the CLI once opted into', async () => { + vi.stubEnv(REAL_AGENT_TESTS_ENV, '1') + + const { realClaudeAvailable } = await loadAvailability() + + // The mocked probe fails, so this proves the gate, not a real CLI. + expect(realClaudeAvailable).toBe(false) + expect(spawnSyncMock).toHaveBeenCalledTimes(1) + expect(spawnSyncMock.mock.calls[0]?.[1]).toContain('--version') + }) +}) + +describe('realAgentTestsEnabled', () => { + it.each([ + [undefined, false], + ['', false], + ['0', false], + ['true', false], + ['1', true] + ])('%j -> %s', (value, expected) => { + expect(realAgentTestsEnabled({ [REAL_AGENT_TESTS_ENV]: value })).toBe(expected) + }) +}) diff --git a/src/main/claude/claude-real-cli-availability-test-support.ts b/src/main/claude/claude-real-cli-availability-test-support.ts index faa322c456b1..bf61a1ba0abe 100644 --- a/src/main/claude/claude-real-cli-availability-test-support.ts +++ b/src/main/claude/claude-real-cli-availability-test-support.ts @@ -1,5 +1,5 @@ // Whether a real, signed-in Claude CLI is present — the gate every real-CLI -// suite skips on. Probed once per test process. +// suite skips on. Probed once per test process, and only with ORCA_RUN_REAL_AGENT_TESTS=1. import { homedir } from 'node:os' import { join } from 'node:path' @@ -7,6 +7,7 @@ import { runProcessSync, type ProcessResult } from '../../shared/child-process/r import { withCliRuntimeOnPath } from '../../shared/node-cli-command-resolution' import { CLAUDE_AUTH_ENV_VARS } from '../claude-accounts/environment' import { resolveClaudeCommand } from '../codex-cli/command' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' export const realClaudeCommand = resolveClaudeCommand() @@ -25,7 +26,8 @@ function probeRealClaude(args: string[]): ProcessResult | null { } } -export const realClaudeAvailable = probeRealClaude(['--version'])?.code === 0 +export const realClaudeAvailable = + realAgentTestsEnabled() && probeRealClaude(['--version'])?.code === 0 /** The CLI's own account report — the only source of truth for where it writes that * is not derived from Orca's own path expressions. */ diff --git a/src/main/claude/claude-supervised-stop-real-cli.test.ts b/src/main/claude/claude-supervised-stop-real-cli.test.ts index 82432e3fd6e6..aa25217b89bd 100644 --- a/src/main/claude/claude-supervised-stop-real-cli.test.ts +++ b/src/main/claude/claude-supervised-stop-real-cli.test.ts @@ -5,6 +5,7 @@ import { randomUUID } from 'node:crypto' import { afterEach, describe, expect, it } from 'vitest' import { runProcess } from '../../shared/child-process/run-process' import { resolveSessionFilePath } from '../native-chat/session-file-resolver' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' import { SUPERVISED_GRACEFUL_EXIT_MS } from './claude-child-exit-proof-ladder' import { openClaudeStreamJsonConnection, @@ -16,9 +17,11 @@ import { } from './claude-structured-launch-resolution' // Opt-in only: spends a real (haiku) turn per case. Run it in an isolated HOME with -// ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS when auth lives in a settings file. +// ORCA_RUN_REAL_AGENT_TESTS=1 and ORCA_REAL_CLAUDE_BIN set, and ORCA_REAL_CLAUDE_SETTINGS +// when auth lives in a settings file. const CLAUDE_BIN = process.env.ORCA_REAL_CLAUDE_BIN ?? '' const enabled = + realAgentTestsEnabled() && process.env.ORCA_REAL_CLAUDE_SUPERVISED_STOP === '1' && process.platform !== 'win32' && CLAUDE_BIN.length > 0 diff --git a/src/main/codex/codex-index-heal-binary-contract.test.ts b/src/main/codex/codex-index-heal-binary-contract.test.ts index 8954f618a6ad..ca06ee7dc048 100644 --- a/src/main/codex/codex-index-heal-binary-contract.test.ts +++ b/src/main/codex/codex-index-heal-binary-contract.test.ts @@ -6,6 +6,7 @@ import { promisify } from 'node:util' import { zstdCompressSync } from 'node:zlib' import { afterEach, beforeAll, describe, expect, it } from 'vitest' import { CODEX_SHORT_LIVED_PROBE_APP_SERVER_ARGS } from '../codex-cli/codex-read-only-app-server-args' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' import SyncDatabase from '../sqlite/sync-database' import { runCodexAppServerSession, type CodexAppServerRpc } from './codex-app-server-session' import { findNewestCodexStateDbPath, readCodexStateDbBackfillStatus } from './codex-state-db' @@ -26,17 +27,18 @@ import { findNewestCodexStateDbPath, readCodexStateDbBackfillStatus } from './co const execFileAsync = promisify(execFile) const binary = process.env.ORCA_CODEX_CONTRACT_BINARY const expectedVersion = process.env.ORCA_CODEX_CONTRACT_VERSION -const describeCodexContract = binary ? describe : describe.skip +const contractRunnable = Boolean(binary) && realAgentTestsEnabled() +const describeCodexContract = contractRunnable ? describe : describe.skip // Why this guard: skipping is the right local-dev default, but a CI job whose whole // purpose is the real binary must not pass by reporting zero assertions. The job sets -// ORCA_CODEX_CONTRACT_REQUIRED=1, which turns a missing binary into a red test. -describe.runIf(process.env.ORCA_CODEX_CONTRACT_REQUIRED === '1' && !binary)( +// ORCA_CODEX_CONTRACT_REQUIRED=1, which turns a missing binary or opt-in into a red test. +describe.runIf(process.env.ORCA_CODEX_CONTRACT_REQUIRED === '1' && !contractRunnable)( 'codex binary index-heal contract prerequisites', () => { it('was given a Codex binary to run against', () => { expect.fail( - 'ORCA_CODEX_CONTRACT_REQUIRED=1 but ORCA_CODEX_CONTRACT_BINARY is unset, so the contract would have silently skipped' + 'ORCA_CODEX_CONTRACT_REQUIRED=1 needs ORCA_CODEX_CONTRACT_BINARY and ORCA_RUN_REAL_AGENT_TESTS=1, or the contract silently skips' ) }) } diff --git a/src/main/hermes/hook-service.test.ts b/src/main/hermes/hook-service.test.ts index 3158b551480a..945a54d090cf 100644 --- a/src/main/hermes/hook-service.test.ts +++ b/src/main/hermes/hook-service.test.ts @@ -7,6 +7,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { parse } from 'yaml' import { makePaneKey } from '../../shared/stable-pane-id' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' import { HermesHookService, _internals } from './hook-service' const PANE_KEY = makePaneKey('tab-1', '11111111-1111-4111-8111-111111111111') @@ -107,7 +108,9 @@ describe('HermesHookService', () => { }) it('is visible to the real hermes CLI when hermes is installed', () => { - const hermesAvailable = spawnSync('hermes', ['--version'], { encoding: 'utf-8' }).status === 0 + const hermesAvailable = + realAgentTestsEnabled() && + spawnSync('hermes', ['--version'], { encoding: 'utf-8' }).status === 0 if (!hermesAvailable) { return } diff --git a/src/main/real-agent-tests-opt-in-test-support.ts b/src/main/real-agent-tests-opt-in-test-support.ts new file mode 100644 index 000000000000..37be86cf779e --- /dev/null +++ b/src/main/real-agent-tests-opt-in-test-support.ts @@ -0,0 +1,8 @@ +// The one switch every test that starts a real agent CLI checks first. Those tests run the +// CLI against the developer's own config and account, so a broad local run must never start one. + +export const REAL_AGENT_TESTS_ENV = 'ORCA_RUN_REAL_AGENT_TESTS' + +export function realAgentTestsEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return env[REAL_AGENT_TESTS_ENV] === '1' +} diff --git a/src/main/runtime/orchestration-codex-real-pty.integration.test.ts b/src/main/runtime/orchestration-codex-real-pty.integration.test.ts index 28be6b513a75..d72481a2f70f 100644 --- a/src/main/runtime/orchestration-codex-real-pty.integration.test.ts +++ b/src/main/runtime/orchestration-codex-real-pty.integration.test.ts @@ -10,6 +10,7 @@ import { getSyntheticAgentTerminalTitle } from '../../shared/synthetic-agent-tit import { extractAllOscTitles } from '../../shared/osc-title-extraction' import { extractOscTitleScanTail } from '../../shared/osc-title-scan-tail' import { settledWriteStub } from '../providers/settled-pty-write-stub' +import { realAgentTestsEnabled } from '../real-agent-tests-opt-in-test-support' import { createBoundRun, createDatabase, @@ -36,7 +37,7 @@ const trials = (['before', 'after'] as const).flatMap((arrival) => ) const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) -it.skipIf(!binary || process.platform === 'win32').each(trials)( +it.skipIf(!binary || !realAgentTestsEnabled() || process.platform === 'win32').each(trials)( 'submits mail arriving $arrival a real Codex completion (trial $trial)', async ({ arrival }) => { const directory = realpathSync(mkdtempSync(join(tmpdir(), 'orca-codex-mailbox-'))) diff --git a/tests/tools/omp-close-lifecycle.md b/tests/tools/omp-close-lifecycle.md index 03e2dd6fc31d..a7f4e79e4a61 100644 --- a/tests/tools/omp-close-lifecycle.md +++ b/tests/tools/omp-close-lifecycle.md @@ -9,7 +9,7 @@ to represent OMP typed into a shell. This isolates termination policy; it does n exercise Agent button delivery or terminal-tab/handle routing. ```sh -ORCA_BACKGROUND_LAUNCH=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ +ORCA_BACKGROUND_LAUNCH=1 ORCA_RUN_REAL_AGENT_TESTS=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts \ tests/tools/omp-close-lifecycle.test.mjs ``` @@ -17,7 +17,8 @@ ORCA_BACKGROUND_LAUNCH=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ The probe defaults to zsh on macOS and bash on other POSIX hosts. Set `ORCA_OMP_PROBE_SHELL` to the absolute path of either shell to override. Windows is skipped. It requires the existing node-pty native dependency for the current -Node runtime. The normal unit suite skips the test unless a binary is supplied. +Node runtime. The normal unit suite skips the test unless a binary is supplied +and `ORCA_RUN_REAL_AGENT_TESTS=1` is set. Each case waits five seconds for OMP startup, captures the owned process tree, requests explicit close or local quit cleanup, and verifies those exact process diff --git a/tests/tools/omp-close-lifecycle.test.mjs b/tests/tools/omp-close-lifecycle.test.mjs index 47df8325c563..1a3162ee5e93 100644 --- a/tests/tools/omp-close-lifecycle.test.mjs +++ b/tests/tools/omp-close-lifecycle.test.mjs @@ -7,6 +7,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs' import { join } from 'node:path' import { tmpdir } from 'node:os' import { runProcess } from '../../src/shared/child-process/run-process.ts' +import { realAgentTestsEnabled } from '../../src/main/real-agent-tests-opt-in-test-support.ts' import { captureDescendantSnapshot, readProcessTable @@ -44,7 +45,7 @@ const ownedPidRows = async (pids) => { } return result.stdout.trim() } -it.skipIf(!binary || process.platform === 'win32')( +it.skipIf(!binary || !realAgentTestsEnabled() || process.platform === 'win32')( 'observes actual OMP under production owned-PTY closure policy', async () => { const fixtures = join(process.cwd(), '.bench-fixtures') diff --git a/tests/tools/omp-relay-close-lifecycle.md b/tests/tools/omp-relay-close-lifecycle.md index d0ef5052ba39..bb359e09d414 100644 --- a/tests/tools/omp-relay-close-lifecycle.md +++ b/tests/tools/omp-relay-close-lifecycle.md @@ -5,7 +5,7 @@ This opt-in probe uses a real installed OMP binary and native PTYs behind produc transport; no SSH connection or rendered client is exercised. OMP source is read-only. ```sh -ORCA_BACKGROUND_LAUNCH=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ +ORCA_BACKGROUND_LAUNCH=1 ORCA_RUN_REAL_AGENT_TESTS=1 ORCA_OMP_PROBE_BINARY=/absolute/path/to/omp \ ORCA_OMP_PROBE_SHELL=/bin/bash \ node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts \ tests/tools/omp-relay-close-lifecycle.test.mjs diff --git a/tests/tools/omp-relay-close-lifecycle.test.mjs b/tests/tools/omp-relay-close-lifecycle.test.mjs index 2730356ebd1e..86af651b09ea 100644 --- a/tests/tools/omp-relay-close-lifecycle.test.mjs +++ b/tests/tools/omp-relay-close-lifecycle.test.mjs @@ -11,12 +11,13 @@ import { readProcessTable } from '../../src/main/pty-descendant-termination.ts' import { runProcess } from '../../src/shared/child-process/run-process.ts' +import { realAgentTestsEnabled } from '../../src/main/real-agent-tests-opt-in-test-support.ts' const binary = process.env.ORCA_OMP_PROBE_BINARY const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) const quote = (value) => `'${value.replaceAll("'", "'\\''")}'` -it.skipIf(!binary || process.platform === 'win32')( +it.skipIf(!binary || !realAgentTestsEnabled() || process.platform === 'win32')( 'closes actual OMP detached tools through the relay host', async () => { const fixtures = join(process.cwd(), '.bench-fixtures')