diff --git a/.changeset/ponytail-host-contract-tests.md b/.changeset/ponytail-host-contract-tests.md new file mode 100644 index 000000000..8d1377b10 --- /dev/null +++ b/.changeset/ponytail-host-contract-tests.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": minor +--- + +Remove the test-only host-contract smoke harness from `agent-bundle/api`: `evaluateHostContract`, `compareInstalledHostContract`, `compareLocalHostContract`, `parseHostContractManifest`, `parseRedactedEventEnvelopes`, `nativeHostContractComparisonEnabled`, and their `HostContract*`, `CompareInstalledHostContractOptions`, `NativeHost`, and `RedactedEventEnvelope` types are no longer exported; the native Claude and Codex host proofs (`runNativeClaudeSmoke`, `runCodexNativeSmoke`, …) now live in the test suite. (#655) diff --git a/docs/audits/2026-09-03-claude-live-session-proofs.md b/docs/audits/2026-09-03-claude-live-session-proofs.md index c6aeaa903..c96fe2821 100644 --- a/docs/audits/2026-09-03-claude-live-session-proofs.md +++ b/docs/audits/2026-09-03-claude-live-session-proofs.md @@ -83,8 +83,8 @@ is reproduced here beyond counts, codes, and field names. `normalHome: 'settings-and-plugins-unchanged'`; `packed-native-smoke.test.ts` gained a unit test that a `.claude.json` rewrite passes while a `settings.json` or `plugins/` change fails. -3. **`runNativeClaudeSmoke` (product code, - `packages/agent-bundle/src/host-contracts/native-claude-contract.ts`) has +3. **`runNativeClaudeSmoke` (then product code at + `packages/agent-bundle/src/host-contracts/native-claude-contract.ts`, since moved to `packages/agent-bundle/tests/support/native-claude-smoke.ts`) has the same guard** and therefore cannot pass on Claude Code 2.1.257: its `snapshotClaudeNormalHome` digests `.claude.json` (`claudeJson`) beside `config.json`, `settings.local.json`, `plugins/`, and `settings.json`, and diff --git a/packages/agent-bundle/src/api.ts b/packages/agent-bundle/src/api.ts index ace61c136..39467d815 100644 --- a/packages/agent-bundle/src/api.ts +++ b/packages/agent-bundle/src/api.ts @@ -287,30 +287,6 @@ import { import { runWithPlatform, withTempDirectory } from './effect/platform.ts'; import { liftPromise } from './effect/lift.ts'; -export { - compareInstalledHostContract, - compareLocalHostContract, - evaluateHostContract, - nativeHostContractComparisonEnabled, - parseHostContractManifest, - parseRedactedEventEnvelopes, -} from './host-contracts/host-contract.ts'; -export type { - CompareInstalledHostContractOptions, - HostContractCommand, - HostContractCommandResult, - HostContractCommandRunner, - HostContractDiagnostic, - HostContractEvidence, - HostContractHelpProbe, - HostContractManifest, - HostContractProbe, - HostContractProbeKind, - HostContractReport, - HostContractStatus, - NativeHost, - RedactedEventEnvelope, -} from './host-contracts/host-contract.ts'; export { validateClaudePlugin, validateCodexPlugin, validateCursorPlugin, validatePortablePlugin }; export type { ClaudePluginValidationReport, diff --git a/packages/agent-bundle/src/core/semver.ts b/packages/agent-bundle/src/core/semver.ts index 38f37fa6d..dc7f69b82 100644 --- a/packages/agent-bundle/src/core/semver.ts +++ b/packages/agent-bundle/src/core/semver.ts @@ -5,9 +5,9 @@ export interface SemanticVersion { readonly prerelease: boolean; } -/** Finds the first semver-shaped token inside surrounding CLI banner text. */ +/** Finds the first semver-shaped token inside surrounding CLI banner text; a fourth dotted component disqualifies the token. */ export const parseSemanticVersion = (value: string): SemanticVersion | undefined => { - const match = /(?:^|[^0-9])(\d+)\.(\d+)\.(\d+)(-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?(?:$|[^0-9])/u.exec(value); + const match = /(?:^|[^0-9.])(\d+)\.(\d+)\.(\d+)(-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?(?=$|[^0-9A-Za-z.-])/u.exec(value); if (match === null) return undefined; return Object.freeze({ major: Number(match[1]), diff --git a/packages/agent-bundle/src/host-contracts/native-claude-contract.ts b/packages/agent-bundle/src/host-contracts/native-claude-contract.ts index fed2e0e32..8732f3af9 100644 --- a/packages/agent-bundle/src/host-contracts/native-claude-contract.ts +++ b/packages/agent-bundle/src/host-contracts/native-claude-contract.ts @@ -1,13 +1,6 @@ -import { createHash } from 'node:crypto'; - -import { digest } from '../core/digest.ts'; -import { isErrno } from '../core/errors.ts'; -import { isRecord } from '../core/strict-json.ts'; -import { readFileString, runWithPlatform } from '../effect/platform.ts'; import { spawn } from 'node:child_process'; -import { lstat, readFile, readdir } from 'node:fs/promises'; -import { homedir } from 'node:os'; -import { join } from 'node:path'; + +import { meetsMinimumVersion, parseSemanticVersion, type SemanticVersion } from '../core/semver.ts'; export interface NativeClaudeCommandOptions { readonly model?: string; @@ -54,129 +47,6 @@ export const createNativeClaudeChildEnvironment = ( Object.entries(environment).filter(([name]) => !isProviderApiKey(name) && !removesSubscriptionBypass(name)), ); -export type ClaudeActivationEvidence = 'observed' | 'unavailable'; -export type ClaudeInitAuthSource = 'environment-key' | 'non-environment' | 'unavailable'; - -export interface RedactedClaudeEnvelope { - readonly fields: readonly string[]; - readonly subtype?: string; - readonly type?: string; -} - -export interface NativeClaudeStreamEvidence { - readonly activationEvidence: ClaudeActivationEvidence; - readonly authSource: ClaudeInitAuthSource; - readonly envelopes: readonly RedactedClaudeEnvelope[]; - readonly errorEnvelopes: readonly RedactedClaudeEnvelope[]; - readonly hookEnvelopes: readonly RedactedClaudeEnvelope[]; - readonly mcp: Readonly<{ - readonly configuredServers: number; - readonly toolCalls: number; - }>; - readonly plugins: readonly string[]; -} - -export interface NativeClaudeStreamNormalizationOptions { - readonly allowedPluginNames?: readonly string[]; - readonly candidateSkillEventName?: string; -} - -const isSafeLabel = (value: unknown): value is string => - typeof value === 'string' && /^[A-Za-z][A-Za-z0-9_.-]{0,127}$/u.test(value); - -const parseStreamRecords = (raw: string): readonly Readonly>[] => Object.freeze( - raw - .split(/\r?\n/u) - .filter((line) => line.trim().length > 0) - .map((line) => { - const value = JSON.parse(line) as unknown; - if (!isRecord(value)) throw new TypeError('Claude stream event must be a JSON object.'); - return value; - }), -); - -const redactEnvelope = (value: Readonly>): RedactedClaudeEnvelope => Object.freeze({ - fields: Object.freeze(Object.keys(value).sort()), - ...(isSafeLabel(value.subtype) ? { subtype: value.subtype } : {}), - ...(isSafeLabel(value.type) ? { type: value.type } : {}), -}); - -const namesFromRecords = (value: unknown): readonly string[] => !Array.isArray(value) - ? Object.freeze([]) - : Object.freeze(value.flatMap((entry) => isRecord(entry) && isSafeLabel(entry.name) ? [entry.name] : [])); - -const toolUseNames = (value: Readonly>): readonly string[] => { - const message = value.message; - if (!isRecord(message) || !Array.isArray(message.content)) return Object.freeze([]); - return Object.freeze(message.content.flatMap((content) => - isRecord(content) && content.type === 'tool_use' && isSafeLabel(content.name) ? [content.name] : [])); -}; - -const hasCandidateSkillUse = (value: Readonly>, candidateSkillEventName: string | undefined): boolean => { - if (candidateSkillEventName === undefined) return false; - const message = value.message; - if (!isRecord(message) || !Array.isArray(message.content)) return false; - return message.content.some((content) => - isRecord(content) - && content.type === 'tool_use' - && content.name === 'Skill' - && isRecord(content.input) - && content.input.skill === candidateSkillEventName); -}; - -const normalizeInitAuthSource = (value: unknown): ClaudeInitAuthSource => { - if (typeof value !== 'string') return 'unavailable'; - return /(?:environment|env|api[ _-]?key)/iu.test(value) ? 'environment-key' : 'non-environment'; -}; - -export const normalizeNativeClaudeStream = ( - raw: string, - options: NativeClaudeStreamNormalizationOptions = {}, -): NativeClaudeStreamEvidence => { - const records = parseStreamRecords(raw); - const envelopes = Object.freeze(records.map(redactEnvelope)); - const allowedPluginNames = options.allowedPluginNames === undefined - ? undefined - : new Set(options.allowedPluginNames); - const pluginNames = new Set(); - let activationEvidence: ClaudeActivationEvidence = 'unavailable'; - let authSource: ClaudeInitAuthSource = 'unavailable'; - let configuredServers = 0; - let toolCalls = 0; - const errorEnvelopes: RedactedClaudeEnvelope[] = []; - const hookEnvelopes: RedactedClaudeEnvelope[] = []; - - for (let index = 0; index < records.length; index += 1) { - const record = records[index]!; - const envelope = envelopes[index]!; - for (const plugin of namesFromRecords(record.plugins)) { - if (allowedPluginNames === undefined || allowedPluginNames.has(plugin)) pluginNames.add(plugin); - } - configuredServers += namesFromRecords(record.mcp_servers).length; - const tools = toolUseNames(record); - if (hasCandidateSkillUse(record, options.candidateSkillEventName)) activationEvidence = 'observed'; - toolCalls += tools.filter((name) => name.startsWith('mcp__')).length; - const recordAuthSource = normalizeInitAuthSource(record.apiKeySource ?? record.authSource ?? record.auth_source); - if (recordAuthSource === 'environment-key' || authSource === 'unavailable') authSource = recordAuthSource; - if ( - record.hook_event_name !== undefined - || record.hook_event !== undefined - || envelope.subtype?.startsWith('hook_') === true - ) hookEnvelopes.push(envelope); - if (envelope.type === 'error' || envelope.subtype === 'error') errorEnvelopes.push(envelope); - } - - return Object.freeze({ - activationEvidence, - authSource, - envelopes, - errorEnvelopes: Object.freeze(errorEnvelopes), - hookEnvelopes: Object.freeze(hookEnvelopes), - mcp: Object.freeze({ configuredServers, toolCalls }), - plugins: Object.freeze([...pluginNames].sort()), - }); -}; - export interface NativeClaudeProcessRequest { readonly args: readonly string[]; readonly cwd: string; @@ -201,258 +71,13 @@ export interface NativeClaudeProcessOptions { readonly timeoutMs?: number; } -export interface NativeClaudeSmokeOptions extends NativeClaudeCommandOptions { - readonly candidatePluginName: string; - readonly candidateSkillName: string; - readonly cwd: string; - readonly enabled: boolean; - readonly environment?: Readonly; - /** Testable authority for the default Claude state directory; production uses the OS home directory. */ - readonly homeDirectory?: string; - readonly run?: NativeClaudeProcessRunner; - readonly signal?: AbortSignal; - /** Per-process timeout override for slow or heavily loaded machines. */ - readonly timeoutMs?: number; -} +const minimumClaudeVersion: SemanticVersion = Object.freeze({ major: 2, minor: 1, patch: 232, prerelease: false }); -export interface NativeClaudeSmokeDiagnostic { - readonly code: string; - readonly message: string; -} +export const parseClaudeVersion = parseSemanticVersion; -export interface NativeClaudeSmokeEvidence { - readonly authentication: Readonly<{ - readonly status: 'subscription-session'; - }>; - readonly command: Readonly<{ - readonly args: readonly string[]; - readonly executable: 'claude'; - }>; - readonly stderr: Readonly<{ - readonly lineCount: number; - readonly present: boolean; - }>; - readonly stream: NativeClaudeStreamEvidence; - readonly validation: Readonly<{ - readonly exitCode: number | null; - }>; - readonly version: string; -} - -export interface NativeClaudeSmokeReport { - readonly diagnostics: readonly NativeClaudeSmokeDiagnostic[]; - readonly evidence?: NativeClaudeSmokeEvidence; - readonly normalHome?: 'unchanged'; - readonly status: 'harness-failure' | 'passed' | 'skipped'; -} +export const formatClaudeVersion = (version: SemanticVersion): string => `${version.major}.${version.minor}.${version.patch}`; -/** - * The normal-home surface the smoke must leave untouched: the settings files, - * the installed plugin tree, and the user-scope MCP registrations inside the - * sibling `.claude.json` state file. The rest of that file is host - * bookkeeping Claude Code rewrites on every signed-in turn (cached feature - * flags, first-start and machine identity, notification and usage counters, - * per-project session statistics — 2.1.257+ even under - * `--no-session-persistence`), so digesting it whole made the guard trip on - * every real run (#439); `mcpServers` is the one durable configuration the - * file carries that a plugin smoke could plausibly alter. - */ -interface ClaudeNormalHomeSnapshot { - readonly config: string; - readonly localSettings: string; - readonly plugins: string; - readonly settings: string; - readonly stateMcpServers: string; -} - -const candidateSkillEventName = (pluginName: string, skillName: string): string => `${pluginName}:${skillName}`; - -const nativeClaudeSmokeCommandShape = Object.freeze({ - args: Object.freeze([ - '-p', - '--plugin-dir', - '', - '--output-format', - 'stream-json', - '--verbose', - '--include-hook-events', - '--no-session-persistence', - '', - ]), - executable: 'claude' as const, -}); - -const diagnostic = (code: string, message: string): readonly NativeClaudeSmokeDiagnostic[] => - Object.freeze([Object.freeze({ code, message })]); - -const stderrEvidence = (stderr: string): NativeClaudeSmokeEvidence['stderr'] => Object.freeze({ - lineCount: stderr.trim().length === 0 ? 0 : stderr.trim().split(/\r?\n/u).length, - present: stderr.trim().length > 0, -}); - -const evidenceFor = ( - authentication: NativeClaudeSmokeEvidence['authentication'], - version: string, - validation: NativeClaudeProcessResult, - execution: NativeClaudeProcessResult, - stream: NativeClaudeStreamEvidence, -): NativeClaudeSmokeEvidence => Object.freeze({ - authentication, - command: nativeClaudeSmokeCommandShape, - stderr: stderrEvidence(execution.stderr), - stream, - validation: Object.freeze({ exitCode: validation.exitCode }), - version, -}); - -/** Stays on `lstat` + `Dirent`: the digest records link identity, which `stat` would follow. */ -const digestClaudeFileTree = async (path: string, includeContents = true): Promise => { - try { - const entry = await lstat(path); - const digest = createHash('sha256'); - if (entry.isFile()) { - digest.update(`file\0${entry.mode}\0${entry.size}\0`); - if (includeContents) digest.update(await readFile(path)); - else digest.update(`${entry.mtimeMs}\0`); - return digest.digest('hex'); - } - if (entry.isDirectory()) { - digest.update('directory\0'); - const children = await readdir(path, { withFileTypes: true }); - for (const child of children.sort((left, right) => left.name.localeCompare(right.name))) { - digest.update(`${child.name}\0${await digestClaudeFileTree(join(path, child.name), includeContents)}\0`); - } - return digest.digest('hex'); - } - digest.update(`other\0${entry.mode}\0`); - return digest.digest('hex'); - } catch (error) { - if (isErrno(error, 'ENOENT')) return 'absent'; - throw error; - } -}; - -interface ClaudeNormalHomePaths { - readonly directory: string; - readonly stateFile: string; -} - -const resolveClaudeNormalHome = ( - environment: Readonly, - homeDirectory = homedir(), -): ClaudeNormalHomePaths => { - const configuredDirectory = environment.CLAUDE_CONFIG_DIR; - if (configuredDirectory !== undefined) { - return Object.freeze({ directory: configuredDirectory, stateFile: join(configuredDirectory, '.claude.json') }); - } - return Object.freeze({ directory: join(homeDirectory, '.claude'), stateFile: join(homeDirectory, '.claude.json') }); -}; - -/** - * Digests the user-scope `mcpServers` registrations of Claude's `.claude.json` - * and nothing else in it. An absent file and a file without the key both mean - * "no registrations" (a first start in a fresh home creates the file without - * any), a file that is not a JSON object digests to its own constant, so the - * guard still notices the smoke creating registrations or corrupting the - * file, while the bookkeeping keys the host rewrites on every turn never enter - * the digest. - */ -const digestClaudeStateMcpServers = async (path: string): Promise => { - let text: string; - try { - text = await runWithPlatform(readFileString(path)); - } catch (error) { - if (isErrno(error, 'ENOENT')) return 'none'; - throw error; - } - let parsed: unknown; - try { - parsed = JSON.parse(text); - } catch { - return 'unparsable'; - } - if (!isRecord(parsed)) return 'unparsable'; - if (!('mcpServers' in parsed)) return 'none'; - return digest(parsed.mcpServers); -}; - -const snapshotClaudeNormalHome = async (paths: ClaudeNormalHomePaths): Promise => Object.freeze({ - config: await digestClaudeFileTree(join(paths.directory, 'config.json')), - localSettings: await digestClaudeFileTree(join(paths.directory, 'settings.local.json')), - plugins: await digestClaudeFileTree(join(paths.directory, 'plugins')), - settings: await digestClaudeFileTree(join(paths.directory, 'settings.json')), - stateMcpServers: await digestClaudeStateMcpServers(paths.stateFile), -}); - -const sameClaudeNormalHome = (left: ClaudeNormalHomeSnapshot, right: ClaudeNormalHomeSnapshot): boolean => - left.config === right.config - && left.localSettings === right.localSettings - && left.plugins === right.plugins - && left.settings === right.settings - && left.stateMcpServers === right.stateMcpServers; - -const normalHomeFailure = (code: string, message: string): NativeClaudeSmokeReport => Object.freeze({ - diagnostics: diagnostic(code, message), - status: 'harness-failure', -}); - -const normalHomeChangedDiagnostic = Object.freeze({ - code: 'claude-native.normal-home.changed', - message: 'Claude normal config/settings/plugins state or user-scope MCP registrations changed; inspect local state without retaining its output.', -}); - -const isMissingExecutableError = (error: unknown): boolean => isErrno(error, 'ENOENT'); - -const looksUnauthenticated = (output: string): boolean => - /(?:not\s+logged\s+in|authentication|authenticate|unauthorized|subscription)/iu.test(output); - -export interface ClaudeVersion { - readonly major: number; - readonly minor: number; - readonly patch: number; - readonly prerelease: boolean; -} - -const minimumClaudeVersion = Object.freeze({ major: 2, minor: 1, patch: 232, prerelease: false }); - -export const parseClaudeVersion = (output: string): ClaudeVersion | undefined => { - const match = /(?:^|[^0-9])(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?=$|[^0-9A-Za-z.-])/u.exec(output); - if (match === null) return undefined; - return Object.freeze({ - major: Number(match[1]), - minor: Number(match[2]), - patch: Number(match[3]), - prerelease: match[4] !== undefined, - }); -}; - -export const formatClaudeVersion = (version: ClaudeVersion): string => `${version.major}.${version.minor}.${version.patch}`; - -export const isCompatibleClaudeVersion = (version: ClaudeVersion): boolean => { - for (const field of ['major', 'minor', 'patch'] as const) { - if (version[field] !== minimumClaudeVersion[field]) return version[field] > minimumClaudeVersion[field]; - } - return !version.prerelease; -}; - -const parseSubscriptionAuthentication = ( - output: string, -): NativeClaudeSmokeEvidence['authentication'] | undefined => { - let value: unknown; - try { - value = JSON.parse(output) as unknown; - } catch { - return undefined; - } - if (!isRecord(value) || value.loggedIn !== true) return undefined; - const authMethod = typeof value.authMethod === 'string' ? value.authMethod.toLowerCase() : ''; - const subscriptionType = typeof value.subscriptionType === 'string' ? value.subscriptionType.toLowerCase() : ''; - const apiProvider = typeof value.apiProvider === 'string' ? value.apiProvider.toLowerCase() : ''; - const usesAlternateProvider = /(?:api[ _-]?key|bedrock|vertex|foundry)/iu.test(`${authMethod}\n${apiProvider}`); - const supportedMethod = authMethod === 'claude.ai' || authMethod === 'oauth' || authMethod.includes('session'); - if (usesAlternateProvider || !supportedMethod || subscriptionType.length === 0 || subscriptionType === 'none') return undefined; - return Object.freeze({ status: 'subscription-session' }); -}; +export const isCompatibleClaudeVersion = (version: SemanticVersion): boolean => meetsMinimumVersion(version, minimumClaudeVersion); const nativeClaudeProcessDefaults = Object.freeze({ gracePeriodMs: 1_000, @@ -492,10 +117,6 @@ const appendBounded = (chunks: Buffer[], chunk: Buffer, maxBytes: number, curren return currentBytes + retained.byteLength; }; -export const nativeClaudeSmokeEnabled = ( - environment: Readonly = process.env, -): boolean => environment.AGENT_BUNDLE_NATIVE_CLAUDE_SMOKE === '1'; - export const runNativeClaudeProcess = ( request: NativeClaudeProcessRequest, signalOrOptions?: AbortSignal | NativeClaudeProcessOptions, @@ -557,287 +178,3 @@ export const runNativeClaudeProcess = ( if (options.signal?.aborted) abort(); else options.signal?.addEventListener('abort', abort, { once: true }); }); - -const runNativeClaudeSmokeUnchecked = async (options: NativeClaudeSmokeOptions): Promise => { - if (!options.enabled) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.opt-in.required', - 'Set AGENT_BUNDLE_NATIVE_CLAUDE_SMOKE=1 to run the signed-in Claude native smoke.', - ), - status: 'skipped', - }); - } - - const environment = createNativeClaudeChildEnvironment(options.environment); - const run = options.run ?? ((request: NativeClaudeProcessRequest) => runNativeClaudeProcess(request, { - signal: options.signal, - timeoutMs: options.timeoutMs, - })); - const versionRequest: NativeClaudeProcessRequest = Object.freeze({ - args: Object.freeze(['--version']), - cwd: options.cwd, - environment, - executable: 'claude', - }); - let versionOutput: NativeClaudeProcessResult; - try { - versionOutput = await run(versionRequest); - } catch (error) { - return Object.freeze({ - diagnostics: diagnostic( - isMissingExecutableError(error) ? 'claude-native.cli.missing' : 'claude-native.version.unavailable', - isMissingExecutableError(error) - ? 'Claude is not installed or is not on PATH; install Claude Code 2.1.232 or newer.' - : 'Claude version preflight could not start; inspect the local CLI without retaining its output.', - ), - status: 'harness-failure', - }); - } - if (versionOutput.exitCode !== 0) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.version.failed', - 'Claude version preflight failed; inspect the local CLI without retaining its output.', - ), - status: 'harness-failure', - }); - } - const version = parseClaudeVersion(versionOutput.stdout); - if (version === undefined) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.version.unparseable', - 'Claude did not report a semantic version for the 2.1.232 native contract.', - ), - status: 'harness-failure', - }); - } - const formattedVersion = formatClaudeVersion(version); - if (!isCompatibleClaudeVersion(version)) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.version.incompatible', - `Claude Code ${formattedVersion} is older than the required 2.1.232 native contract; upgrade the CLI.`, - ), - status: 'harness-failure', - }); - } - - const authRequest: NativeClaudeProcessRequest = Object.freeze({ - args: Object.freeze(['auth', 'status', '--json']), - cwd: options.cwd, - environment, - executable: 'claude', - }); - let authenticationResult: NativeClaudeProcessResult; - try { - authenticationResult = await run(authRequest); - } catch (error) { - return Object.freeze({ - diagnostics: diagnostic( - isMissingExecutableError(error) ? 'claude-native.cli.missing' : 'claude-native.auth.unavailable', - isMissingExecutableError(error) - ? 'Claude is not installed or is not on PATH; install Claude Code 2.1.232 or newer.' - : 'Claude authentication preflight could not start; inspect the local CLI without retaining its output.', - ), - status: 'harness-failure', - }); - } - if (authenticationResult.exitCode !== 0) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.auth.failed', - 'Claude authentication preflight failed; sign in with Claude Code and retry.', - ), - status: 'harness-failure', - }); - } - const authentication = parseSubscriptionAuthentication(authenticationResult.stdout); - if (authentication === undefined) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.auth.unsupported', - 'Claude is not signed in with a supported subscription/session; sign in with Claude Code and retry.', - ), - status: 'harness-failure', - }); - } - - // Name the plugin manifest, not the directory: with `.claude-plugin/marketplace.json` beside it, - // a directory run validates the marketplace and never opens hooks/, skills/, or agents/. - const validationRequest: NativeClaudeProcessRequest = Object.freeze({ - args: Object.freeze(['plugin', 'validate', '--strict', join(options.pluginDirectory, '.claude-plugin', 'plugin.json')]), - cwd: options.cwd, - environment, - executable: 'claude', - }); - let validation: NativeClaudeProcessResult; - try { - validation = await run(validationRequest); - } catch (error) { - return Object.freeze({ - diagnostics: diagnostic( - isMissingExecutableError(error) ? 'claude-native.cli.missing' : 'claude-native.validation.unavailable', - isMissingExecutableError(error) - ? 'Claude is not installed or is not on PATH; install Claude Code 2.1.232 or newer.' - : 'Claude strict plugin validation could not start; inspect the local CLI without retaining its output.', - ), - status: 'harness-failure', - }); - } - if (validation.exitCode !== 0) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.plugin-validation.failed', - 'Claude strict plugin validation failed; inspect the candidate locally without retaining its output.', - ), - status: 'harness-failure', - }); - } - - let execution: NativeClaudeProcessResult; - try { - const command = createNativeClaudeCommand(options); - execution = await run(Object.freeze({ - ...command, - cwd: options.cwd, - environment, - })); - } catch (error) { - return Object.freeze({ - diagnostics: diagnostic( - isMissingExecutableError(error) ? 'claude-native.cli.missing' : 'claude-native.execution.unavailable', - isMissingExecutableError(error) - ? 'Claude is not installed or is not on PATH; install Claude Code 2.1.232 or newer.' - : 'Claude native execution could not start; inspect the local CLI without retaining its output.', - ), - status: 'harness-failure', - }); - } - - let stream: NativeClaudeStreamEvidence; - try { - stream = normalizeNativeClaudeStream(execution.stdout, { - allowedPluginNames: [options.candidatePluginName], - candidateSkillEventName: candidateSkillEventName(options.candidatePluginName, options.candidateSkillName), - }); - } catch { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.stream.invalid', - 'Claude native execution did not return a valid stream-JSON trace; inspect the local CLI without retaining its output.', - ), - status: 'harness-failure', - }); - } - const evidence = evidenceFor(authentication, formattedVersion, validation, execution, stream); - if (execution.exitCode !== 0) { - return Object.freeze({ - diagnostics: diagnostic( - looksUnauthenticated(`${execution.stdout}\n${execution.stderr}`) - ? 'claude-native.authentication.unavailable' - : 'claude-native.execution.failed', - looksUnauthenticated(`${execution.stdout}\n${execution.stderr}`) - ? 'Claude is not authenticated with a usable subscription/session; sign in with Claude Code and retry.' - : 'Claude native execution failed; inspect the local CLI without retaining its output.', - ), - evidence, - status: 'harness-failure', - }); - } - if (stream.authSource === 'environment-key') { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.auth.environment-key', - 'Claude reported an environment-key auth source; remove provider credentials before running the subscription smoke.', - ), - evidence, - status: 'harness-failure', - }); - } - if (!stream.plugins.includes(options.candidatePluginName)) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.plugin.not-loaded', - 'Claude did not report the explicit candidate plugin as loaded; inspect the local CLI without retaining its output.', - ), - evidence, - status: 'harness-failure', - }); - } - if (stream.activationEvidence !== 'observed') { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.activation.unobserved', - 'Claude did not emit the exact candidate Skill tool event; inspect the local CLI without retaining its output.', - ), - evidence, - status: 'harness-failure', - }); - } - if (stream.envelopes.length === 0 || !stream.envelopes.some((envelope) => envelope.type === 'result')) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.stream.result.missing', - 'Claude native execution did not emit a terminal result event; inspect the local CLI without retaining its output.', - ), - evidence, - status: 'harness-failure', - }); - } - if (stream.errorEnvelopes.length > 0) { - return Object.freeze({ - diagnostics: diagnostic( - 'claude-native.stream.error', - 'Claude native execution emitted a host error event; inspect the local CLI without retaining its output.', - ), - evidence, - status: 'harness-failure', - }); - } - - return Object.freeze({ diagnostics: Object.freeze([]), evidence, status: 'passed' }); -}; - -export const runNativeClaudeSmoke = async (options: NativeClaudeSmokeOptions): Promise => { - if (!options.enabled) return runNativeClaudeSmokeUnchecked(options); - - const environment = options.environment ?? process.env; - const normalClaudeHome = resolveClaudeNormalHome(environment, options.homeDirectory); - let before: ClaudeNormalHomeSnapshot; - try { - before = await snapshotClaudeNormalHome(normalClaudeHome); - } catch { - return normalHomeFailure( - 'claude-native.normal-home.unavailable', - 'Claude normal config/settings/plugins could not be inspected; inspect local state without retaining its output.', - ); - } - - const result = await runNativeClaudeSmokeUnchecked(options); - let after: ClaudeNormalHomeSnapshot; - try { - after = await snapshotClaudeNormalHome(normalClaudeHome); - } catch { - return Object.freeze({ - ...result, - diagnostics: Object.freeze([ - ...result.diagnostics, - ...diagnostic( - 'claude-native.normal-home.unavailable', - 'Claude normal config/settings/plugins could not be inspected after the smoke; inspect local state without retaining its output.', - ), - ]), - status: 'harness-failure', - }); - } - if (!sameClaudeNormalHome(before, after)) { - return Object.freeze({ - ...result, - diagnostics: Object.freeze([...result.diagnostics, normalHomeChangedDiagnostic]), - status: 'harness-failure', - }); - } - return Object.freeze({ ...result, normalHome: 'unchanged' as const }); -}; diff --git a/packages/agent-bundle/src/host-contracts/native-codex-contract.ts b/packages/agent-bundle/src/host-contracts/native-codex-contract.ts index 156d458c7..379f50c03 100644 --- a/packages/agent-bundle/src/host-contracts/native-codex-contract.ts +++ b/packages/agent-bundle/src/host-contracts/native-codex-contract.ts @@ -1,130 +1,11 @@ -import { execFile } from 'node:child_process'; -import { lstat } from 'node:fs/promises'; -import { homedir, tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { promisify } from 'node:util'; +import { dirname } from 'node:path'; import { Effect, FileSystem } from 'effect'; import type { PlatformError } from 'effect/PlatformError'; import { runWithPlatform } from '../effect/platform.ts'; -import { meetsMinimumVersion, parseSemanticVersion } from '../core/semver.ts'; import { isCredentialKey, isProviderEndpointKey } from '../core/credentials.ts'; -import { parseRedactedEventEnvelopes, type RedactedEventEnvelope } from './host-contract.ts'; -import { - digestFileTree, - nativeSmokeOptIn, - sameDigestSnapshot, - snapshotDigestSites, - withoutEnvironmentKeysMatching, - type DigestSnapshot, -} from './native-host-spine.ts'; -import { runBoundedChildProcess } from './process.ts'; -import { YieldableFrameworkError } from '../effect/errors.ts'; - -const codexExecutable = 'codex'; -const minimumCodexVersion = '0.147.0'; -const candidatePluginName = 'agent-bundle-codex-smoke'; -const candidateMarketplaceName = 'agent-bundle-codex-smoke-marketplace'; -const smokeSkillSentinel = 'agent-bundle-codex-skill-sentinel'; -const smokePrompt = 'Complete the Agent Bundle Codex smoke attestation by following its Skill, then reply with its exact sentinel and nothing else.'; -const defaultProcessLimits = Object.freeze({ - killGraceMs: 1_000, - maxOutputBytes: 256 * 1024, - timeoutMs: 120_000, -}); - -type CodexNativeSmokeStage = 'auth' | 'candidate' | 'cleanup' | 'exec' | 'fixture' | 'marketplace.add' | 'normal-home' | 'plugin.add' | 'plugin.list' | 'temp-home' | 'version'; - -export interface CodexNativeSmokeProcessLimits { - readonly killGraceMs: number; - readonly maxOutputBytes: number; - readonly timeoutMs: number; -} - -export interface CodexNativeSmokeCommand { - readonly args: readonly string[]; - readonly id: Exclude; -} - -export interface CodexNativeSmokeCommandResult { - readonly exitCode: number; - readonly failure?: 'output-limit' | 'timeout'; - readonly stderr: string; - readonly stdout: string; -} - -export interface CodexNativeSmokeProcessCommand { - readonly args: readonly string[]; - readonly cwd: string; - readonly environment: NodeJS.ProcessEnv; - readonly limits: CodexNativeSmokeProcessLimits; -} - -export type CodexNativeSmokeCommandRunner = ( - command: CodexNativeSmokeProcessCommand, -) => Promise; - -export interface CodexNativeSmokeFailureInput { - readonly code?: string; - readonly failure?: 'output-limit' | 'timeout'; - readonly output?: string; - readonly stage: CodexNativeSmokeStage; - readonly version?: string; -} - -export interface CodexNativeSmokeFailure { - readonly code: string; - readonly kind: 'harness-failure'; -} - -export interface CodexNativeSmokeOptions { - readonly candidateDirectory: string; - readonly environment?: Readonly; - readonly fixtureDirectory: string; - readonly cleanupTemporaryRoot?: (root: string) => Promise; - readonly initializeFixture?: (fixtureDirectory: string) => Promise; - readonly normalCodexHome?: string; - readonly processLimits?: Partial; - readonly run?: CodexNativeSmokeCommandRunner; - readonly temporaryDirectoryParent?: string; -} - -export interface CodexNativeSmokeResult { - readonly activation: Readonly<{ - readonly automatic: 'inferred' | 'unavailable'; - readonly pluginAvailability: 'observed' | 'unavailable'; - }>; - readonly cleanup?: Readonly<{ readonly status: 'failed' }>; - readonly diagnostic?: CodexNativeSmokeFailure; - readonly eventEnvelopes: readonly RedactedEventEnvelope[]; - readonly normalHome: Readonly<{ - readonly auth: 'unchanged' | 'unknown'; - readonly config: 'unchanged' | 'unknown'; - readonly plugins: 'unchanged' | 'unknown'; - }>; - readonly status: 'harness-failure' | 'passed' | 'skipped'; -} - -type CodexStateSite = 'auth' | 'config' | 'plugins'; -type CodexStateSnapshot = DigestSnapshot; - -class SmokeStepError extends YieldableFrameworkError { - readonly code?: string; - readonly failure?: 'output-limit' | 'timeout'; - readonly output?: string; - readonly stage: CodexNativeSmokeStage; - readonly version?: string; - - constructor(input: CodexNativeSmokeFailureInput) { - super(input.stage); - this.code = input.code; - this.failure = input.failure; - this.output = input.output; - this.stage = input.stage; - this.version = input.version; - } -} +import { withoutEnvironmentKeysMatching } from './native-host-spine.ts'; // Shared union credential classifier plus provider endpoint routing, so the // hermetic child cannot see credential material or an env-configured endpoint. @@ -134,71 +15,6 @@ const providerApiKeyName = (name: string): boolean => export const withoutProviderApiKeys = (environment: Readonly): NodeJS.ProcessEnv => withoutEnvironmentKeysMatching(environment, providerApiKeyName); -export const nativeCodexSmokeEnabled = ( - environment: Readonly = process.env, -): boolean => nativeSmokeOptIn(environment, 'AGENT_BUNDLE_NATIVE_CODEX_SMOKE'); - -export const createCodexNativeSmokePlan = ( - paths: Readonly<{ readonly candidateDirectory: string; readonly fixtureDirectory: string }>, -): readonly CodexNativeSmokeCommand[] => Object.freeze([ - Object.freeze({ - args: Object.freeze(['plugin', 'marketplace', 'add', paths.candidateDirectory]), - id: 'marketplace.add' as const, - }), - Object.freeze({ - args: Object.freeze(['plugin', 'add', `${candidatePluginName}@${candidateMarketplaceName}`]), - id: 'plugin.add' as const, - }), - Object.freeze({ args: Object.freeze(['plugin', 'list', '--json']), id: 'plugin.list' as const }), - Object.freeze({ - args: Object.freeze([ - 'exec', - '--strict-config', - '--ephemeral', - '--json', - '-s', - 'read-only', - '-C', - paths.fixtureDirectory, - smokePrompt, - ]), - id: 'exec' as const, - }), -]); - -export const normalizeCodexNativeSmokeEvents = (raw: string): readonly RedactedEventEnvelope[] => - parseRedactedEventEnvelopes(raw); - -const isCompatibleVersion = (value: string): boolean => { - const observed = parseSemanticVersion(value); - const minimum = parseSemanticVersion(minimumCodexVersion)!; - if (observed === undefined) return false; - return meetsMinimumVersion(observed, minimum); -}; - -const authenticationFailure = (output: string | undefined): boolean => - output !== undefined && /(?:\bauth(?:entication)?\b|\blog[ -]?in\b|\bsign[ -]?in\b|\bsubscription\b)/iu.test(output); - -export const classifyCodexNativeSmokeFailure = ( - input: CodexNativeSmokeFailureInput, -): CodexNativeSmokeFailure => { - if (input.failure === 'timeout') return Object.freeze({ code: `native-codex.${input.stage}.timeout`, kind: 'harness-failure' }); - if (input.failure === 'output-limit') return Object.freeze({ code: `native-codex.${input.stage}.output-limit`, kind: 'harness-failure' }); - if (input.stage === 'auth' && input.code === 'ENOENT') { - return Object.freeze({ code: 'native-codex.auth.missing', kind: 'harness-failure' }); - } - if (input.stage === 'version' && input.code === 'ENOENT') { - return Object.freeze({ code: 'native-codex.cli.missing', kind: 'harness-failure' }); - } - if (input.stage === 'version' && input.version !== undefined && !isCompatibleVersion(input.version)) { - return Object.freeze({ code: 'native-codex.cli.incompatible', kind: 'harness-failure' }); - } - if (authenticationFailure(input.output)) { - return Object.freeze({ code: 'native-codex.cli.unauthenticated', kind: 'harness-failure' }); - } - return Object.freeze({ code: `native-codex.${input.stage}.failed`, kind: 'harness-failure' }); -}; - /** Copies `auth.json` bytes and permission bits into a temporary home; nothing inspects the contents. */ export const copyOpaqueCodexAuthStateProgram = Effect.fnUntraced(function* ( source: string, @@ -213,327 +29,3 @@ export const copyOpaqueCodexAuthStateProgram = Effect.fnUntraced(function* ( export const copyOpaqueCodexAuthState = (source: string, destination: string): Promise => runWithPlatform(copyOpaqueCodexAuthStateProgram(source, destination)); - -const snapshotCodexState = (codexHome: string): Promise => - snapshotDigestSites(Object.freeze({ - auth: () => digestFileTree(join(codexHome, 'auth.json')), - config: () => digestFileTree(join(codexHome, 'config.toml')), - plugins: () => digestFileTree(join(codexHome, 'plugins')), - })); - -const normalHomeResult = (before: CodexStateSnapshot | undefined, after: CodexStateSnapshot | undefined) => Object.freeze({ - auth: before !== undefined && after !== undefined && before.auth === after.auth ? 'unchanged' as const : 'unknown' as const, - config: before !== undefined && after !== undefined && before.config === after.config ? 'unchanged' as const : 'unknown' as const, - plugins: before !== undefined && after !== undefined && before.plugins === after.plugins ? 'unchanged' as const : 'unknown' as const, -}); - -const boundedPositiveInteger = (value: number | undefined, fallback: number): number => - Number.isSafeInteger(value) && value !== undefined && value > 0 ? value : fallback; - -const resolveProcessLimits = ( - requested: Partial | undefined, -): CodexNativeSmokeProcessLimits => Object.freeze({ - killGraceMs: boundedPositiveInteger(requested?.killGraceMs, defaultProcessLimits.killGraceMs), - maxOutputBytes: boundedPositiveInteger(requested?.maxOutputBytes, defaultProcessLimits.maxOutputBytes), - timeoutMs: boundedPositiveInteger(requested?.timeoutMs, defaultProcessLimits.timeoutMs), -}); - -const defaultCodexRunner: CodexNativeSmokeCommandRunner = async (command) => { - const result = await runBoundedChildProcess(Object.freeze({ - args: command.args, - cwd: command.cwd, - environment: command.environment, - executable: codexExecutable, - }), Object.freeze({ - discardAfterTermination: true, - forceFinishMs: command.limits.killGraceMs * 2, - gracePeriodMs: command.limits.killGraceMs, - labels: Object.freeze({ outputLimit: 'output-limit', timedOut: 'timeout' }), - maxOutputBytes: command.limits.maxOutputBytes, - overflow: 'truncate', - outputBudget: 'separate', - timeoutMs: command.limits.timeoutMs, - windowsHide: true, - })); - return Object.freeze({ - exitCode: result.exitCode ?? 1, - failure: result.termination, - stderr: result.stderr, - stdout: result.stdout, - }); -}; - -const executeFileAsync = promisify(execFile); - -const initializeCodexSmokeFixture = async (fixtureDirectory: string): Promise => { - await executeFileAsync('git', ['init', '--quiet', fixtureDirectory], { - encoding: 'utf8', - windowsHide: true, - }); -}; - -const outputContainsPlugin = (output: string): boolean => { - try { - const parsed = JSON.parse(output) as unknown; - return JSON.stringify(parsed).includes(candidatePluginName); - } catch { - return output.split(/\r?\n/u).some((line) => { - try { - return JSON.stringify(JSON.parse(line) as unknown).includes(candidatePluginName); - } catch { - return false; - } - }); - } -}; - -const failedResult = ( - failure: CodexNativeSmokeFailure, - before: CodexStateSnapshot | undefined, - after: CodexStateSnapshot | undefined, - eventEnvelopes: readonly RedactedEventEnvelope[] = Object.freeze([]), -): CodexNativeSmokeResult => Object.freeze({ - activation: Object.freeze({ automatic: 'unavailable', pluginAvailability: 'unavailable' }), - diagnostic: failure, - eventEnvelopes, - normalHome: normalHomeResult(before, after), - status: 'harness-failure', -}); - -const skippedCodexNativeSmokeResult: CodexNativeSmokeResult = Object.freeze({ - activation: Object.freeze({ automatic: 'unavailable', pluginAvailability: 'unavailable' }), - eventEnvelopes: Object.freeze([]), - normalHome: Object.freeze({ auth: 'unknown', config: 'unknown', plugins: 'unknown' }), - status: 'skipped', -}); - -const errorCode = (error: unknown): string | undefined => - typeof error === 'object' && error !== null && 'code' in error && typeof error.code === 'string' - ? error.code - : undefined; - -const failedStep = (stage: CodexNativeSmokeStage, error: unknown): SmokeStepError => - error instanceof SmokeStepError ? error : new SmokeStepError({ code: errorCode(error), stage }); - -type CodexSmokeExecutor = ( - stage: CodexNativeSmokeStage, - command: CodexNativeSmokeProcessCommand, -) => Promise; - -const createCodexSmokeExecutor = (runner: CodexNativeSmokeCommandRunner): CodexSmokeExecutor => - async (stage, command) => { - try { - const commandResult = await runner(command); - if (commandResult.failure !== undefined) { - throw new SmokeStepError({ failure: commandResult.failure, stage }); - } - return commandResult; - } catch (error) { - throw failedStep(stage, error); - } - }; - -interface CodexSmokeStaging { - readonly candidate: string; - readonly fixture: string; - readonly home: string; - readonly root: string; -} - -/** Events parsed so far; kept outside the phases so a failing phase still reports them. */ -interface CodexSmokeEvidence { - events: readonly RedactedEventEnvelope[]; -} - -/** - * The smoke root is not a `withTempDirectory` bracket: its removal is - * injectable (`cleanupTemporaryRoot`), and a removal failure is reported in - * the result (`cleanup.status`), not thrown — `removeCodexSmokeRoot` owns it. - */ -const createCodexSmokeRoot = async (temporaryDirectoryParent: string): Promise => { - try { - return await runWithPlatform(Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - yield* fs.makeDirectory(temporaryDirectoryParent, { recursive: true }); - return yield* fs.makeTempDirectory({ directory: temporaryDirectoryParent, prefix: 'agent-bundle-codex-smoke-' }); - })); - } catch (error) { - throw failedStep('temp-home', error); - } -}; - -const codexSmokeStagingFor = (root: string): CodexSmokeStaging => Object.freeze({ - candidate: join(root, 'candidate'), - fixture: join(root, 'fixture'), - home: join(root, 'home'), - root, -}); - -const stageCodexSmokeInputs = async ( - options: CodexNativeSmokeOptions, - staging: CodexSmokeStaging, -): Promise => { - try { - await runWithPlatform(Effect.flatMap(FileSystem.FileSystem, (fs) => fs.makeDirectory(staging.home, { recursive: true }))); - } catch (error) { - throw failedStep('temp-home', error); - } - try { - // `lstat` stays raw: a candidate that is a dangling symlink must fail as `candidate`, not inside the copy. - await lstat(options.candidateDirectory); - await runWithPlatform(Effect.flatMap(FileSystem.FileSystem, (fs) => fs.copy(options.candidateDirectory, staging.candidate, { overwrite: true }))); - } catch (error) { - throw failedStep('candidate', error); - } - try { - await runWithPlatform(Effect.flatMap(FileSystem.FileSystem, (fs) => fs.copy(options.fixtureDirectory, staging.fixture, { overwrite: true }))); - await (options.initializeFixture ?? initializeCodexSmokeFixture)(staging.fixture); - } catch (error) { - throw failedStep('fixture', error); - } -}; - -const runCodexVersionPreflight = async ( - execute: CodexSmokeExecutor, - cwd: string, - environment: NodeJS.ProcessEnv, - limits: CodexNativeSmokeProcessLimits, -): Promise => { - const version = await execute('version', { args: ['--version'], cwd, environment, limits }); - if (version.exitCode !== 0) throw new SmokeStepError({ output: `${version.stdout}\n${version.stderr}`, stage: 'version' }); - if (!isCompatibleVersion(version.stdout)) throw new SmokeStepError({ stage: 'version', version: version.stdout }); -}; - -const adoptCodexSmokeAuth = async (normalCodexHome: string, temporaryHome: string): Promise => { - try { - await copyOpaqueCodexAuthState(join(normalCodexHome, 'auth.json'), join(temporaryHome, 'auth.json')); - } catch (error) { - throw failedStep('auth', error); - } -}; - -const executeCodexSmokePlan = async ( - execute: CodexSmokeExecutor, - staging: CodexSmokeStaging, - environment: NodeJS.ProcessEnv, - limits: CodexNativeSmokeProcessLimits, - evidence: CodexSmokeEvidence, -): Promise => { - const commands = createCodexNativeSmokePlan({ - candidateDirectory: staging.candidate, - fixtureDirectory: staging.fixture, - }); - let pluginAvailability: CodexNativeSmokeResult['activation']['pluginAvailability'] = 'unavailable'; - let automatic: CodexNativeSmokeResult['activation']['automatic'] = 'unavailable'; - for (const command of commands) { - const commandResult = await execute(command.id, { - args: command.args, - cwd: staging.fixture, - environment, - limits, - }); - if (command.id === 'plugin.list' && !outputContainsPlugin(commandResult.stdout)) { - throw new SmokeStepError({ stage: command.id }); - } - if (command.id === 'plugin.list') pluginAvailability = 'observed'; - if (command.id === 'exec') { - try { - evidence.events = normalizeCodexNativeSmokeEvents(commandResult.stdout); - } catch { - throw new SmokeStepError({ stage: command.id }); - } - if (commandResult.stdout.includes(smokeSkillSentinel)) automatic = 'inferred'; - } - if (commandResult.exitCode !== 0) { - throw new SmokeStepError({ output: `${commandResult.stdout}\n${commandResult.stderr}`, stage: command.id }); - } - } - return Object.freeze({ automatic, pluginAvailability }); -}; - -const removeCodexSmokeRoot = async ( - options: CodexNativeSmokeOptions, - root: string, - result: CodexNativeSmokeResult, -): Promise => { - try { - await (options.cleanupTemporaryRoot ?? ((temporaryRoot: string) => runWithPlatform( - Effect.flatMap(FileSystem.FileSystem, (fs) => fs.remove(temporaryRoot, { force: true, recursive: true })), - )))(root); - return result; - } catch { - if (result.status === 'passed') { - return Object.freeze({ - ...result, - cleanup: Object.freeze({ status: 'failed' as const }), - diagnostic: Object.freeze({ code: 'native-codex.cleanup.failed', kind: 'harness-failure' as const }), - status: 'harness-failure', - }); - } - return Object.freeze({ ...result, cleanup: Object.freeze({ status: 'failed' as const }) }); - } -}; - -export const runCodexNativeSmoke = async (options: CodexNativeSmokeOptions): Promise => { - const environment = options.environment ?? process.env; - if (!nativeCodexSmokeEnabled(environment)) return skippedCodexNativeSmokeResult; - - const normalCodexHome = options.normalCodexHome ?? environment.CODEX_HOME ?? join(homedir(), '.codex'); - const temporaryDirectoryParent = options.temporaryDirectoryParent ?? tmpdir(); - const execute = createCodexSmokeExecutor(options.run ?? defaultCodexRunner); - const limits = resolveProcessLimits(options.processLimits); - const evidence: CodexSmokeEvidence = { events: Object.freeze([]) }; - let before: CodexStateSnapshot | undefined; - let after: CodexStateSnapshot | undefined; - let root: string | undefined; - let result: CodexNativeSmokeResult; - - try { - try { - before = await snapshotCodexState(normalCodexHome); - } catch (error) { - throw failedStep('normal-home', error); - } - root = await createCodexSmokeRoot(temporaryDirectoryParent); - const staging = codexSmokeStagingFor(root); - await stageCodexSmokeInputs(options, staging); - const childEnvironment = Object.freeze({ - ...withoutProviderApiKeys(environment), - CODEX_HOME: staging.home, - }); - await runCodexVersionPreflight(execute, staging.fixture, childEnvironment, limits); - await adoptCodexSmokeAuth(normalCodexHome, staging.home); - const activation = await executeCodexSmokePlan(execute, staging, childEnvironment, limits, evidence); - - try { - after = await snapshotCodexState(normalCodexHome); - } catch (error) { - throw failedStep('normal-home', error); - } - if (!sameDigestSnapshot(before, after)) { - result = failedResult(Object.freeze({ code: 'native-codex.normal-home.changed', kind: 'harness-failure' }), before, after, evidence.events); - } else { - result = Object.freeze({ - activation, - eventEnvelopes: evidence.events, - normalHome: normalHomeResult(before, after), - status: 'passed', - }); - } - } catch (error) { - if (before !== undefined) { - try { - after = await snapshotCodexState(normalCodexHome); - } catch { - // The primary structured failure remains authoritative. - } - } - const input = error instanceof SmokeStepError ? error : new SmokeStepError({ stage: 'exec' }); - result = failedResult(classifyCodexNativeSmokeFailure(input), before, after, evidence.events); - } - - return root === undefined ? result : removeCodexSmokeRoot(options, root, result); -}; - -export const codexNativeSmokeReportPath = (repositoryRoot: string): string => - join(repositoryRoot, '.agent-bundle', 'w2-codex-native-contract-evidence.json'); diff --git a/packages/agent-bundle/tests/core.test.ts b/packages/agent-bundle/tests/core.test.ts index e59aaf654..d9a0f8285 100644 --- a/packages/agent-bundle/tests/core.test.ts +++ b/packages/agent-bundle/tests/core.test.ts @@ -8,6 +8,7 @@ import { import { digest, stableJson } from '../src/core/digest.ts'; import { assertInside } from '../src/core/paths.ts'; import { typeScriptTransformFlags } from '../src/core/runtime.ts'; +import { meetsMinimumVersion, parseSemanticVersion } from '../src/core/semver.ts'; import type { McpTransport } from '../src/index.ts'; // Type-level contract: only modern MCP transports are public. @@ -143,3 +144,24 @@ it('defaults to the flags this process accepts, so a child over process.execPath expect(flags).toEqual(typeScriptTransformFlags(process.allowedNodeEnvironmentFlags)); for (const flag of flags) expect(process.allowedNodeEnvironmentFlags.has(flag)).toBe(true); }); + +it('parses the first well-delimited semver token from a CLI banner', () => { + const parsed = (value: string) => { + const version = parseSemanticVersion(value); + return version && `${version.major}.${version.minor}.${version.patch}${version.prerelease ? '-pre' : ''}`; + }; + expect(parsed('2.1.232 (Claude Code)')).toBe('2.1.232'); + expect(parsed('codex-cli 0.147.0')).toBe('0.147.0'); + expect(parsed('1.2.3-rc.1+build.7')).toBe('1.2.3-pre'); + expect(parsed('2.1.232abc')).toBeUndefined(); + expect(parsed('2.1.232.4')).toBeUndefined(); + expect(parsed('codex-cli 0.146.9.1')).toBeUndefined(); + expect(parsed('v22.19.0')).toBe('22.19.0'); + expect(parsed('no version here')).toBeUndefined(); + + const minimum = parseSemanticVersion('2.1.232')!; + expect(meetsMinimumVersion(parseSemanticVersion('2.1.232')!, minimum)).toBe(true); + expect(meetsMinimumVersion(parseSemanticVersion('2.1.232-beta')!, minimum)).toBe(false); + expect(meetsMinimumVersion(parseSemanticVersion('2.2.0-beta')!, minimum)).toBe(true); + expect(meetsMinimumVersion(parseSemanticVersion('2.1.231')!, minimum)).toBe(false); +}); diff --git a/packages/agent-bundle/tests/host-contract.test.ts b/packages/agent-bundle/tests/host-contract.test.ts index b44d1e798..dd980082a 100644 --- a/packages/agent-bundle/tests/host-contract.test.ts +++ b/packages/agent-bundle/tests/host-contract.test.ts @@ -32,7 +32,7 @@ const readContractFixture = async (host: Host) => { }; }; -const loadContractModule = async () => import('../src/host-contracts/host-contract.ts').catch(() => undefined); +const loadContractModule = async () => import('./support/host-contract.ts').catch(() => undefined); const nativeIt = process.env.AGENT_BUNDLE_NATIVE_HOST_CONTRACTS === '1' ? it : it.skip; const jsonStringValues = (value: unknown): readonly string[] => { diff --git a/packages/agent-bundle/tests/native-claude-contract.test.ts b/packages/agent-bundle/tests/native-claude-contract.test.ts index 9ebd291db..f3e3e8cc9 100644 --- a/packages/agent-bundle/tests/native-claude-contract.test.ts +++ b/packages/agent-bundle/tests/native-claude-contract.test.ts @@ -5,7 +5,7 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from '@rstest/core'; -const loadNativeClaudeContract = async () => import('../src/host-contracts/native-claude-contract.ts').catch(() => undefined); +const loadNativeClaudeContract = async () => import('./support/native-claude-smoke.ts').catch(() => undefined); const nativeIt = process.env.AGENT_BUNDLE_NATIVE_CLAUDE_SMOKE === '1' ? it : it.skip; const candidatePluginName = 'agent-bundle-native-smoke'; const candidateSkillName = 'agent-bundle-native-smoke'; diff --git a/packages/agent-bundle/tests/native-codex-contract.test.ts b/packages/agent-bundle/tests/native-codex-contract.test.ts index 4e2e2674c..141398f72 100644 --- a/packages/agent-bundle/tests/native-codex-contract.test.ts +++ b/packages/agent-bundle/tests/native-codex-contract.test.ts @@ -7,7 +7,7 @@ import { expect, it } from '@rstest/core'; const fixtureRoot = new URL('../../../fixtures/contracts/hosts/codex/', import.meta.url); const nativeIt = process.env.AGENT_BUNDLE_NATIVE_CODEX_SMOKE === '1' ? it : it.skip; -const loadContractModule = async () => import('../src/host-contracts/native-codex-contract.ts').catch(() => undefined); +const loadContractModule = async () => import('./support/native-codex-smoke.ts').catch(() => undefined); it('builds the bounded temporary-home Codex lifecycle without API-key arguments', async () => { const contracts = await loadContractModule(); diff --git a/packages/agent-bundle/src/host-contracts/host-contract.ts b/packages/agent-bundle/tests/support/host-contract.ts similarity index 98% rename from packages/agent-bundle/src/host-contracts/host-contract.ts rename to packages/agent-bundle/tests/support/host-contract.ts index ffdaa4aa9..8a0366644 100644 --- a/packages/agent-bundle/src/host-contracts/host-contract.ts +++ b/packages/agent-bundle/tests/support/host-contract.ts @@ -1,13 +1,12 @@ import { execFile as executeFile } from 'node:child_process'; import { promisify } from 'node:util'; -import { isRecord } from '../core/strict-json.ts'; -import { escapeRegExp } from '../core/strings.ts'; -import { isMissingExecutableError } from './native-host-spine.ts'; -import type { NativeHost } from './native-hosts.ts'; +import { isRecord } from '../../src/core/strict-json.ts'; +import { escapeRegExp } from '../../src/core/strings.ts'; +import { isMissingExecutableError } from '../../src/host-contracts/native-host-spine.ts'; +import type { NativeHost } from '../../src/host-contracts/native-hosts.ts'; // Pure parsing and opt-in probing contract for subscription-backed native host CLIs. -export type { NativeHost } from './native-hosts.ts'; export type HostContractStatus = 'changed' | 'compatible' | 'incompatible' | 'missing' | 'skipped'; diff --git a/packages/agent-bundle/tests/support/native-claude-smoke.ts b/packages/agent-bundle/tests/support/native-claude-smoke.ts new file mode 100644 index 000000000..6e2f41f77 --- /dev/null +++ b/packages/agent-bundle/tests/support/native-claude-smoke.ts @@ -0,0 +1,630 @@ +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +import { digest } from '../../src/core/digest.ts'; +import { isErrno } from '../../src/core/errors.ts'; +import { isRecord } from '../../src/core/strict-json.ts'; +import { readFileString, runWithPlatform } from '../../src/effect/platform.ts'; +import { + createNativeClaudeChildEnvironment, + createNativeClaudeCommand, + formatClaudeVersion, + isCompatibleClaudeVersion, + parseClaudeVersion, + runNativeClaudeProcess, + type NativeClaudeCommandOptions, + type NativeClaudeProcessRequest, + type NativeClaudeProcessResult, + type NativeClaudeProcessRunner, +} from '../../src/host-contracts/native-claude-contract.ts'; +import { digestFileTree, isMissingExecutableError } from '../../src/host-contracts/native-host-spine.ts'; + +export { createNativeClaudeChildEnvironment, createNativeClaudeCommand, runNativeClaudeProcess }; + +export type ClaudeActivationEvidence = 'observed' | 'unavailable'; +export type ClaudeInitAuthSource = 'environment-key' | 'non-environment' | 'unavailable'; + +export interface RedactedClaudeEnvelope { + readonly fields: readonly string[]; + readonly subtype?: string; + readonly type?: string; +} + +export interface NativeClaudeStreamEvidence { + readonly activationEvidence: ClaudeActivationEvidence; + readonly authSource: ClaudeInitAuthSource; + readonly envelopes: readonly RedactedClaudeEnvelope[]; + readonly errorEnvelopes: readonly RedactedClaudeEnvelope[]; + readonly hookEnvelopes: readonly RedactedClaudeEnvelope[]; + readonly mcp: Readonly<{ + readonly configuredServers: number; + readonly toolCalls: number; + }>; + readonly plugins: readonly string[]; +} + +export interface NativeClaudeStreamNormalizationOptions { + readonly allowedPluginNames?: readonly string[]; + readonly candidateSkillEventName?: string; +} + +const isSafeLabel = (value: unknown): value is string => + typeof value === 'string' && /^[A-Za-z][A-Za-z0-9_.-]{0,127}$/u.test(value); + +const parseStreamRecords = (raw: string): readonly Readonly>[] => Object.freeze( + raw + .split(/\r?\n/u) + .filter((line) => line.trim().length > 0) + .map((line) => { + const value = JSON.parse(line) as unknown; + if (!isRecord(value)) throw new TypeError('Claude stream event must be a JSON object.'); + return value; + }), +); + +const redactEnvelope = (value: Readonly>): RedactedClaudeEnvelope => Object.freeze({ + fields: Object.freeze(Object.keys(value).sort()), + ...(isSafeLabel(value.subtype) ? { subtype: value.subtype } : {}), + ...(isSafeLabel(value.type) ? { type: value.type } : {}), +}); + +const namesFromRecords = (value: unknown): readonly string[] => !Array.isArray(value) + ? Object.freeze([]) + : Object.freeze(value.flatMap((entry) => isRecord(entry) && isSafeLabel(entry.name) ? [entry.name] : [])); + +const toolUseNames = (value: Readonly>): readonly string[] => { + const message = value.message; + if (!isRecord(message) || !Array.isArray(message.content)) return Object.freeze([]); + return Object.freeze(message.content.flatMap((content) => + isRecord(content) && content.type === 'tool_use' && isSafeLabel(content.name) ? [content.name] : [])); +}; + +const hasCandidateSkillUse = (value: Readonly>, candidateSkillEventName: string | undefined): boolean => { + if (candidateSkillEventName === undefined) return false; + const message = value.message; + if (!isRecord(message) || !Array.isArray(message.content)) return false; + return message.content.some((content) => + isRecord(content) + && content.type === 'tool_use' + && content.name === 'Skill' + && isRecord(content.input) + && content.input.skill === candidateSkillEventName); +}; + +const normalizeInitAuthSource = (value: unknown): ClaudeInitAuthSource => { + if (typeof value !== 'string') return 'unavailable'; + return /(?:environment|env|api[ _-]?key)/iu.test(value) ? 'environment-key' : 'non-environment'; +}; + +export const normalizeNativeClaudeStream = ( + raw: string, + options: NativeClaudeStreamNormalizationOptions = {}, +): NativeClaudeStreamEvidence => { + const records = parseStreamRecords(raw); + const envelopes = Object.freeze(records.map(redactEnvelope)); + const allowedPluginNames = options.allowedPluginNames === undefined + ? undefined + : new Set(options.allowedPluginNames); + const pluginNames = new Set(); + let activationEvidence: ClaudeActivationEvidence = 'unavailable'; + let authSource: ClaudeInitAuthSource = 'unavailable'; + let configuredServers = 0; + let toolCalls = 0; + const errorEnvelopes: RedactedClaudeEnvelope[] = []; + const hookEnvelopes: RedactedClaudeEnvelope[] = []; + + for (let index = 0; index < records.length; index += 1) { + const record = records[index]!; + const envelope = envelopes[index]!; + for (const plugin of namesFromRecords(record.plugins)) { + if (allowedPluginNames === undefined || allowedPluginNames.has(plugin)) pluginNames.add(plugin); + } + configuredServers += namesFromRecords(record.mcp_servers).length; + const tools = toolUseNames(record); + if (hasCandidateSkillUse(record, options.candidateSkillEventName)) activationEvidence = 'observed'; + toolCalls += tools.filter((name) => name.startsWith('mcp__')).length; + const recordAuthSource = normalizeInitAuthSource(record.apiKeySource ?? record.authSource ?? record.auth_source); + if (recordAuthSource === 'environment-key' || authSource === 'unavailable') authSource = recordAuthSource; + if ( + record.hook_event_name !== undefined + || record.hook_event !== undefined + || envelope.subtype?.startsWith('hook_') === true + ) hookEnvelopes.push(envelope); + if (envelope.type === 'error' || envelope.subtype === 'error') errorEnvelopes.push(envelope); + } + + return Object.freeze({ + activationEvidence, + authSource, + envelopes, + errorEnvelopes: Object.freeze(errorEnvelopes), + hookEnvelopes: Object.freeze(hookEnvelopes), + mcp: Object.freeze({ configuredServers, toolCalls }), + plugins: Object.freeze([...pluginNames].sort()), + }); +}; + +export interface NativeClaudeSmokeOptions extends NativeClaudeCommandOptions { + readonly candidatePluginName: string; + readonly candidateSkillName: string; + readonly cwd: string; + readonly enabled: boolean; + readonly environment?: Readonly; + /** Testable authority for the default Claude state directory; production uses the OS home directory. */ + readonly homeDirectory?: string; + readonly run?: NativeClaudeProcessRunner; + readonly signal?: AbortSignal; + /** Per-process timeout override for slow or heavily loaded machines. */ + readonly timeoutMs?: number; +} + +export interface NativeClaudeSmokeDiagnostic { + readonly code: string; + readonly message: string; +} + +export interface NativeClaudeSmokeEvidence { + readonly authentication: Readonly<{ + readonly status: 'subscription-session'; + }>; + readonly command: Readonly<{ + readonly args: readonly string[]; + readonly executable: 'claude'; + }>; + readonly stderr: Readonly<{ + readonly lineCount: number; + readonly present: boolean; + }>; + readonly stream: NativeClaudeStreamEvidence; + readonly validation: Readonly<{ + readonly exitCode: number | null; + }>; + readonly version: string; +} + +export interface NativeClaudeSmokeReport { + readonly diagnostics: readonly NativeClaudeSmokeDiagnostic[]; + readonly evidence?: NativeClaudeSmokeEvidence; + readonly normalHome?: 'unchanged'; + readonly status: 'harness-failure' | 'passed' | 'skipped'; +} + +/** + * The normal-home surface the smoke must leave untouched: the settings files, + * the installed plugin tree, and the user-scope MCP registrations inside the + * sibling `.claude.json` state file. The rest of that file is host + * bookkeeping Claude Code rewrites on every signed-in turn (cached feature + * flags, first-start and machine identity, notification and usage counters, + * per-project session statistics — 2.1.257+ even under + * `--no-session-persistence`), so digesting it whole made the guard trip on + * every real run (#439); `mcpServers` is the one durable configuration the + * file carries that a plugin smoke could plausibly alter. + */ +interface ClaudeNormalHomeSnapshot { + readonly config: string; + readonly localSettings: string; + readonly plugins: string; + readonly settings: string; + readonly stateMcpServers: string; +} + +const candidateSkillEventName = (pluginName: string, skillName: string): string => `${pluginName}:${skillName}`; + +const nativeClaudeSmokeCommandShape = Object.freeze({ + args: Object.freeze([ + '-p', + '--plugin-dir', + '', + '--output-format', + 'stream-json', + '--verbose', + '--include-hook-events', + '--no-session-persistence', + '', + ]), + executable: 'claude' as const, +}); + +const diagnostic = (code: string, message: string): readonly NativeClaudeSmokeDiagnostic[] => + Object.freeze([Object.freeze({ code, message })]); + +const stderrEvidence = (stderr: string): NativeClaudeSmokeEvidence['stderr'] => Object.freeze({ + lineCount: stderr.trim().length === 0 ? 0 : stderr.trim().split(/\r?\n/u).length, + present: stderr.trim().length > 0, +}); + +const evidenceFor = ( + authentication: NativeClaudeSmokeEvidence['authentication'], + version: string, + validation: NativeClaudeProcessResult, + execution: NativeClaudeProcessResult, + stream: NativeClaudeStreamEvidence, +): NativeClaudeSmokeEvidence => Object.freeze({ + authentication, + command: nativeClaudeSmokeCommandShape, + stderr: stderrEvidence(execution.stderr), + stream, + validation: Object.freeze({ exitCode: validation.exitCode }), + version, +}); + +const digestClaudeFileTree = (path: string): Promise => digestFileTree(path, { includeIdentity: true }); + +interface ClaudeNormalHomePaths { + readonly directory: string; + readonly stateFile: string; +} + +const resolveClaudeNormalHome = ( + environment: Readonly, + homeDirectory = homedir(), +): ClaudeNormalHomePaths => { + const configuredDirectory = environment.CLAUDE_CONFIG_DIR; + if (configuredDirectory !== undefined) { + return Object.freeze({ directory: configuredDirectory, stateFile: join(configuredDirectory, '.claude.json') }); + } + return Object.freeze({ directory: join(homeDirectory, '.claude'), stateFile: join(homeDirectory, '.claude.json') }); +}; + +/** + * Digests the user-scope `mcpServers` registrations of Claude's `.claude.json` + * and nothing else in it. An absent file and a file without the key both mean + * "no registrations" (a first start in a fresh home creates the file without + * any), a file that is not a JSON object digests to its own constant, so the + * guard still notices the smoke creating registrations or corrupting the + * file, while the bookkeeping keys the host rewrites on every turn never enter + * the digest. + */ +const digestClaudeStateMcpServers = async (path: string): Promise => { + let text: string; + try { + text = await runWithPlatform(readFileString(path)); + } catch (error) { + if (isErrno(error, 'ENOENT')) return 'none'; + throw error; + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return 'unparsable'; + } + if (!isRecord(parsed)) return 'unparsable'; + if (!('mcpServers' in parsed)) return 'none'; + return digest(parsed.mcpServers); +}; + +const snapshotClaudeNormalHome = async (paths: ClaudeNormalHomePaths): Promise => Object.freeze({ + config: await digestClaudeFileTree(join(paths.directory, 'config.json')), + localSettings: await digestClaudeFileTree(join(paths.directory, 'settings.local.json')), + plugins: await digestClaudeFileTree(join(paths.directory, 'plugins')), + settings: await digestClaudeFileTree(join(paths.directory, 'settings.json')), + stateMcpServers: await digestClaudeStateMcpServers(paths.stateFile), +}); + +const sameClaudeNormalHome = (left: ClaudeNormalHomeSnapshot, right: ClaudeNormalHomeSnapshot): boolean => + left.config === right.config + && left.localSettings === right.localSettings + && left.plugins === right.plugins + && left.settings === right.settings + && left.stateMcpServers === right.stateMcpServers; + +const normalHomeFailure = (code: string, message: string): NativeClaudeSmokeReport => Object.freeze({ + diagnostics: diagnostic(code, message), + status: 'harness-failure', +}); + +const normalHomeChangedDiagnostic = Object.freeze({ + code: 'claude-native.normal-home.changed', + message: 'Claude normal config/settings/plugins state or user-scope MCP registrations changed; inspect local state without retaining its output.', +}); + +const looksUnauthenticated = (output: string): boolean => + /(?:not\s+logged\s+in|authentication|authenticate|unauthorized|subscription)/iu.test(output); + +const parseSubscriptionAuthentication = ( + output: string, +): NativeClaudeSmokeEvidence['authentication'] | undefined => { + let value: unknown; + try { + value = JSON.parse(output) as unknown; + } catch { + return undefined; + } + if (!isRecord(value) || value.loggedIn !== true) return undefined; + const authMethod = typeof value.authMethod === 'string' ? value.authMethod.toLowerCase() : ''; + const subscriptionType = typeof value.subscriptionType === 'string' ? value.subscriptionType.toLowerCase() : ''; + const apiProvider = typeof value.apiProvider === 'string' ? value.apiProvider.toLowerCase() : ''; + const usesAlternateProvider = /(?:api[ _-]?key|bedrock|vertex|foundry)/iu.test(`${authMethod}\n${apiProvider}`); + const supportedMethod = authMethod === 'claude.ai' || authMethod === 'oauth' || authMethod.includes('session'); + if (usesAlternateProvider || !supportedMethod || subscriptionType.length === 0 || subscriptionType === 'none') return undefined; + return Object.freeze({ status: 'subscription-session' }); +}; + +export const nativeClaudeSmokeEnabled = ( + environment: Readonly = process.env, +): boolean => environment.AGENT_BUNDLE_NATIVE_CLAUDE_SMOKE === '1'; + +const runNativeClaudeSmokeUnchecked = async (options: NativeClaudeSmokeOptions): Promise => { + if (!options.enabled) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.opt-in.required', + 'Set AGENT_BUNDLE_NATIVE_CLAUDE_SMOKE=1 to run the signed-in Claude native smoke.', + ), + status: 'skipped', + }); + } + + const environment = createNativeClaudeChildEnvironment(options.environment); + const run = options.run ?? ((request: NativeClaudeProcessRequest) => runNativeClaudeProcess(request, { + signal: options.signal, + timeoutMs: options.timeoutMs, + })); + const versionRequest: NativeClaudeProcessRequest = Object.freeze({ + args: Object.freeze(['--version']), + cwd: options.cwd, + environment, + executable: 'claude', + }); + let versionOutput: NativeClaudeProcessResult; + try { + versionOutput = await run(versionRequest); + } catch (error) { + return Object.freeze({ + diagnostics: diagnostic( + isMissingExecutableError(error) ? 'claude-native.cli.missing' : 'claude-native.version.unavailable', + isMissingExecutableError(error) + ? 'Claude is not installed or is not on PATH; install Claude Code 2.1.232 or newer.' + : 'Claude version preflight could not start; inspect the local CLI without retaining its output.', + ), + status: 'harness-failure', + }); + } + if (versionOutput.exitCode !== 0) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.version.failed', + 'Claude version preflight failed; inspect the local CLI without retaining its output.', + ), + status: 'harness-failure', + }); + } + const version = parseClaudeVersion(versionOutput.stdout); + if (version === undefined) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.version.unparseable', + 'Claude did not report a semantic version for the 2.1.232 native contract.', + ), + status: 'harness-failure', + }); + } + const formattedVersion = formatClaudeVersion(version); + if (!isCompatibleClaudeVersion(version)) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.version.incompatible', + `Claude Code ${formattedVersion} is older than the required 2.1.232 native contract; upgrade the CLI.`, + ), + status: 'harness-failure', + }); + } + + const authRequest: NativeClaudeProcessRequest = Object.freeze({ + args: Object.freeze(['auth', 'status', '--json']), + cwd: options.cwd, + environment, + executable: 'claude', + }); + let authenticationResult: NativeClaudeProcessResult; + try { + authenticationResult = await run(authRequest); + } catch (error) { + return Object.freeze({ + diagnostics: diagnostic( + isMissingExecutableError(error) ? 'claude-native.cli.missing' : 'claude-native.auth.unavailable', + isMissingExecutableError(error) + ? 'Claude is not installed or is not on PATH; install Claude Code 2.1.232 or newer.' + : 'Claude authentication preflight could not start; inspect the local CLI without retaining its output.', + ), + status: 'harness-failure', + }); + } + if (authenticationResult.exitCode !== 0) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.auth.failed', + 'Claude authentication preflight failed; sign in with Claude Code and retry.', + ), + status: 'harness-failure', + }); + } + const authentication = parseSubscriptionAuthentication(authenticationResult.stdout); + if (authentication === undefined) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.auth.unsupported', + 'Claude is not signed in with a supported subscription/session; sign in with Claude Code and retry.', + ), + status: 'harness-failure', + }); + } + + // Name the plugin manifest, not the directory: with `.claude-plugin/marketplace.json` beside it, + // a directory run validates the marketplace and never opens hooks/, skills/, or agents/. + const validationRequest: NativeClaudeProcessRequest = Object.freeze({ + args: Object.freeze(['plugin', 'validate', '--strict', join(options.pluginDirectory, '.claude-plugin', 'plugin.json')]), + cwd: options.cwd, + environment, + executable: 'claude', + }); + let validation: NativeClaudeProcessResult; + try { + validation = await run(validationRequest); + } catch (error) { + return Object.freeze({ + diagnostics: diagnostic( + isMissingExecutableError(error) ? 'claude-native.cli.missing' : 'claude-native.validation.unavailable', + isMissingExecutableError(error) + ? 'Claude is not installed or is not on PATH; install Claude Code 2.1.232 or newer.' + : 'Claude strict plugin validation could not start; inspect the local CLI without retaining its output.', + ), + status: 'harness-failure', + }); + } + if (validation.exitCode !== 0) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.plugin-validation.failed', + 'Claude strict plugin validation failed; inspect the candidate locally without retaining its output.', + ), + status: 'harness-failure', + }); + } + + let execution: NativeClaudeProcessResult; + try { + const command = createNativeClaudeCommand(options); + execution = await run(Object.freeze({ + ...command, + cwd: options.cwd, + environment, + })); + } catch (error) { + return Object.freeze({ + diagnostics: diagnostic( + isMissingExecutableError(error) ? 'claude-native.cli.missing' : 'claude-native.execution.unavailable', + isMissingExecutableError(error) + ? 'Claude is not installed or is not on PATH; install Claude Code 2.1.232 or newer.' + : 'Claude native execution could not start; inspect the local CLI without retaining its output.', + ), + status: 'harness-failure', + }); + } + + let stream: NativeClaudeStreamEvidence; + try { + stream = normalizeNativeClaudeStream(execution.stdout, { + allowedPluginNames: [options.candidatePluginName], + candidateSkillEventName: candidateSkillEventName(options.candidatePluginName, options.candidateSkillName), + }); + } catch { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.stream.invalid', + 'Claude native execution did not return a valid stream-JSON trace; inspect the local CLI without retaining its output.', + ), + status: 'harness-failure', + }); + } + const evidence = evidenceFor(authentication, formattedVersion, validation, execution, stream); + if (execution.exitCode !== 0) { + return Object.freeze({ + diagnostics: diagnostic( + looksUnauthenticated(`${execution.stdout}\n${execution.stderr}`) + ? 'claude-native.authentication.unavailable' + : 'claude-native.execution.failed', + looksUnauthenticated(`${execution.stdout}\n${execution.stderr}`) + ? 'Claude is not authenticated with a usable subscription/session; sign in with Claude Code and retry.' + : 'Claude native execution failed; inspect the local CLI without retaining its output.', + ), + evidence, + status: 'harness-failure', + }); + } + if (stream.authSource === 'environment-key') { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.auth.environment-key', + 'Claude reported an environment-key auth source; remove provider credentials before running the subscription smoke.', + ), + evidence, + status: 'harness-failure', + }); + } + if (!stream.plugins.includes(options.candidatePluginName)) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.plugin.not-loaded', + 'Claude did not report the explicit candidate plugin as loaded; inspect the local CLI without retaining its output.', + ), + evidence, + status: 'harness-failure', + }); + } + if (stream.activationEvidence !== 'observed') { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.activation.unobserved', + 'Claude did not emit the exact candidate Skill tool event; inspect the local CLI without retaining its output.', + ), + evidence, + status: 'harness-failure', + }); + } + if (stream.envelopes.length === 0 || !stream.envelopes.some((envelope) => envelope.type === 'result')) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.stream.result.missing', + 'Claude native execution did not emit a terminal result event; inspect the local CLI without retaining its output.', + ), + evidence, + status: 'harness-failure', + }); + } + if (stream.errorEnvelopes.length > 0) { + return Object.freeze({ + diagnostics: diagnostic( + 'claude-native.stream.error', + 'Claude native execution emitted a host error event; inspect the local CLI without retaining its output.', + ), + evidence, + status: 'harness-failure', + }); + } + + return Object.freeze({ diagnostics: Object.freeze([]), evidence, status: 'passed' }); +}; + +export const runNativeClaudeSmoke = async (options: NativeClaudeSmokeOptions): Promise => { + if (!options.enabled) return runNativeClaudeSmokeUnchecked(options); + + const environment = options.environment ?? process.env; + const normalClaudeHome = resolveClaudeNormalHome(environment, options.homeDirectory); + let before: ClaudeNormalHomeSnapshot; + try { + before = await snapshotClaudeNormalHome(normalClaudeHome); + } catch { + return normalHomeFailure( + 'claude-native.normal-home.unavailable', + 'Claude normal config/settings/plugins could not be inspected; inspect local state without retaining its output.', + ); + } + + const result = await runNativeClaudeSmokeUnchecked(options); + let after: ClaudeNormalHomeSnapshot; + try { + after = await snapshotClaudeNormalHome(normalClaudeHome); + } catch { + return Object.freeze({ + ...result, + diagnostics: Object.freeze([ + ...result.diagnostics, + ...diagnostic( + 'claude-native.normal-home.unavailable', + 'Claude normal config/settings/plugins could not be inspected after the smoke; inspect local state without retaining its output.', + ), + ]), + status: 'harness-failure', + }); + } + if (!sameClaudeNormalHome(before, after)) { + return Object.freeze({ + ...result, + diagnostics: Object.freeze([...result.diagnostics, normalHomeChangedDiagnostic]), + status: 'harness-failure', + }); + } + return Object.freeze({ ...result, normalHome: 'unchanged' as const }); +}; diff --git a/packages/agent-bundle/tests/support/native-codex-smoke.ts b/packages/agent-bundle/tests/support/native-codex-smoke.ts new file mode 100644 index 000000000..15616eae1 --- /dev/null +++ b/packages/agent-bundle/tests/support/native-codex-smoke.ts @@ -0,0 +1,516 @@ +import { execFile } from 'node:child_process'; +import { lstat } from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +import { Effect, FileSystem } from 'effect'; + +import { meetsMinimumVersion, parseSemanticVersion } from '../../src/core/semver.ts'; +import { YieldableFrameworkError } from '../../src/effect/errors.ts'; +import { runWithPlatform } from '../../src/effect/platform.ts'; +import { copyOpaqueCodexAuthState, withoutProviderApiKeys } from '../../src/host-contracts/native-codex-contract.ts'; +import { + digestFileTree, + nativeSmokeOptIn, + sameDigestSnapshot, + snapshotDigestSites, + type DigestSnapshot, +} from '../../src/host-contracts/native-host-spine.ts'; +import { runBoundedChildProcess } from '../../src/host-contracts/process.ts'; +import { parseRedactedEventEnvelopes, type RedactedEventEnvelope } from './host-contract.ts'; + +export { copyOpaqueCodexAuthState, withoutProviderApiKeys }; + +const codexExecutable = 'codex'; +const minimumCodexVersion = '0.147.0'; +const candidatePluginName = 'agent-bundle-codex-smoke'; +const candidateMarketplaceName = 'agent-bundle-codex-smoke-marketplace'; +const smokeSkillSentinel = 'agent-bundle-codex-skill-sentinel'; +const smokePrompt = 'Complete the Agent Bundle Codex smoke attestation by following its Skill, then reply with its exact sentinel and nothing else.'; +const defaultProcessLimits = Object.freeze({ + killGraceMs: 1_000, + maxOutputBytes: 256 * 1024, + timeoutMs: 120_000, +}); + +type CodexNativeSmokeStage = 'auth' | 'candidate' | 'cleanup' | 'exec' | 'fixture' | 'marketplace.add' | 'normal-home' | 'plugin.add' | 'plugin.list' | 'temp-home' | 'version'; + +export interface CodexNativeSmokeProcessLimits { + readonly killGraceMs: number; + readonly maxOutputBytes: number; + readonly timeoutMs: number; +} + +export interface CodexNativeSmokeCommand { + readonly args: readonly string[]; + readonly id: Exclude; +} + +export interface CodexNativeSmokeCommandResult { + readonly exitCode: number; + readonly failure?: 'output-limit' | 'timeout'; + readonly stderr: string; + readonly stdout: string; +} + +export interface CodexNativeSmokeProcessCommand { + readonly args: readonly string[]; + readonly cwd: string; + readonly environment: NodeJS.ProcessEnv; + readonly limits: CodexNativeSmokeProcessLimits; +} + +export type CodexNativeSmokeCommandRunner = ( + command: CodexNativeSmokeProcessCommand, +) => Promise; + +export interface CodexNativeSmokeFailureInput { + readonly code?: string; + readonly failure?: 'output-limit' | 'timeout'; + readonly output?: string; + readonly stage: CodexNativeSmokeStage; + readonly version?: string; +} + +export interface CodexNativeSmokeFailure { + readonly code: string; + readonly kind: 'harness-failure'; +} + +export interface CodexNativeSmokeOptions { + readonly candidateDirectory: string; + readonly environment?: Readonly; + readonly fixtureDirectory: string; + readonly cleanupTemporaryRoot?: (root: string) => Promise; + readonly initializeFixture?: (fixtureDirectory: string) => Promise; + readonly normalCodexHome?: string; + readonly processLimits?: Partial; + readonly run?: CodexNativeSmokeCommandRunner; + readonly temporaryDirectoryParent?: string; +} + +export interface CodexNativeSmokeResult { + readonly activation: Readonly<{ + readonly automatic: 'inferred' | 'unavailable'; + readonly pluginAvailability: 'observed' | 'unavailable'; + }>; + readonly cleanup?: Readonly<{ readonly status: 'failed' }>; + readonly diagnostic?: CodexNativeSmokeFailure; + readonly eventEnvelopes: readonly RedactedEventEnvelope[]; + readonly normalHome: Readonly<{ + readonly auth: 'unchanged' | 'unknown'; + readonly config: 'unchanged' | 'unknown'; + readonly plugins: 'unchanged' | 'unknown'; + }>; + readonly status: 'harness-failure' | 'passed' | 'skipped'; +} + +type CodexStateSite = 'auth' | 'config' | 'plugins'; +type CodexStateSnapshot = DigestSnapshot; + +class SmokeStepError extends YieldableFrameworkError { + readonly code?: string; + readonly failure?: 'output-limit' | 'timeout'; + readonly output?: string; + readonly stage: CodexNativeSmokeStage; + readonly version?: string; + + constructor(input: CodexNativeSmokeFailureInput) { + super(input.stage); + this.code = input.code; + this.failure = input.failure; + this.output = input.output; + this.stage = input.stage; + this.version = input.version; + } +} + +export const nativeCodexSmokeEnabled = ( + environment: Readonly = process.env, +): boolean => nativeSmokeOptIn(environment, 'AGENT_BUNDLE_NATIVE_CODEX_SMOKE'); + +export const createCodexNativeSmokePlan = ( + paths: Readonly<{ readonly candidateDirectory: string; readonly fixtureDirectory: string }>, +): readonly CodexNativeSmokeCommand[] => Object.freeze([ + Object.freeze({ + args: Object.freeze(['plugin', 'marketplace', 'add', paths.candidateDirectory]), + id: 'marketplace.add' as const, + }), + Object.freeze({ + args: Object.freeze(['plugin', 'add', `${candidatePluginName}@${candidateMarketplaceName}`]), + id: 'plugin.add' as const, + }), + Object.freeze({ args: Object.freeze(['plugin', 'list', '--json']), id: 'plugin.list' as const }), + Object.freeze({ + args: Object.freeze([ + 'exec', + '--strict-config', + '--ephemeral', + '--json', + '-s', + 'read-only', + '-C', + paths.fixtureDirectory, + smokePrompt, + ]), + id: 'exec' as const, + }), +]); + +export const normalizeCodexNativeSmokeEvents = (raw: string): readonly RedactedEventEnvelope[] => + parseRedactedEventEnvelopes(raw); + +const isCompatibleVersion = (value: string): boolean => { + const observed = parseSemanticVersion(value); + const minimum = parseSemanticVersion(minimumCodexVersion)!; + if (observed === undefined) return false; + return meetsMinimumVersion(observed, minimum); +}; + +const authenticationFailure = (output: string | undefined): boolean => + output !== undefined && /(?:\bauth(?:entication)?\b|\blog[ -]?in\b|\bsign[ -]?in\b|\bsubscription\b)/iu.test(output); + +export const classifyCodexNativeSmokeFailure = ( + input: CodexNativeSmokeFailureInput, +): CodexNativeSmokeFailure => { + if (input.failure === 'timeout') return Object.freeze({ code: `native-codex.${input.stage}.timeout`, kind: 'harness-failure' }); + if (input.failure === 'output-limit') return Object.freeze({ code: `native-codex.${input.stage}.output-limit`, kind: 'harness-failure' }); + if (input.stage === 'auth' && input.code === 'ENOENT') { + return Object.freeze({ code: 'native-codex.auth.missing', kind: 'harness-failure' }); + } + if (input.stage === 'version' && input.code === 'ENOENT') { + return Object.freeze({ code: 'native-codex.cli.missing', kind: 'harness-failure' }); + } + if (input.stage === 'version' && input.version !== undefined && !isCompatibleVersion(input.version)) { + return Object.freeze({ code: 'native-codex.cli.incompatible', kind: 'harness-failure' }); + } + if (authenticationFailure(input.output)) { + return Object.freeze({ code: 'native-codex.cli.unauthenticated', kind: 'harness-failure' }); + } + return Object.freeze({ code: `native-codex.${input.stage}.failed`, kind: 'harness-failure' }); +}; + +const snapshotCodexState = (codexHome: string): Promise => + snapshotDigestSites(Object.freeze({ + auth: () => digestFileTree(join(codexHome, 'auth.json')), + config: () => digestFileTree(join(codexHome, 'config.toml')), + plugins: () => digestFileTree(join(codexHome, 'plugins')), + })); + +const normalHomeResult = (before: CodexStateSnapshot | undefined, after: CodexStateSnapshot | undefined) => Object.freeze({ + auth: before !== undefined && after !== undefined && before.auth === after.auth ? 'unchanged' as const : 'unknown' as const, + config: before !== undefined && after !== undefined && before.config === after.config ? 'unchanged' as const : 'unknown' as const, + plugins: before !== undefined && after !== undefined && before.plugins === after.plugins ? 'unchanged' as const : 'unknown' as const, +}); + +const boundedPositiveInteger = (value: number | undefined, fallback: number): number => + Number.isSafeInteger(value) && value !== undefined && value > 0 ? value : fallback; + +const resolveProcessLimits = ( + requested: Partial | undefined, +): CodexNativeSmokeProcessLimits => Object.freeze({ + killGraceMs: boundedPositiveInteger(requested?.killGraceMs, defaultProcessLimits.killGraceMs), + maxOutputBytes: boundedPositiveInteger(requested?.maxOutputBytes, defaultProcessLimits.maxOutputBytes), + timeoutMs: boundedPositiveInteger(requested?.timeoutMs, defaultProcessLimits.timeoutMs), +}); + +const defaultCodexRunner: CodexNativeSmokeCommandRunner = async (command) => { + const result = await runBoundedChildProcess(Object.freeze({ + args: command.args, + cwd: command.cwd, + environment: command.environment, + executable: codexExecutable, + }), Object.freeze({ + discardAfterTermination: true, + forceFinishMs: command.limits.killGraceMs * 2, + gracePeriodMs: command.limits.killGraceMs, + labels: Object.freeze({ outputLimit: 'output-limit', timedOut: 'timeout' }), + maxOutputBytes: command.limits.maxOutputBytes, + overflow: 'truncate', + outputBudget: 'separate', + timeoutMs: command.limits.timeoutMs, + windowsHide: true, + })); + return Object.freeze({ + exitCode: result.exitCode ?? 1, + failure: result.termination, + stderr: result.stderr, + stdout: result.stdout, + }); +}; + +const executeFileAsync = promisify(execFile); + +const initializeCodexSmokeFixture = async (fixtureDirectory: string): Promise => { + await executeFileAsync('git', ['init', '--quiet', fixtureDirectory], { + encoding: 'utf8', + windowsHide: true, + }); +}; + +const outputContainsPlugin = (output: string): boolean => { + try { + const parsed = JSON.parse(output) as unknown; + return JSON.stringify(parsed).includes(candidatePluginName); + } catch { + return output.split(/\r?\n/u).some((line) => { + try { + return JSON.stringify(JSON.parse(line) as unknown).includes(candidatePluginName); + } catch { + return false; + } + }); + } +}; + +const failedResult = ( + failure: CodexNativeSmokeFailure, + before: CodexStateSnapshot | undefined, + after: CodexStateSnapshot | undefined, + eventEnvelopes: readonly RedactedEventEnvelope[] = Object.freeze([]), +): CodexNativeSmokeResult => Object.freeze({ + activation: Object.freeze({ automatic: 'unavailable', pluginAvailability: 'unavailable' }), + diagnostic: failure, + eventEnvelopes, + normalHome: normalHomeResult(before, after), + status: 'harness-failure', +}); + +const skippedCodexNativeSmokeResult: CodexNativeSmokeResult = Object.freeze({ + activation: Object.freeze({ automatic: 'unavailable', pluginAvailability: 'unavailable' }), + eventEnvelopes: Object.freeze([]), + normalHome: Object.freeze({ auth: 'unknown', config: 'unknown', plugins: 'unknown' }), + status: 'skipped', +}); + +const errorCode = (error: unknown): string | undefined => + typeof error === 'object' && error !== null && 'code' in error && typeof error.code === 'string' + ? error.code + : undefined; + +const failedStep = (stage: CodexNativeSmokeStage, error: unknown): SmokeStepError => + error instanceof SmokeStepError ? error : new SmokeStepError({ code: errorCode(error), stage }); + +type CodexSmokeExecutor = ( + stage: CodexNativeSmokeStage, + command: CodexNativeSmokeProcessCommand, +) => Promise; + +const createCodexSmokeExecutor = (runner: CodexNativeSmokeCommandRunner): CodexSmokeExecutor => + async (stage, command) => { + try { + const commandResult = await runner(command); + if (commandResult.failure !== undefined) { + throw new SmokeStepError({ failure: commandResult.failure, stage }); + } + return commandResult; + } catch (error) { + throw failedStep(stage, error); + } + }; + +interface CodexSmokeStaging { + readonly candidate: string; + readonly fixture: string; + readonly home: string; + readonly root: string; +} + +/** Events parsed so far; kept outside the phases so a failing phase still reports them. */ +interface CodexSmokeEvidence { + events: readonly RedactedEventEnvelope[]; +} + +/** + * The smoke root is not a `withTempDirectory` bracket: its removal is + * injectable (`cleanupTemporaryRoot`), and a removal failure is reported in + * the result (`cleanup.status`), not thrown — `removeCodexSmokeRoot` owns it. + */ +const createCodexSmokeRoot = async (temporaryDirectoryParent: string): Promise => { + try { + return await runWithPlatform(Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(temporaryDirectoryParent, { recursive: true }); + return yield* fs.makeTempDirectory({ directory: temporaryDirectoryParent, prefix: 'agent-bundle-codex-smoke-' }); + })); + } catch (error) { + throw failedStep('temp-home', error); + } +}; + +const codexSmokeStagingFor = (root: string): CodexSmokeStaging => Object.freeze({ + candidate: join(root, 'candidate'), + fixture: join(root, 'fixture'), + home: join(root, 'home'), + root, +}); + +const stageCodexSmokeInputs = async ( + options: CodexNativeSmokeOptions, + staging: CodexSmokeStaging, +): Promise => { + try { + await runWithPlatform(Effect.flatMap(FileSystem.FileSystem, (fs) => fs.makeDirectory(staging.home, { recursive: true }))); + } catch (error) { + throw failedStep('temp-home', error); + } + try { + // `lstat` stays raw: a candidate that is a dangling symlink must fail as `candidate`, not inside the copy. + await lstat(options.candidateDirectory); + await runWithPlatform(Effect.flatMap(FileSystem.FileSystem, (fs) => fs.copy(options.candidateDirectory, staging.candidate, { overwrite: true }))); + } catch (error) { + throw failedStep('candidate', error); + } + try { + await runWithPlatform(Effect.flatMap(FileSystem.FileSystem, (fs) => fs.copy(options.fixtureDirectory, staging.fixture, { overwrite: true }))); + await (options.initializeFixture ?? initializeCodexSmokeFixture)(staging.fixture); + } catch (error) { + throw failedStep('fixture', error); + } +}; + +const runCodexVersionPreflight = async ( + execute: CodexSmokeExecutor, + cwd: string, + environment: NodeJS.ProcessEnv, + limits: CodexNativeSmokeProcessLimits, +): Promise => { + const version = await execute('version', { args: ['--version'], cwd, environment, limits }); + if (version.exitCode !== 0) throw new SmokeStepError({ output: `${version.stdout}\n${version.stderr}`, stage: 'version' }); + if (!isCompatibleVersion(version.stdout)) throw new SmokeStepError({ stage: 'version', version: version.stdout }); +}; + +const adoptCodexSmokeAuth = async (normalCodexHome: string, temporaryHome: string): Promise => { + try { + await copyOpaqueCodexAuthState(join(normalCodexHome, 'auth.json'), join(temporaryHome, 'auth.json')); + } catch (error) { + throw failedStep('auth', error); + } +}; + +const executeCodexSmokePlan = async ( + execute: CodexSmokeExecutor, + staging: CodexSmokeStaging, + environment: NodeJS.ProcessEnv, + limits: CodexNativeSmokeProcessLimits, + evidence: CodexSmokeEvidence, +): Promise => { + const commands = createCodexNativeSmokePlan({ + candidateDirectory: staging.candidate, + fixtureDirectory: staging.fixture, + }); + let pluginAvailability: CodexNativeSmokeResult['activation']['pluginAvailability'] = 'unavailable'; + let automatic: CodexNativeSmokeResult['activation']['automatic'] = 'unavailable'; + for (const command of commands) { + const commandResult = await execute(command.id, { + args: command.args, + cwd: staging.fixture, + environment, + limits, + }); + if (command.id === 'plugin.list' && !outputContainsPlugin(commandResult.stdout)) { + throw new SmokeStepError({ stage: command.id }); + } + if (command.id === 'plugin.list') pluginAvailability = 'observed'; + if (command.id === 'exec') { + try { + evidence.events = normalizeCodexNativeSmokeEvents(commandResult.stdout); + } catch { + throw new SmokeStepError({ stage: command.id }); + } + if (commandResult.stdout.includes(smokeSkillSentinel)) automatic = 'inferred'; + } + if (commandResult.exitCode !== 0) { + throw new SmokeStepError({ output: `${commandResult.stdout}\n${commandResult.stderr}`, stage: command.id }); + } + } + return Object.freeze({ automatic, pluginAvailability }); +}; + +const removeCodexSmokeRoot = async ( + options: CodexNativeSmokeOptions, + root: string, + result: CodexNativeSmokeResult, +): Promise => { + try { + await (options.cleanupTemporaryRoot ?? ((temporaryRoot: string) => runWithPlatform( + Effect.flatMap(FileSystem.FileSystem, (fs) => fs.remove(temporaryRoot, { force: true, recursive: true })), + )))(root); + return result; + } catch { + if (result.status === 'passed') { + return Object.freeze({ + ...result, + cleanup: Object.freeze({ status: 'failed' as const }), + diagnostic: Object.freeze({ code: 'native-codex.cleanup.failed', kind: 'harness-failure' as const }), + status: 'harness-failure', + }); + } + return Object.freeze({ ...result, cleanup: Object.freeze({ status: 'failed' as const }) }); + } +}; + +export const runCodexNativeSmoke = async (options: CodexNativeSmokeOptions): Promise => { + const environment = options.environment ?? process.env; + if (!nativeCodexSmokeEnabled(environment)) return skippedCodexNativeSmokeResult; + + const normalCodexHome = options.normalCodexHome ?? environment.CODEX_HOME ?? join(homedir(), '.codex'); + const temporaryDirectoryParent = options.temporaryDirectoryParent ?? tmpdir(); + const execute = createCodexSmokeExecutor(options.run ?? defaultCodexRunner); + const limits = resolveProcessLimits(options.processLimits); + const evidence: CodexSmokeEvidence = { events: Object.freeze([]) }; + let before: CodexStateSnapshot | undefined; + let after: CodexStateSnapshot | undefined; + let root: string | undefined; + let result: CodexNativeSmokeResult; + + try { + try { + before = await snapshotCodexState(normalCodexHome); + } catch (error) { + throw failedStep('normal-home', error); + } + root = await createCodexSmokeRoot(temporaryDirectoryParent); + const staging = codexSmokeStagingFor(root); + await stageCodexSmokeInputs(options, staging); + const childEnvironment = Object.freeze({ + ...withoutProviderApiKeys(environment), + CODEX_HOME: staging.home, + }); + await runCodexVersionPreflight(execute, staging.fixture, childEnvironment, limits); + await adoptCodexSmokeAuth(normalCodexHome, staging.home); + const activation = await executeCodexSmokePlan(execute, staging, childEnvironment, limits, evidence); + + try { + after = await snapshotCodexState(normalCodexHome); + } catch (error) { + throw failedStep('normal-home', error); + } + if (!sameDigestSnapshot(before, after)) { + result = failedResult(Object.freeze({ code: 'native-codex.normal-home.changed', kind: 'harness-failure' }), before, after, evidence.events); + } else { + result = Object.freeze({ + activation, + eventEnvelopes: evidence.events, + normalHome: normalHomeResult(before, after), + status: 'passed', + }); + } + } catch (error) { + if (before !== undefined) { + try { + after = await snapshotCodexState(normalCodexHome); + } catch { + // The primary structured failure remains authoritative. + } + } + const input = error instanceof SmokeStepError ? error : new SmokeStepError({ stage: 'exec' }); + result = failedResult(classifyCodexNativeSmokeFailure(input), before, after, evidence.events); + } + + return root === undefined ? result : removeCodexSmokeRoot(options, root, result); +}; + +export const codexNativeSmokeReportPath = (repositoryRoot: string): string => + join(repositoryRoot, '.agent-bundle', 'w2-codex-native-contract-evidence.json');