diff --git a/src/harness-identity.test.ts b/src/harness-identity.test.ts new file mode 100644 index 0000000..b639cba --- /dev/null +++ b/src/harness-identity.test.ts @@ -0,0 +1,114 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { HARNESS_PACKAGES } from "./harness.js"; +import { HARNESS_RECORD_FLAG, harnessIdentity, harnessRecordEnabled } from "./harness-identity.js"; +import { enqueueRun } from "./runtime.js"; +import type { ShipRuntime } from "./runtime.js"; + +type Started = { type: string; at?: string; data: Record }; + +function captureRuntime(): { runtime: ShipRuntime; started: Started[] } { + const started: Started[] = []; + const runtime = { + kind: "file", + evidence: { forRepo: async () => null }, + projects: { list: async () => [], forRepo: async () => null }, + governance: { get: async () => ({ authority: {}, windows: {}, reviewers: [] }) }, + store: { + append: async (_runId: string, event: Started) => { + if (event.type === "run-started") started.push(event); + }, + }, + saveMeta: async () => {}, + } as unknown as ShipRuntime; + return { runtime, started }; +} + +async function withEnv(values: Record, fn: () => Promise): Promise { + const saved: Record = {}; + for (const [k, v] of Object.entries(values)) { + saved[k] = process.env[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + try { + return await fn(); + } finally { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +} + +const QUIET = { SHIP_TESTS: undefined, SHIP_TELEMETRY: undefined, SHIP_PREVIEW: undefined, SHIP_TEST_DETECT: "0", SHIP_HARNESS_ATTEMPTS: undefined }; + +async function enqueue(env: Record, runId = "run-id") { + return withEnv({ ...QUIET, ...env }, async () => { + const { runtime, started } = captureRuntime(); + await enqueueRun(runtime, { runId, task: "t", model: "ship-model", repo: "tyler/a" }); + const event = started[0]!; + delete event.at; // wall clock + return event; + }); +} + +test("the flag is on only for an explicit on/1/true", () => { + assert.equal(harnessRecordEnabled({}), false); + for (const v of ["", "0", "off", "no", "yes", "enabled"]) assert.equal(harnessRecordEnabled({ [HARNESS_RECORD_FLAG]: v }), false, v); + for (const v of ["on", "ON", "1", "true", " on "]) assert.equal(harnessRecordEnabled({ [HARNESS_RECORD_FLAG]: v }), true, v); +}); + +test("identity: native records Ship's model and no vendor revision", () => { + const id = harnessIdentity({ harness: { id: "native", version: "1" }, shipModel: "ship-model", env: {} }); + assert.deepEqual(id.harness, { id: "native", adapterVersion: "1" }); + assert.deepEqual(id.model, { ship: "ship-model", harness: null }); + assert.equal(id.expectedRevision, null); + assert.deepEqual(id.configuration.forwardedEnvNames, []); +}); + +test("identity: an external harness records the harness model, expected revision and configuration, names only", () => { + const env = { SHIP_HARNESS_MODEL: "sonnet", SHIP_HARNESS_TIMEOUT_MS: "60000", ANTHROPIC_API_KEY: "sk-ant-never-recorded" }; + const id = harnessIdentity({ harness: { id: "claude-code", version: "1" }, shipModel: "ship-model", env }); + assert.equal(id.model.harness, "sonnet"); + assert.deepEqual(id.expectedRevision, { npm: HARNESS_PACKAGES["claude-code"]!.npm, version: HARNESS_PACKAGES["claude-code"]!.version }); + assert.equal(id.configuration.timeoutMs, 60000); + assert.ok(id.configuration.forwardedEnvNames.includes("ANTHROPIC_API_KEY")); + assert.ok(!JSON.stringify(id).includes("sk-ant-never-recorded"), "a credential value is never part of the identity"); + // The digest moves with the configuration and with nothing else. + const same = harnessIdentity({ harness: { id: "claude-code", version: "1" }, shipModel: "other-ship-model", env: { ...env, ANTHROPIC_API_KEY: "different" } }); + assert.equal(same.configuration.digest, id.configuration.digest); + assert.notEqual(harnessIdentity({ harness: { id: "claude-code", version: "1" }, shipModel: "m", env: { ...env, SHIP_HARNESS_MODEL: "opus" } }).configuration.digest, id.configuration.digest); + assert.notEqual(harnessIdentity({ harness: { id: "claude-code", version: "1" }, shipModel: "m", env: { ...env, SHIP_CLAUDE_BARE: "1" } }).configuration.digest, id.configuration.digest); +}); + +test("identity: multi-attempt runs record every harness they may execute under", () => { + const id = harnessIdentity({ + harness: { id: "native", version: "1" }, + attempts: [{ id: "claude-code", version: "1" }, { id: "opencode", version: "1" }], + shipModel: "m", + env: {}, + }); + assert.deepEqual(id.attempts?.map((a) => a.id), ["claude-code", "opencode"]); +}); + +test("enqueue, flag off: the recorded run-started event carries no identity (default behaviour unchanged)", async () => { + const off = await enqueue({ [HARNESS_RECORD_FLAG]: undefined }); + assert.ok(!("harnessIdentity" in off.data)); + assert.deepEqual(Object.keys(off.data).sort(), ["input", "stepFingerprint", "taskRootRunId", "workflow"]); + for (const v of ["off", "0", ""]) assert.equal(JSON.stringify(await enqueue({ [HARNESS_RECORD_FLAG]: v })), JSON.stringify(off), `flag=${v}`); +}); + +test("enqueue, flag on: identity is added beside the input and nothing else in the event changes", async () => { + const off = await enqueue({ [HARNESS_RECORD_FLAG]: undefined, SHIP_HARNESS: "claude-code", SHIP_HARNESS_MODEL: "sonnet" }); + const on = await enqueue({ [HARNESS_RECORD_FLAG]: "on", SHIP_HARNESS: "claude-code", SHIP_HARNESS_MODEL: "sonnet" }); + const { harnessIdentity: recorded, ...rest } = on.data as { harnessIdentity: ReturnType } & Record; + // Byte-identical once the one additive key is set aside: same input, same + // fingerprint, so no step sequence and no stored shape moved. + assert.equal(JSON.stringify({ ...on, data: rest }), JSON.stringify(off)); + assert.equal(recorded.harness.id, "claude-code"); + assert.equal(recorded.model.ship, "ship-model"); + assert.equal(recorded.model.harness, "sonnet"); + assert.equal(recorded.expectedRevision?.version, HARNESS_PACKAGES["claude-code"]!.version); +}); diff --git a/src/harness-identity.ts b/src/harness-identity.ts new file mode 100644 index 0000000..4c0e691 --- /dev/null +++ b/src/harness-identity.ts @@ -0,0 +1,79 @@ +import { createHash } from "node:crypto"; + +import { HARNESS_PACKAGES } from "./harness.js"; +import type { HarnessRef } from "./harness.js"; +import { externalHarnessConfig } from "./harness-external.js"; + +/** + * Which model, harness, harness revision and configuration wrote a run (S13). + * + * Today the run input says `harness: {id, version}` where `version` is the + * ADAPTER contract version ("1"), not the vendor binary, and nothing says what + * configuration the harness ran under. Two runs that differ only in + * SHIP_HARNESS_MODEL or the forwarded credential names look identical in the + * log. This records the missing identity. + * + * Where it lives: a SIBLING of `input` on the run-started event, next to + * `stepFingerprint` (step-fingerprint.ts), never a field inside `input`. The + * recorded input is what gates step presence and what the fingerprint hashes; + * a key added there would be a stored-shape change. The engine reads `workflow` + * and `input` only, so a sibling key is inert to replay. + * + * Off by default (SHIP_HARNESS_RECORD=on): off writes nothing, so the event is + * byte-identical to one written before this existed. + * + * What is NOT recorded here: the binary revision the sandbox ACTUALLY ran. That + * is only knowable inside the sandbox, and the `harness-preflight` step already + * records it (version, expected, pinned). The page reads it from there; this + * record carries the revision Ship EXPECTED to run, so the two can be compared. + * Credential VALUES are never recorded, only the names that were configured. + */ + +export const HARNESS_RECORD_FLAG = "SHIP_HARNESS_RECORD"; + +/** On only for an explicit "on" (or 1/true). Anything else, including unset, is off. */ +export function harnessRecordEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return /^(on|1|true)$/i.test((env[HARNESS_RECORD_FLAG] ?? "").trim()); +} + +export interface HarnessIdentity { + v: 1; + harness: { id: string; adapterVersion: string }; + /** Ship's own model (native loop) and the model passed to the harness, if any. */ + model: { ship: string; harness: string | null }; + /** The vendor package Ship expected the image to carry; null for native. */ + expectedRevision: { npm: string; version: string } | null; + /** The configuration the harness runs under: names and numbers, never secret values. */ + configuration: { timeoutMs: number | null; claudeBare: boolean | null; forwardedEnvNames: string[]; digest: string }; + /** Other harnesses a multi-attempt run may execute under. */ + attempts?: { id: string; adapterVersion: string }[]; +} + +function digest(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16); +} + +export function harnessIdentity(opts: { + harness: HarnessRef; + attempts?: readonly HarnessRef[]; + shipModel: string; + env?: NodeJS.ProcessEnv; +}): HarnessIdentity { + const env = opts.env ?? process.env; + const pkg = HARNESS_PACKAGES[opts.harness.id]; + const external = pkg !== undefined; + const config = external ? externalHarnessConfig(opts.harness.id, env) : undefined; + const fields = { + timeoutMs: config?.timeoutMs ?? null, + claudeBare: config?.claudeBare ?? null, + forwardedEnvNames: config?.forward ?? [], + }; + return { + v: 1, + harness: { id: opts.harness.id, adapterVersion: opts.harness.version }, + model: { ship: opts.shipModel, harness: config?.model ?? null }, + expectedRevision: external ? { npm: pkg.npm, version: pkg.version } : null, + configuration: { ...fields, digest: digest({ id: opts.harness.id, model: config?.model ?? null, ...fields }) }, + ...((opts.attempts?.length ?? 0) >= 2 ? { attempts: opts.attempts!.map((a) => ({ id: a.id, adapterVersion: a.version })) } : {}), + }; +} diff --git a/src/harness-probes.test.ts b/src/harness-probes.test.ts new file mode 100644 index 0000000..ee51a6e --- /dev/null +++ b/src/harness-probes.test.ts @@ -0,0 +1,136 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { LocalExecutor } from "@neutron-build/agents"; +import { MemoryEventStore, executeRun } from "@neutron-build/workflow"; + +import { durableAgent } from "./durable.js"; +import type { DurableAgentOutput } from "./durable.js"; +import { HARNESS_PACKAGES } from "./harness.js"; +import { STEER_UNSUPPORTED_MESSAGE, declarationFor } from "./harness-capabilities.js"; +import type { HarnessDeclaration } from "./harness-capabilities.js"; +import { externalAdapters } from "./harness-external.js"; +import { PROBED_OPERATIONS, credentialResidue, fakeExternalHarness, probeHarness, probeOperation } from "./harness-probes.js"; + +const IDS = ["native", "claude-code", "opencode"] as const; + +test("S13 probes: every declared harness conforms to its own declaration under real adapter probes", async () => { + for (const id of IDS) { + const report = await probeHarness(id); + assert.deepEqual(report.results.map((r) => r.operation), [...PROBED_OPERATIONS]); + assert.equal(report.ok, true, `${id}: ${JSON.stringify(report.results.flatMap((r) => r.mismatches))}`); + // The rest were not driven. That must be visible, not folded into "ok". + assert.deepEqual([...report.unprobed].sort(), ["approval-park", "browser", "interrupt", "recovery", "use-tools"]); + } +}); + +test("S13 probes: observations are what the adapters really did, not what the table says", async () => { + // Native honours all three: parks on the plan, reads the note, frames the scan. + const n = (op: Parameters[1]) => probeOperation("native", op); + assert.deepEqual(await n("plan-review"), { operation: "plan-review", accepted: true, sideEffects: true, honoured: true }); + assert.equal((await n("steer")).honoured, true); + assert.equal((await n("investigate")).honoured, true); + + for (const id of ["claude-code", "opencode"] as const) { + // Plan review: the adapter throws before putFile/exec, so no agent starts and nothing is edited. + const plan = await probeOperation(id, "plan-review"); + assert.equal(plan.accepted, false); + assert.equal(plan.sideEffects, false, "a refused plan review must not have launched the agent"); + assert.match(plan.refusalMessage ?? "", /Plan review requires the native harness\. No external agent was started/); + // Steer: refused at the gate with the real message, nothing stored. + assert.deepEqual(await probeOperation(id, "steer"), { operation: "steer", accepted: false, sideEffects: false, refusalMessage: STEER_UNSUPPORTED_MESSAGE }); + // Scan: the vendor agent really got the read-only framing. + assert.deepEqual(await probeOperation(id, "investigate"), { operation: "investigate", accepted: true, sideEffects: true, honoured: true }); + } +}); + +test("S13 probes: an adapter that accepts a steering note and drops it is caught, whichever way the table is wrong", async () => { + // The gate lets the note through (a table that claims support). The external + // adapter reads only its starting prompt, so the note never reaches the binary. + const dropped = await probeOperation("claude-code", "steer", { admitSteer: () => true }); + assert.equal(dropped.accepted, true); + assert.equal(dropped.honoured, false, "the note text must not appear in what the harness received"); + + // Against the published declaration (steering refused): accepted a refused op. + const real = await probeHarness("claude-code", { admitSteer: () => true }); + assert.deepEqual(real.results.find((r) => r.operation === "steer")!.mismatches.map((m) => m.kind), ["accepted-unsupported"]); + + // Against a declaration that CLAIMS steering is supported: the silent drop. + const claim: HarnessDeclaration = structuredClone(declarationFor("claude-code")!); + claim.capabilities.steering = { status: "supported", reason: "claimed, not true" }; + const lying = await probeHarness("claude-code", { declaration: claim, admitSteer: () => true }); + assert.equal(lying.ok, false); + assert.deepEqual(lying.results.find((r) => r.operation === "steer")!.mismatches.map((m) => m.kind), ["silently-dropped"]); + + // Negative control: the same override on the harness that DOES read notes is clean. + assert.equal((await probeHarness("native", { admitSteer: () => true })).ok, true); + + // And the same for opencode, so the finding is not specific to one adapter. + const oc: HarnessDeclaration = structuredClone(declarationFor("opencode")!); + oc.capabilities.steering = { status: "supported", reason: "claimed, not true" }; + const ocReport = await probeHarness("opencode", { declaration: oc, admitSteer: () => true }); + assert.deepEqual(ocReport.results.find((r) => r.operation === "steer")!.mismatches.map((m) => m.kind), ["silently-dropped"]); +}); + +test("S13 probes: a table that claims plan review for an external harness is caught as a refusal of a supported op", async () => { + const claim: HarnessDeclaration = structuredClone(declarationFor("opencode")!); + claim.capabilities.planning = { status: "supported", reason: "claimed, not true" }; + const report = await probeHarness("opencode", { declaration: claim }); + assert.deepEqual(report.results.find((r) => r.operation === "plan-review")!.mismatches.map((m) => m.kind), ["refused-supported"]); +}); + +// --- credential lifecycle ------------------------------------------------- + +const SECRET = "sk-ant-oat-probe-SECRET-1234567890"; + +async function runWithCredential(opts: { extraScript?: string[]; exitCode?: number }) { + const rig = await fakeExternalHarness("claude-code", opts); + const wf = durableAgent({ + model: { provider: "never", modelId: "never", async doGenerate() { throw new Error("no model"); }, async *doStream() { throw new Error("unused"); } }, + executor: rig.provider, + workdir: ".", + harnesses: externalAdapters({ env: { CLAUDE_CODE_OAUTH_TOKEN: SECRET } }), + }); + const store = new MemoryEventStore(); + const outcome = await executeRun({ workflow: wf, runId: "cred", store, input: { task: "t", harness: { id: "claude-code", version: "1" } } }); + return { rig, out: outcome.output as DurableAgentOutput, events: await store.load("cred") }; +} + +test("credential lifecycle: a credential reaches the harness for the run and is gone from the sandbox after exit", async () => { + const { rig, out, events } = await runWithCredential({}); + assert.equal(out.status, "finished"); + // It was delivered: the binary saw it in its environment... + assert.match(await rig.seenEnv(), new RegExp(`CLAUDE_CODE_OAUTH_TOKEN=${SECRET}`)); + // ...but the file it travelled in was already removed before the binary started, + // so the agent could not read it back off disk. + assert.doesNotMatch(await rig.seenAgentDir(), /harness\.env/); + // After exit: no file in the sandbox holds it, and the executor's own env never did. + assert.deepEqual(await credentialResidue(rig.root, rig.executor, [SECRET]), []); + // The scan does look at files the run wrote: the prompt file is removed too. + assert.deepEqual(await credentialResidue(rig.root, rig.executor, ["unattended inside a sandbox"]), []); + // And the durable log carries names, never the value. + assert.ok(!JSON.stringify(events).includes(SECRET)); +}); + +test("credential lifecycle: a harness that crashes still leaves no credential file behind", async () => { + const { rig, out } = await runWithCredential({ exitCode: 9 }); + assert.equal(out.status, "error"); + assert.deepEqual(await credentialResidue(rig.root, rig.executor, [SECRET]), []); +}); + +test("credential lifecycle negative control: the residue scan finds a credential a misbehaving harness persists", async () => { + // A harness that writes its environment into the tree is a leak the adapter + // cannot prevent; the check must be able to see it or its clean result means nothing. + const { rig } = await runWithCredential({ extraScript: [`env > leaked-env.txt`] }); + assert.deepEqual(await credentialResidue(rig.root, rig.executor, [SECRET]), ["leaked-env.txt"]); +}); + +test("credential lifecycle negative control: a credential in the executor's own environment is reported", async () => { + const rig = await fakeExternalHarness("claude-code"); + const leaky = new LocalExecutor({ root: rig.root, env: { PATH: process.env.PATH ?? "", CLAUDE_CODE_OAUTH_TOKEN: SECRET } }); + assert.deepEqual(await credentialResidue(rig.root, leaky, [SECRET]), ["(executor environment)"]); +}); + +test("declared harness packages exist for every external probe id", () => { + for (const id of ["claude-code", "opencode"]) assert.ok(HARNESS_PACKAGES[id]); +}); diff --git a/src/harness-probes.ts b/src/harness-probes.ts new file mode 100644 index 0000000..b0c8bd4 --- /dev/null +++ b/src/harness-probes.ts @@ -0,0 +1,271 @@ +import { chmod, mkdir, mkdtemp, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { AdapterGenerateResult, ModelAdapter } from "@neutron-build/ai"; +import { LocalExecutor } from "@neutron-build/agents"; +import type { AgentExecutor } from "@neutron-build/agents"; +import { MemoryEventStore, executeRun } from "@neutron-build/workflow"; + +import { PLAN_EVENT, durableAgent } from "./durable.js"; +import type { DurableAgentOutput, ExecutorProvider } from "./durable.js"; +import { HARNESS_PACKAGES, HARNESS_VERSIONS } from "./harness.js"; +import { + OPERATION_CAPABILITY, STEER_UNSUPPORTED_MESSAGE, conformanceCheck, declarationFor, harnessSupports, +} from "./harness-capabilities.js"; +import type { ConformanceMismatch, HarnessDeclaration, HarnessOperation, ObservedBehaviour } from "./harness-capabilities.js"; +import { externalAdapters, shq } from "./harness-external.js"; + +/** + * Real adapter probes for `conformanceCheck` (S13). + * + * `conformanceCheck` is pure: it compares a declaration with an observation + * somebody hands it. Until now nobody did, so the declaration table was a claim + * checked only against itself. These probes DRIVE each adapter through the + * boundary its own tests already use and report what happened: + * + * native durableAgent + a scripted ModelAdapter + LocalExecutor + * claude-code durableAgent + the real externalAdapters() + a fake `claude` + * opencode durableAgent + the real externalAdapters() + a fake `opencode` + * + * The fake binary is the only stand-in: the adapter code, the executor + * abstraction, the command line, the prompt file and the env file are the + * production ones. No live harness and no model is run. What a probe can tell + * you is what Ship's side did with the operation; it cannot tell you what the + * real vendor binary would do with the same arguments. + * + * Only three operations are probed: plan-review, steer and investigate. The + * rest (approvals, recovery, browser, interrupt, tools) have no accept/refuse + * point to drive from here, and a probe that invented one would be a pass that + * means nothing. They are listed as `unprobed`, which is not the same as ok. + */ + +export const PROBED_OPERATIONS: readonly HarnessOperation[] = ["plan-review", "steer", "investigate"]; + +type ProbedId = "native" | "claude-code" | "opencode"; + +export interface ProbeOptions { + /** + * The steer admission gate. Default: the same `harnessSupports` the run + * page's steer route calls before it stores a note (web/src/routes/runs/[id].tsx). + * A test that wants to know what an adapter does WHEN the gate lets a note + * through (a table that wrongly claims support) overrides it. + */ + admitSteer?: (harnessId: string) => boolean; +} + +/** One scripted ModelAdapter for the native loop that records everything it was shown. */ +function scriptedNativeModel(plan: string): { model: ModelAdapter; seen: () => string; calls: () => number } { + const shown: string[] = []; + let calls = 0; + const model: ModelAdapter = { + provider: "probe", + modelId: "probe-1", + async doGenerate(options): Promise { + calls++; + for (const m of options.messages) shown.push(typeof m.content === "string" ? m.content : JSON.stringify(m.content)); + // With a plan configured the first call is the plan; every other turn finishes. + const text = calls === 1 && plan !== "" ? plan : "```finish\nNo findings.\n```"; + return { content: [{ type: "text", text }], finishReason: "stop", usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, raw: null }; + }, + async *doStream() { + throw new Error("unused"); + }, + }; + return { model, seen: () => shown.join("\n"), calls: () => calls }; +} + +export interface FakeExternalRig { + provider: ExecutorProvider; + executor: LocalExecutor; + /** The sandbox root the executor runs in. */ + root: string; + runs(): Promise; + /** Everything the binary was invoked with (one argument per line). */ + args(): Promise; + /** The binary's environment as it saw it. */ + seenEnv(): Promise; + /** The binary's view of .teploy-agent/ at the moment it started. */ + seenAgentDir(): Promise; + edited(): Promise; +} + +const CLAUDE_STREAM = JSON.stringify({ type: "result", subtype: "success", is_error: false, num_turns: 1, result: "done", usage: { input_tokens: 1, output_tokens: 1 } }); +const OPENCODE_STREAM = [ + JSON.stringify({ type: "text", part: { type: "text", text: "done" } }), + JSON.stringify({ type: "step_finish", part: { tokens: { input: 1, output: 1, reasoning: 0, cache: { read: 0, write: 0 } }, cost: 0 } }), +].join("\n"); + +/** + * A fake vendor binary on the sandbox PATH. It records how it was invoked, + * what environment and `.teploy-agent/` directory it saw, leaves an edit in + * the tree, and prints a minimal valid stream. `extraScript` lines run before + * the stream (a harness that misbehaves, for the negative controls). + */ +export async function fakeExternalHarness( + id: "claude-code" | "opencode", + opts: { extraScript?: string[]; exitCode?: number } = {}, +): Promise { + const bin = await mkdtemp(join(tmpdir(), "probe-bin-")); + const root = await mkdtemp(join(tmpdir(), "probe-work-")); + const rec = await mkdtemp(join(tmpdir(), "probe-rec-")); + const binary = HARNESS_PACKAGES[id]!.binary; + const stream = id === "claude-code" ? CLAUDE_STREAM : OPENCODE_STREAM; + await mkdir(bin, { recursive: true }); + const script = [ + "#!/bin/sh", + `if [ "$1" = "--version" ]; then echo "${HARNESS_PACKAGES[id]!.version} (probe)"; exit 0; fi`, + `echo run >> ${shq(join(rec, "runs"))}`, + `printf '%s\\n' "$@" > ${shq(join(rec, "args"))}`, + `env > ${shq(join(rec, "env"))}`, + `ls -a .teploy-agent > ${shq(join(rec, "agentdir"))} 2>&1`, + `echo edited > edited.txt`, + ...(opts.extraScript ?? []), + // A crash prints no result stream: with one, the adapter correctly trusts it. + ...(opts.exitCode !== undefined ? [`echo "boom" >&2`, `exit ${opts.exitCode}`] : []), + `cat <<'EOF'\n${stream}\nEOF`, + ].join("\n"); + await writeFile(join(bin, binary), `${script}\n`); + await chmod(join(bin, binary), 0o755); + const executor = new LocalExecutor({ root, env: { PATH: `${bin}:${process.env.PATH ?? ""}` } }); + const read = (name: string) => readFile(join(rec, name), "utf8").catch(() => ""); + return { + provider: { isolated: true, async create() { return { handle: root }; }, attach: () => executor }, + executor, + root, + runs: async () => (await read("runs")).split("\n").filter((l) => l === "run").length, + args: () => read("args"), + seenEnv: () => read("env"), + seenAgentDir: () => read("agentdir"), + edited: async () => (await stat(join(root, "edited.txt")).then(() => true, () => false)), + }; +} + +async function drive(wf: ReturnType, input: Record) { + const store = new MemoryEventStore(); + const outcome = await executeRun({ workflow: wf, runId: "probe", store, input: input as never }); + return { outcome, out: outcome.output as DurableAgentOutput | undefined, events: await store.load("probe") }; +} + +async function nativeRig(plan: string, steer?: { drain: () => Promise }) { + const root = await mkdtemp(join(tmpdir(), "probe-native-")); + const executor = new LocalExecutor({ root }); + const provider: ExecutorProvider = { async create() { return { handle: root }; }, attach: () => executor }; + const model = scriptedNativeModel(plan); + const wf = durableAgent({ model: model.model, executor: provider, workdir: ".", maxSteps: 6, ...(steer !== undefined ? { steer } : {}) }); + return { wf, model }; +} + +/** Drive one operation against one harness and report what was observed. */ +export async function probeOperation(id: ProbedId, operation: HarnessOperation, options: ProbeOptions = {}): Promise { + const harness = { id, version: HARNESS_VERSIONS[id]! }; + const external = id !== "native"; + const rig = external ? await fakeExternalHarness(id) : undefined; + const note = "probe-steer-note: prefer the smaller change"; + const store = { added: 0 }; + let started = false; + let honoured: boolean | undefined; + let message: string | undefined; + + const input: Record = { task: "probe task", ...(external ? { harness } : {}) }; + if (operation === "plan-review") input.plan = true; + else if (operation === "investigate") input.mode = "scan"; + else if (operation === "steer") { + const admit = (options.admitSteer ?? ((hid: string) => harnessSupports({ harness: { id: hid } }, "steer")))(id); + if (!admit) { + // Refused at the gate, before any note is stored and before anything runs. + return { operation, accepted: false, sideEffects: store.added > 0, refusalMessage: STEER_UNSUPPORTED_MESSAGE }; + } + store.added++; + input.steer = true; + } else { + throw new Error(`probeOperation: ${operation} has no probe; see PROBED_OPERATIONS`); + } + + const steer = operation === "steer" ? { drain: async () => [note] } : undefined; + const native = external ? undefined : await nativeRig(operation === "plan-review" ? "1. Write the file\n2. Verify it" : "", steer); + const wf = external + ? durableAgent({ + model: scriptedNativeModel("").model, + executor: rig!.provider, + workdir: ".", + harnesses: externalAdapters({ env: {} }), + maxSteps: 6, + ...(steer !== undefined ? { steer } : {}), + }) + : native!.wf; + const { outcome, out } = await drive(wf, input); + + if (external) started = (await rig!.runs()) > 0; + else started = native!.model.calls() > 0; + if (!started && out?.status === "error") message = out.summary; + + // What the harness actually RECEIVED, as opposed to what Ship was handed. + const received = external ? `${await rig!.args()}\n${await rig!.seenEnv()}` : native!.model.seen(); + if (operation === "plan-review") honoured = outcome.status === "waiting" && outcome.eventName === PLAN_EVENT; + if (operation === "investigate") honoured = /read-only/i.test(received); + if (operation === "steer") honoured = received.includes(note); + + const sideEffects = started || (external && (await rig!.edited())) || store.added > 0; + return { + operation, + accepted: started, + sideEffects, + ...(started ? { honoured: honoured === true } : {}), + ...(message !== undefined ? { refusalMessage: message } : {}), + }; +} + +export interface ProbeResult { + operation: HarnessOperation; + observed: ObservedBehaviour; + ok: boolean; + mismatches: ConformanceMismatch[]; +} + +export interface ProbeReport { + harness: string; + results: ProbeResult[]; + /** Operations nothing was driven for. Not a pass: nothing was observed. */ + unprobed: HarnessOperation[]; + ok: boolean; +} + +/** + * Probe every probed operation for a harness and judge each against + * `declaration` (default: the published one). Passing a different declaration + * is how a test asks "would this table have been caught?". + */ +export async function probeHarness(id: ProbedId, options: ProbeOptions & { declaration?: HarnessDeclaration } = {}): Promise { + const declaration = options.declaration ?? declarationFor(id); + if (declaration === undefined) throw new Error(`no declaration for harness "${id}"`); + const results: ProbeResult[] = []; + for (const operation of PROBED_OPERATIONS) { + const observed = await probeOperation(id, operation, options); + const verdict = conformanceCheck(declaration, observed); + results.push({ operation, observed, ok: verdict.ok, mismatches: verdict.mismatches }); + } + const unprobed = (Object.keys(OPERATION_CAPABILITY) as HarnessOperation[]).filter((op) => !PROBED_OPERATIONS.includes(op)); + return { harness: id, results, unprobed, ok: results.every((r) => r.ok) }; +} + +/** + * Where a credential was left behind after a harness run: a file under the + * sandbox root that contains it, or the executor's own environment. Empty + * means none of those places held it, nothing about places not looked at. + */ +export async function credentialResidue(root: string, executor: AgentExecutor, secrets: readonly string[]): Promise { + const found: string[] = []; + const hit = (text: string) => secrets.some((s) => s !== "" && text.includes(s)); + const walk = async (dir: string): Promise => { + for (const entry of await readdir(dir, { withFileTypes: true }).catch(() => [])) { + const path = join(dir, entry.name); + if (entry.isDirectory()) await walk(path); + else if (entry.isFile() && (await stat(path)).size < 2_000_000 && hit(await readFile(path, "utf8").catch(() => ""))) found.push(path.slice(root.length + 1)); + } + }; + await walk(root); + const env = await executor.exec("env", { timeoutMs: 10_000 }); + if (hit(env.stdout)) found.push("(executor environment)"); + return found; +} diff --git a/src/runtime.ts b/src/runtime.ts index 3434796..09531d4 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -41,6 +41,7 @@ import { FileRepoStatsStore, NucleusRepoStatsStore } from "./repo-stats.js"; import type { RepoStatsStore } from "./repo-stats.js"; import { attemptsCount, harnessAttempts, harnessRef } from "./harness.js"; import { PLAN_REVIEW_REFUSAL, refusalFor } from "./harness-capabilities.js"; +import { harnessIdentity, harnessRecordEnabled } from "./harness-identity.js"; import type { HarnessRef } from "./harness.js"; import type { EvidenceStore } from "./evidence.js"; import { FileProjectStore, NucleusProjectStore, ProjectEvidenceStore } from "./projects.js"; @@ -1465,6 +1466,10 @@ export async function enqueueRun( // The engine reads `workflow` and `input` and ignores everything else on // this event, so this key is inert to replay. stepFingerprint: stepFingerprint(input), + // Who ran it (S13): model, harness, expected revision, configuration + // identity. A sibling like the fingerprint, and only under + // SHIP_HARNESS_RECORD=on; off writes no key at all. + ...(harnessRecordEnabled() ? { harnessIdentity: harnessIdentity({ harness, attempts, shipModel: options.model }) } : {}), }, }; const meta: RunMeta = { diff --git a/web/src/lib/harness-identity-view.test.ts b/web/src/lib/harness-identity-view.test.ts new file mode 100644 index 0000000..44f4bb5 --- /dev/null +++ b/web/src/lib/harness-identity-view.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { harnessIdentityView } from "./harness-identity-view.js"; + +const started = (data: Record) => ({ type: "run-started", data: { workflow: "w", input: {}, ...data } }); +const preflight = (result: unknown, name = "harness-preflight") => ({ type: "step-completed", name, data: { result } }); +const identity = { + v: 1, + harness: { id: "claude-code", adapterVersion: "1" }, + model: { ship: "ship-model", harness: "sonnet" }, + expectedRevision: { npm: "@anthropic-ai/claude-code", version: "2.1.246" }, + configuration: { timeoutMs: 1_800_000, claudeBare: false, forwardedEnvNames: ["ANTHROPIC_API_KEY"], digest: "abcd1234abcd1234" }, +}; + +test("a run recorded without identity projects to undefined (page unchanged)", () => { + assert.equal(harnessIdentityView([started({})]), undefined); + assert.equal(harnessIdentityView([]), undefined); + assert.equal(harnessIdentityView([started({ harnessIdentity: "nope" }), preflight({ found: true, version: "2.1.246" })]), undefined); + assert.equal(harnessIdentityView([started({ harnessIdentity: {} })]), undefined); +}); + +test("recorded identity is shown with the revision the sandbox actually reported", () => { + const view = harnessIdentityView([started({ harnessIdentity: identity }), preflight({ found: true, version: "2.1.246 (Claude Code)" })])!; + const byLabel = Object.fromEntries(view.rows.map((r) => [r.label, r.value])); + assert.equal(byLabel["Harness"], "claude-code (adapter v1)"); + assert.equal(byLabel["Model"], "sonnet"); + assert.equal(byLabel["Expected revision"], "@anthropic-ai/claude-code@2.1.246"); + assert.equal(byLabel["Observed revision"], "2.1.246 (Claude Code)"); + assert.match(byLabel["Configuration"]!, /config abcd1234abcd1234 · timeout 30 min · forwards ANTHROPIC_API_KEY/); + assert.equal(view.revisionMismatch, false); +}); + +test("a binary that is not the expected revision, or is missing, is surfaced rather than smoothed over", () => { + const drift = harnessIdentityView([started({ harnessIdentity: identity }), preflight({ found: true, version: "2.0.0" })])!; + assert.equal(drift.revisionMismatch, true); + assert.ok(drift.rows.some((r) => r.label === "Observed revision" && /differs from expected/.test(r.value))); + const missing = harnessIdentityView([started({ harnessIdentity: identity }), preflight({ found: false, version: "" })])!; + assert.ok(missing.rows.some((r) => r.value === "binary not found in the sandbox")); + // No preflight at all is "not observed", never an implied match. + const none = harnessIdentityView([started({ harnessIdentity: identity })])!; + assert.ok(none.rows.some((r) => r.label === "Observed revision" && /not observed/.test(r.value))); + assert.equal(none.revisionMismatch, false); +}); + +test("native shows Ship's model and no vendor revision; attempt-prefixed preflights are read", () => { + const native = harnessIdentityView([started({ harnessIdentity: { ...identity, harness: { id: "native", adapterVersion: "1" }, expectedRevision: null } })])!; + assert.equal(native.rows.find((r) => r.label === "Model")!.value, "ship-model"); + assert.ok(!native.rows.some((r) => /revision/i.test(r.label))); + const attempt = harnessIdentityView([started({ harnessIdentity: identity }), preflight({ found: true, version: "2.1.246" }, "attempt-2-harness-preflight")])!; + assert.ok(attempt.rows.some((r) => r.label === "Observed revision" && r.value === "2.1.246")); +}); diff --git a/web/src/lib/harness-identity-view.ts b/web/src/lib/harness-identity-view.ts new file mode 100644 index 0000000..b8cc04b --- /dev/null +++ b/web/src/lib/harness-identity-view.ts @@ -0,0 +1,75 @@ +/** + * The run page's "who ran this" block (S13). A read-time projection of the + * `harnessIdentity` sibling the run-started event carries when the run was + * enqueued with SHIP_HARNESS_RECORD=on, joined with what the sandbox reported + * for the binary it actually ran (the `harness-preflight` step, recorded with + * or without the flag). A run made without the flag has no identity and + * projects to undefined, so its page is unchanged. + * + * Shape-checked field by field: this is JSON out of an event log, and a log + * written by another build is a normal thing to be reading. Display only. + */ +export interface IdentityLogEvent { + type: string; + name?: string; + data?: unknown; +} + +export interface HarnessIdentityView { + /** Label/value pairs, in display order. */ + rows: Array<{ label: string; value: string }>; + /** True when the binary the sandbox reported differs from the one Ship expected. */ + revisionMismatch: boolean; +} + +const str = (v: unknown, max = 120): string | undefined => (typeof v === "string" && v !== "" ? v.slice(0, max) : undefined); + +export function harnessIdentityView(events: readonly IdentityLogEvent[]): HarnessIdentityView | undefined { + const started = events.find((e) => e.type === "run-started"); + const id = (started?.data as { harnessIdentity?: Record } | undefined)?.harnessIdentity; + if (id === null || typeof id !== "object" || Array.isArray(id)) return undefined; + const harness = (id.harness ?? {}) as Record; + const model = (id.model ?? {}) as Record; + const expected = (id.expectedRevision ?? null) as Record | null; + const config = (id.configuration ?? {}) as Record; + const harnessId = str(harness.id); + if (harnessId === undefined) return undefined; + + const rows: HarnessIdentityView["rows"] = [{ label: "Harness", value: `${harnessId} (adapter v${str(harness.adapterVersion, 20) ?? "?"})` }]; + const shipModel = str(model.ship); + const harnessModel = str(model.harness); + rows.push({ label: "Model", value: harnessId === "native" ? (shipModel ?? "unknown") : (harnessModel ?? "the harness's own default") }); + + // Preflight results, one per attempt (steps are `harness-preflight` or `attempt-N-harness-preflight`). + const preflights = events + .filter((e) => e.type === "step-completed" && typeof e.name === "string" && /(^|-)harness-preflight$/.test(e.name)) + .map((e) => (e.data as { result?: Record } | undefined)?.result) + .filter((r): r is Record => r !== undefined && r !== null && typeof r === "object"); + let revisionMismatch = false; + if (expected !== null) { + const want = str(expected.version, 40); + rows.push({ label: "Expected revision", value: `${str(expected.npm, 60) ?? harnessId}@${want ?? "?"}` }); + for (const p of preflights) { + if (p.found !== true) { + rows.push({ label: "Observed revision", value: "binary not found in the sandbox" }); + continue; + } + const seen = str(p.version, 60) ?? "unreported"; + const bad = want !== undefined && !seen.includes(want); + if (bad) revisionMismatch = true; + rows.push({ label: "Observed revision", value: bad ? `${seen} (differs from expected)` : seen }); + } + if (preflights.length === 0) rows.push({ label: "Observed revision", value: "not observed (no preflight recorded)" }); + } + + const digest = str(config.digest, 32); + if (digest !== undefined) { + const names = Array.isArray(config.forwardedEnvNames) ? config.forwardedEnvNames.filter((n): n is string => typeof n === "string").slice(0, 12) : []; + const parts = [`config ${digest}`]; + if (typeof config.timeoutMs === "number") parts.push(`timeout ${Math.round(config.timeoutMs / 60_000)} min`); + if (config.claudeBare === true) parts.push("bare"); + if (names.length > 0) parts.push(`forwards ${names.join(", ")}`); + rows.push({ label: "Configuration", value: parts.join(" · ") }); + } + return { rows, revisionMismatch }; +} diff --git a/web/src/lib/run-data.server.ts b/web/src/lib/run-data.server.ts index da6b48a..1e1951d 100644 --- a/web/src/lib/run-data.server.ts +++ b/web/src/lib/run-data.server.ts @@ -34,6 +34,8 @@ import { taskRootRunId as resolveTaskRoot } from "../../../dist/task-session.js" import { taskRecord } from "teploy-ship/task-record"; import type { DeliveryOutcome } from "teploy-ship/task-record"; import { findingContinuityLines } from "./finding-continuity-view.js"; +import { harnessIdentityView } from "./harness-identity-view.js"; +import type { HarnessIdentityView } from "./harness-identity-view.js"; import { taskStateView } from "./task-state.js"; import type { TaskStateView } from "./task-state.js"; @@ -53,6 +55,8 @@ export interface RunData { testIntegrity: TestIntegrityPanel; /** Advisory comparison with the earlier review (S09); absent unless the run recorded one. */ findingContinuity?: string[]; + /** Model, harness, revision and configuration that wrote the run (S13); absent unless recorded. */ + harnessIdentity?: HarnessIdentityView; /** The run page's preview panel state, decided server-side (expiry uses the server clock). */ previewPanel: PreviewPanel; parentRunId?: string; @@ -371,6 +375,7 @@ export async function runData({ params, request }: { params: { id: string }; req findings: scanned.findings, findingsNotes: scanned.notes, ...(findingContinuityLines(events) !== undefined ? { findingContinuity: findingContinuityLines(events)! } : {}), + ...(harnessIdentityView(events) !== undefined ? { harnessIdentity: harnessIdentityView(events)! } : {}), isScan: (started?.data as { input?: { mode?: string } } | undefined)?.input?.mode === "scan", steerable, steerPending: steerNotes.map((n) => n.text), diff --git a/web/src/routes/runs/[id].tsx b/web/src/routes/runs/[id].tsx index 2ce8b0a..6ed4fe6 100644 --- a/web/src/routes/runs/[id].tsx +++ b/web/src/routes/runs/[id].tsx @@ -420,6 +420,14 @@ export default function RunDetail({ data: initialData }: { data: RunData }) { )} + {data.harnessIdentity !== undefined && ( +
+ Ran on {data.harnessIdentity.rows[0]?.value}{data.harnessIdentity.revisionMismatch ? " · binary differs from expected" : ""} +
+ {data.harnessIdentity.rows.map((r, i) =>
{r.label}:
{r.value}
)} +
+
+ )} {(data.outcome.pr !== undefined || data.outcome.usage !== undefined || data.outcome.repo !== undefined) && (