S13: real adapter probes for conformanceCheck; harness identity record (SHIP_HARNESS_RECORD)

src/harness-identity.test.ts

@@ -0,0 +1,114 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { HARNESS_PACKAGES } from "./harness.js";
+import { HARNESS_RECORD_FLAG, harnessIdentity, harnessRecordEnabled } from "./harness-identity.js";
+import { enqueueRun } from "./runtime.js";
+import type { ShipRuntime } from "./runtime.js";
+
+type Started = { type: string; at?: string; data: Record<string, unknown> };
+
+function captureRuntime(): { runtime: ShipRuntime; started: Started[] } {
+  const started: Started[] = [];
+  const runtime = {
+    kind: "file",
+    evidence: { forRepo: async () => null },
+    projects: { list: async () => [], forRepo: async () => null },
+    governance: { get: async () => ({ authority: {}, windows: {}, reviewers: [] }) },
+    store: {
+      append: async (_runId: string, event: Started) => {
+        if (event.type === "run-started") started.push(event);
+      },
+    },
+    saveMeta: async () => {},
+  } as unknown as ShipRuntime;
+  return { runtime, started };
+}
+
+async function withEnv<T>(values: Record<string, string | undefined>, fn: () => Promise<T>): Promise<T> {
+  const saved: Record<string, string | undefined> = {};
+  for (const [k, v] of Object.entries(values)) {
+    saved[k] = process.env[k];
+    if (v === undefined) delete process.env[k];
+    else process.env[k] = v;
+  }
+  try {
+    return await fn();
+  } finally {
+    for (const [k, v] of Object.entries(saved)) {
+      if (v === undefined) delete process.env[k];
+      else process.env[k] = v;
+    }
+  }
+}
+
+const QUIET = { SHIP_TESTS: undefined, SHIP_TELEMETRY: undefined, SHIP_PREVIEW: undefined, SHIP_TEST_DETECT: "0", SHIP_HARNESS_ATTEMPTS: undefined };
+
+async function enqueue(env: Record<string, string | undefined>, runId = "run-id") {
+  return withEnv({ ...QUIET, ...env }, async () => {
+    const { runtime, started } = captureRuntime();
+    await enqueueRun(runtime, { runId, task: "t", model: "ship-model", repo: "tyler/a" });
+    const event = started[0]!;
+    delete event.at; // wall clock
+    return event;
+  });
+}
+
+test("the flag is on only for an explicit on/1/true", () => {
+  assert.equal(harnessRecordEnabled({}), false);
+  for (const v of ["", "0", "off", "no", "yes", "enabled"]) assert.equal(harnessRecordEnabled({ [HARNESS_RECORD_FLAG]: v }), false, v);
+  for (const v of ["on", "ON", "1", "true", " on "]) assert.equal(harnessRecordEnabled({ [HARNESS_RECORD_FLAG]: v }), true, v);
+});
+
+test("identity: native records Ship's model and no vendor revision", () => {
+  const id = harnessIdentity({ harness: { id: "native", version: "1" }, shipModel: "ship-model", env: {} });
+  assert.deepEqual(id.harness, { id: "native", adapterVersion: "1" });
+  assert.deepEqual(id.model, { ship: "ship-model", harness: null });
+  assert.equal(id.expectedRevision, null);
+  assert.deepEqual(id.configuration.forwardedEnvNames, []);
+});
+
+test("identity: an external harness records the harness model, expected revision and configuration, names only", () => {
+  const env = { SHIP_HARNESS_MODEL: "sonnet", SHIP_HARNESS_TIMEOUT_MS: "60000", ANTHROPIC_API_KEY: "sk-ant-never-recorded" };
+  const id = harnessIdentity({ harness: { id: "claude-code", version: "1" }, shipModel: "ship-model", env });
+  assert.equal(id.model.harness, "sonnet");
+  assert.deepEqual(id.expectedRevision, { npm: HARNESS_PACKAGES["claude-code"]!.npm, version: HARNESS_PACKAGES["claude-code"]!.version });
+  assert.equal(id.configuration.timeoutMs, 60000);
+  assert.ok(id.configuration.forwardedEnvNames.includes("ANTHROPIC_API_KEY"));
+  assert.ok(!JSON.stringify(id).includes("sk-ant-never-recorded"), "a credential value is never part of the identity");
+  // The digest moves with the configuration and with nothing else.
+  const same = harnessIdentity({ harness: { id: "claude-code", version: "1" }, shipModel: "other-ship-model", env: { ...env, ANTHROPIC_API_KEY: "different" } });
+  assert.equal(same.configuration.digest, id.configuration.digest);
+  assert.notEqual(harnessIdentity({ harness: { id: "claude-code", version: "1" }, shipModel: "m", env: { ...env, SHIP_HARNESS_MODEL: "opus" } }).configuration.digest, id.configuration.digest);
+  assert.notEqual(harnessIdentity({ harness: { id: "claude-code", version: "1" }, shipModel: "m", env: { ...env, SHIP_CLAUDE_BARE: "1" } }).configuration.digest, id.configuration.digest);
+});
+
+test("identity: multi-attempt runs record every harness they may execute under", () => {
+  const id = harnessIdentity({
+    harness: { id: "native", version: "1" },
+    attempts: [{ id: "claude-code", version: "1" }, { id: "opencode", version: "1" }],
+    shipModel: "m",
+    env: {},
+  });
+  assert.deepEqual(id.attempts?.map((a) => a.id), ["claude-code", "opencode"]);
+});
+
+test("enqueue, flag off: the recorded run-started event carries no identity (default behaviour unchanged)", async () => {
+  const off = await enqueue({ [HARNESS_RECORD_FLAG]: undefined });
+  assert.ok(!("harnessIdentity" in off.data));
+  assert.deepEqual(Object.keys(off.data).sort(), ["input", "stepFingerprint", "taskRootRunId", "workflow"]);
+  for (const v of ["off", "0", ""]) assert.equal(JSON.stringify(await enqueue({ [HARNESS_RECORD_FLAG]: v })), JSON.stringify(off), `flag=${v}`);
+});
+
+test("enqueue, flag on: identity is added beside the input and nothing else in the event changes", async () => {
+  const off = await enqueue({ [HARNESS_RECORD_FLAG]: undefined, SHIP_HARNESS: "claude-code", SHIP_HARNESS_MODEL: "sonnet" });
+  const on = await enqueue({ [HARNESS_RECORD_FLAG]: "on", SHIP_HARNESS: "claude-code", SHIP_HARNESS_MODEL: "sonnet" });
+  const { harnessIdentity: recorded, ...rest } = on.data as { harnessIdentity: ReturnType<typeof harnessIdentity> } & Record<string, unknown>;
+  // Byte-identical once the one additive key is set aside: same input, same
+  // fingerprint, so no step sequence and no stored shape moved.
+  assert.equal(JSON.stringify({ ...on, data: rest }), JSON.stringify(off));
+  assert.equal(recorded.harness.id, "claude-code");
+  assert.equal(recorded.model.ship, "ship-model");
+  assert.equal(recorded.model.harness, "sonnet");
+  assert.equal(recorded.expectedRevision?.version, HARNESS_PACKAGES["claude-code"]!.version);
+});

src/harness-identity.ts

@@ -0,0 +1,79 @@
+import { createHash } from "node:crypto";
+
+import { HARNESS_PACKAGES } from "./harness.js";
+import type { HarnessRef } from "./harness.js";
+import { externalHarnessConfig } from "./harness-external.js";
+
+/**
+ * Which model, harness, harness revision and configuration wrote a run (S13).
+ *
+ * Today the run input says `harness: {id, version}` where `version` is the
+ * ADAPTER contract version ("1"), not the vendor binary, and nothing says what
+ * configuration the harness ran under. Two runs that differ only in
+ * SHIP_HARNESS_MODEL or the forwarded credential names look identical in the
+ * log. This records the missing identity.
+ *
+ * Where it lives: a SIBLING of `input` on the run-started event, next to
+ * `stepFingerprint` (step-fingerprint.ts), never a field inside `input`. The
+ * recorded input is what gates step presence and what the fingerprint hashes;
+ * a key added there would be a stored-shape change. The engine reads `workflow`
+ * and `input` only, so a sibling key is inert to replay.
+ *
+ * Off by default (SHIP_HARNESS_RECORD=on): off writes nothing, so the event is
+ * byte-identical to one written before this existed.
+ *
+ * What is NOT recorded here: the binary revision the sandbox ACTUALLY ran. That
+ * is only knowable inside the sandbox, and the `harness-preflight` step already
+ * records it (version, expected, pinned). The page reads it from there; this
+ * record carries the revision Ship EXPECTED to run, so the two can be compared.
+ * Credential VALUES are never recorded, only the names that were configured.
+ */
+
+export const HARNESS_RECORD_FLAG = "SHIP_HARNESS_RECORD";
+
+/** On only for an explicit "on" (or 1/true). Anything else, including unset, is off. */
+export function harnessRecordEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
+  return /^(on|1|true)$/i.test((env[HARNESS_RECORD_FLAG] ?? "").trim());
+}
+
+export interface HarnessIdentity {
+  v: 1;
+  harness: { id: string; adapterVersion: string };
+  /** Ship's own model (native loop) and the model passed to the harness, if any. */
+  model: { ship: string; harness: string | null };
+  /** The vendor package Ship expected the image to carry; null for native. */
+  expectedRevision: { npm: string; version: string } | null;
+  /** The configuration the harness runs under: names and numbers, never secret values. */
+  configuration: { timeoutMs: number | null; claudeBare: boolean | null; forwardedEnvNames: string[]; digest: string };
+  /** Other harnesses a multi-attempt run may execute under. */
+  attempts?: { id: string; adapterVersion: string }[];
+}
+
+function digest(value: unknown): string {
+  return createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16);
+}
+
+export function harnessIdentity(opts: {
+  harness: HarnessRef;
+  attempts?: readonly HarnessRef[];
+  shipModel: string;
+  env?: NodeJS.ProcessEnv;
+}): HarnessIdentity {
+  const env = opts.env ?? process.env;
+  const pkg = HARNESS_PACKAGES[opts.harness.id];
+  const external = pkg !== undefined;
+  const config = external ? externalHarnessConfig(opts.harness.id, env) : undefined;
+  const fields = {
+    timeoutMs: config?.timeoutMs ?? null,
+    claudeBare: config?.claudeBare ?? null,
+    forwardedEnvNames: config?.forward ?? [],
+  };
+  return {
+    v: 1,
+    harness: { id: opts.harness.id, adapterVersion: opts.harness.version },
+    model: { ship: opts.shipModel, harness: config?.model ?? null },
+    expectedRevision: external ? { npm: pkg.npm, version: pkg.version } : null,
+    configuration: { ...fields, digest: digest({ id: opts.harness.id, model: config?.model ?? null, ...fields }) },
+    ...((opts.attempts?.length ?? 0) >= 2 ? { attempts: opts.attempts!.map((a) => ({ id: a.id, adapterVersion: a.version })) } : {}),
+  };
+}

src/harness-probes.test.ts

@@ -0,0 +1,136 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { LocalExecutor } from "@neutron-build/agents";
+import { MemoryEventStore, executeRun } from "@neutron-build/workflow";
+
+import { durableAgent } from "./durable.js";
+import type { DurableAgentOutput } from "./durable.js";
+import { HARNESS_PACKAGES } from "./harness.js";
+import { STEER_UNSUPPORTED_MESSAGE, declarationFor } from "./harness-capabilities.js";
+import type { HarnessDeclaration } from "./harness-capabilities.js";
+import { externalAdapters } from "./harness-external.js";
+import { PROBED_OPERATIONS, credentialResidue, fakeExternalHarness, probeHarness, probeOperation } from "./harness-probes.js";
+
+const IDS = ["native", "claude-code", "opencode"] as const;
+
+test("S13 probes: every declared harness conforms to its own declaration under real adapter probes", async () => {
+  for (const id of IDS) {
+    const report = await probeHarness(id);
+    assert.deepEqual(report.results.map((r) => r.operation), [...PROBED_OPERATIONS]);
+    assert.equal(report.ok, true, `${id}: ${JSON.stringify(report.results.flatMap((r) => r.mismatches))}`);
+    // The rest were not driven. That must be visible, not folded into "ok".
+    assert.deepEqual([...report.unprobed].sort(), ["approval-park", "browser", "interrupt", "recovery", "use-tools"]);
+  }
+});
+
+test("S13 probes: observations are what the adapters really did, not what the table says", async () => {
+  // Native honours all three: parks on the plan, reads the note, frames the scan.
+  const n = (op: Parameters<typeof probeOperation>[1]) => probeOperation("native", op);
+  assert.deepEqual(await n("plan-review"), { operation: "plan-review", accepted: true, sideEffects: true, honoured: true });
+  assert.equal((await n("steer")).honoured, true);
+  assert.equal((await n("investigate")).honoured, true);
+
+  for (const id of ["claude-code", "opencode"] as const) {
+    // Plan review: the adapter throws before putFile/exec, so no agent starts and nothing is edited.
+    const plan = await probeOperation(id, "plan-review");
+    assert.equal(plan.accepted, false);
+    assert.equal(plan.sideEffects, false, "a refused plan review must not have launched the agent");
+    assert.match(plan.refusalMessage ?? "", /Plan review requires the native harness\. No external agent was started/);
+    // Steer: refused at the gate with the real message, nothing stored.
+    assert.deepEqual(await probeOperation(id, "steer"), { operation: "steer", accepted: false, sideEffects: false, refusalMessage: STEER_UNSUPPORTED_MESSAGE });
+    // Scan: the vendor agent really got the read-only framing.
+    assert.deepEqual(await probeOperation(id, "investigate"), { operation: "investigate", accepted: true, sideEffects: true, honoured: true });
+  }
+});
+
+test("S13 probes: an adapter that accepts a steering note and drops it is caught, whichever way the table is wrong", async () => {
+  // The gate lets the note through (a table that claims support). The external
+  // adapter reads only its starting prompt, so the note never reaches the binary.
+  const dropped = await probeOperation("claude-code", "steer", { admitSteer: () => true });
+  assert.equal(dropped.accepted, true);
+  assert.equal(dropped.honoured, false, "the note text must not appear in what the harness received");
+
+  // Against the published declaration (steering refused): accepted a refused op.
+  const real = await probeHarness("claude-code", { admitSteer: () => true });
+  assert.deepEqual(real.results.find((r) => r.operation === "steer")!.mismatches.map((m) => m.kind), ["accepted-unsupported"]);
+
+  // Against a declaration that CLAIMS steering is supported: the silent drop.
+  const claim: HarnessDeclaration = structuredClone(declarationFor("claude-code")!);
+  claim.capabilities.steering = { status: "supported", reason: "claimed, not true" };
+  const lying = await probeHarness("claude-code", { declaration: claim, admitSteer: () => true });
+  assert.equal(lying.ok, false);
+  assert.deepEqual(lying.results.find((r) => r.operation === "steer")!.mismatches.map((m) => m.kind), ["silently-dropped"]);
+
+  // Negative control: the same override on the harness that DOES read notes is clean.
+  assert.equal((await probeHarness("native", { admitSteer: () => true })).ok, true);
+
+  // And the same for opencode, so the finding is not specific to one adapter.
+  const oc: HarnessDeclaration = structuredClone(declarationFor("opencode")!);
+  oc.capabilities.steering = { status: "supported", reason: "claimed, not true" };
+  const ocReport = await probeHarness("opencode", { declaration: oc, admitSteer: () => true });
+  assert.deepEqual(ocReport.results.find((r) => r.operation === "steer")!.mismatches.map((m) => m.kind), ["silently-dropped"]);
+});
+
+test("S13 probes: a table that claims plan review for an external harness is caught as a refusal of a supported op", async () => {
+  const claim: HarnessDeclaration = structuredClone(declarationFor("opencode")!);
+  claim.capabilities.planning = { status: "supported", reason: "claimed, not true" };
+  const report = await probeHarness("opencode", { declaration: claim });
+  assert.deepEqual(report.results.find((r) => r.operation === "plan-review")!.mismatches.map((m) => m.kind), ["refused-supported"]);
+});
+
+// --- credential lifecycle -------------------------------------------------
+
+const SECRET = "sk-ant-oat-probe-SECRET-1234567890";
+
+async function runWithCredential(opts: { extraScript?: string[]; exitCode?: number }) {
+  const rig = await fakeExternalHarness("claude-code", opts);
+  const wf = durableAgent({
+    model: { provider: "never", modelId: "never", async doGenerate() { throw new Error("no model"); }, async *doStream() { throw new Error("unused"); } },
+    executor: rig.provider,
+    workdir: ".",
+    harnesses: externalAdapters({ env: { CLAUDE_CODE_OAUTH_TOKEN: SECRET } }),
+  });
+  const store = new MemoryEventStore();
+  const outcome = await executeRun({ workflow: wf, runId: "cred", store, input: { task: "t", harness: { id: "claude-code", version: "1" } } });
+  return { rig, out: outcome.output as DurableAgentOutput, events: await store.load("cred") };
+}
+
+test("credential lifecycle: a credential reaches the harness for the run and is gone from the sandbox after exit", async () => {
+  const { rig, out, events } = await runWithCredential({});
+  assert.equal(out.status, "finished");
+  // It was delivered: the binary saw it in its environment...
+  assert.match(await rig.seenEnv(), new RegExp(`CLAUDE_CODE_OAUTH_TOKEN=${SECRET}`));
+  // ...but the file it travelled in was already removed before the binary started,
+  // so the agent could not read it back off disk.
+  assert.doesNotMatch(await rig.seenAgentDir(), /harness\.env/);
+  // After exit: no file in the sandbox holds it, and the executor's own env never did.
+  assert.deepEqual(await credentialResidue(rig.root, rig.executor, [SECRET]), []);
+  // The scan does look at files the run wrote: the prompt file is removed too.
+  assert.deepEqual(await credentialResidue(rig.root, rig.executor, ["unattended inside a sandbox"]), []);
+  // And the durable log carries names, never the value.
+  assert.ok(!JSON.stringify(events).includes(SECRET));
+});
+
+test("credential lifecycle: a harness that crashes still leaves no credential file behind", async () => {
+  const { rig, out } = await runWithCredential({ exitCode: 9 });
+  assert.equal(out.status, "error");
+  assert.deepEqual(await credentialResidue(rig.root, rig.executor, [SECRET]), []);
+});
+
+test("credential lifecycle negative control: the residue scan finds a credential a misbehaving harness persists", async () => {
+  // A harness that writes its environment into the tree is a leak the adapter
+  // cannot prevent; the check must be able to see it or its clean result means nothing.
+  const { rig } = await runWithCredential({ extraScript: [`env > leaked-env.txt`] });
+  assert.deepEqual(await credentialResidue(rig.root, rig.executor, [SECRET]), ["leaked-env.txt"]);
+});
+
+test("credential lifecycle negative control: a credential in the executor's own environment is reported", async () => {
+  const rig = await fakeExternalHarness("claude-code");
+  const leaky = new LocalExecutor({ root: rig.root, env: { PATH: process.env.PATH ?? "", CLAUDE_CODE_OAUTH_TOKEN: SECRET } });
+  assert.deepEqual(await credentialResidue(rig.root, leaky, [SECRET]), ["(executor environment)"]);
+});
+
+test("declared harness packages exist for every external probe id", () => {
+  for (const id of ["claude-code", "opencode"]) assert.ok(HARNESS_PACKAGES[id]);
+});