From 35106b7fb48bbc06c50fc16b0379be9fe67857e7 Mon Sep 17 00:00:00 2001
From: kjgbot <kjgbot@agentrelay.dev>
Date: Wed, 23 Sep 2026 11:35:56 -0700
Subject: [PATCH 1/8] feat: add defensible durability eval foundation

---
 benchmarks/README.md                          |  48 ++++
 benchmarks/durability/suite.json              |  69 +++++
 benchmarks/workflow-reliability/protocol.json |  77 ++++++
 scripts/benchmark-protocol.test.mjs           |  38 +++
 scripts/run-evals.mjs                         | 241 ++++++++++++++++++
 scripts/run-evals.test.mjs                    | 109 ++++++++
 6 files changed, 582 insertions(+)
 create mode 100644 benchmarks/README.md
 create mode 100644 benchmarks/durability/suite.json
 create mode 100644 benchmarks/workflow-reliability/protocol.json
 create mode 100644 scripts/benchmark-protocol.test.mjs
 create mode 100644 scripts/run-evals.mjs
 create mode 100644 scripts/run-evals.test.mjs

diff --git a/benchmarks/README.md b/benchmarks/README.md
new file mode 100644
index 000000000..ca59bd6ae
--- /dev/null
+++ b/benchmarks/README.md
@@ -0,0 +1,48 @@
+# Relayflows evidence and benchmarks
+
+This directory separates two claims that must not be blended:
+
+1. `durability/suite.json` is a product conformance eval. It repeatedly executes
+   real crash/resume tests and captures their literal output and provenance. It
+   shows which Relayflows invariants held for the tested commit and host.
+2. `workflow-reliability/protocol.json` is a vendor-neutral competitive protocol.
+   It defines the same workloads and success predicates for Relayflows, a
+   build-it-yourself reference, Temporal, and Inngest. No comparative claim is
+   valid until the required participants have run the same pinned protocol.
+
+In this repository, **deterministic** means deterministic orchestration and
+exactly-once declared effects under the tested faults. It does not mean an LLM
+will emit identical text twice. Quality is measured separately by deterministic
+gates and task-specific evals, matching RFC-0001 settled decision 11.
+
+## Run the durability eval
+
+From the repository root:
+
+```bash
+node scripts/run-evals.mjs \
+  --suite benchmarks/durability/suite.json \
+  --output evidence/relayflows-durability-v1.json
+```
+
+The runner refuses a dirty tree by default. `--allow-dirty` exists for local
+development, but permanently marks the artifact `publicationStatus: ineligible`.
+A publishable result also requires the suite's minimum repetition count and zero
+failed trials. Every trial records its argv, working directory, duration, exit
+code, stdout, and stderr; the report records the suite hash, source commit,
+toolchain, and host class.
+
+## Claim discipline
+
+- A passing durability artifact supports only its six named claims. It is not a
+  general proof that every execution path is deterministic.
+- A build-vs-buy claim requires both `relayflows` and `diy-reference` results.
+- A “best in class” claim requires all participants named by the protocol.
+- Latency, cost, and implementation-burden metrics stay separate. A composite
+  score requires preregistered weights for a named customer profile.
+- Environment failures are inconclusive and must remain in the raw results.
+
+The next benchmark rung is implementing the platform-neutral scenario adapters,
+starting with Relayflows and the DIY reference. The internal durability suite is
+not reused as a competitor workload because doing so would privilege Relayflows'
+implementation rather than test the shared behavioral contract.
diff --git a/benchmarks/durability/suite.json b/benchmarks/durability/suite.json
new file mode 100644
index 000000000..8bdab4ba6
--- /dev/null
+++ b/benchmarks/durability/suite.json
@@ -0,0 +1,69 @@
+{
+  "schemaVersion": 1,
+  "kind": "relayflows-eval",
+  "id": "relayflows-durability-v1",
+  "defaultRepetitions": 5,
+  "minimumPublishableRepetitions": 5,
+  "cases": [
+    {
+      "id": "deterministic-boundary-resume",
+      "claim": "A deterministic three-step flow survives a kill at every boundary without replaying a completed effect.",
+      "command": {
+        "file": "cargo",
+        "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "sigkill_sweep_covers_every_hello_step_boundary", "--", "--exact", "--nocapture", "--test-threads=1"],
+        "cwd": "../../kernel",
+        "timeoutMs": 120000
+      }
+    },
+    {
+      "id": "llm-result-memoization",
+      "claim": "A completed llm result and its budget are reused after the server dies during the following step.",
+      "command": {
+        "file": "cargo",
+        "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step", "--", "--exact", "--nocapture", "--test-threads=1"],
+        "cwd": "../../kernel",
+        "timeoutMs": 120000
+      }
+    },
+    {
+      "id": "agent-effect-exactly-once",
+      "claim": "A crash between effect election and provider invocation produces exactly one declared external effect.",
+      "command": {
+        "file": "cargo",
+        "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once", "--", "--exact", "--nocapture", "--test-threads=1"],
+        "cwd": "../../kernel",
+        "timeoutMs": 120000
+      }
+    },
+    {
+      "id": "durable-channel-redelivery",
+      "claim": "An unacknowledged channel message is redelivered after SIGKILL while its external effect remains exactly once.",
+      "command": {
+        "file": "cargo",
+        "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "channels::channels_sigkill_resume_redelivers_unacked_messages_with_exactly_once_effects", "--", "--exact", "--nocapture", "--test-threads=1"],
+        "cwd": "../../kernel",
+        "timeoutMs": 120000
+      }
+    },
+    {
+      "id": "concurrent-resume-single-lease",
+      "claim": "Concurrent resume requests lease exactly one replacement attempt.",
+      "command": {
+        "file": "cargo",
+        "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "concurrency::concurrent_resumes_lease_exactly_one_attempt", "--", "--exact", "--nocapture", "--test-threads=1"],
+        "cwd": "../../kernel",
+        "timeoutMs": 120000
+      }
+    },
+    {
+      "id": "memory-budget-once",
+      "claim": "A crash after memory injection reuses the journaled pack and charges its tokens exactly once.",
+      "command": {
+        "file": "cargo",
+        "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "memory::memory_sigkill_after_injection_replays_pack_and_charges_it_once", "--", "--exact", "--nocapture", "--test-threads=1"],
+        "cwd": "../../kernel",
+        "timeoutMs": 120000
+      }
+    }
+  ]
+}
diff --git a/benchmarks/workflow-reliability/protocol.json b/benchmarks/workflow-reliability/protocol.json
new file mode 100644
index 000000000..c5c20d8b6
--- /dev/null
+++ b/benchmarks/workflow-reliability/protocol.json
@@ -0,0 +1,77 @@
+{
+  "schemaVersion": 1,
+  "kind": "competitive-benchmark-protocol",
+  "id": "workflow-reliability-v1",
+  "claims": {
+    "buildVsBuy": {
+      "requiredParticipants": ["relayflows", "diy-reference"]
+    },
+    "bestInClass": {
+      "requiredParticipants": ["relayflows", "diy-reference", "temporal", "inngest"]
+    }
+  },
+  "fairnessControls": [
+    "Pin every participant, runtime, model, prompt, tool set, and dependency version before the first measured run.",
+    "Use the same immutable workload fixtures and success predicates for every participant.",
+    "Run on the same isolated host class, randomize participant order, and separate cold-start from warm-run results.",
+    "Use at least 30 measured trials per scenario after five unmeasured warmups.",
+    "Capture raw event history, stdout, stderr, wall time, token use, provider cost, and operator interventions for every trial.",
+    "Publish failures and exclusions; an environment failure is inconclusive, never a product failure or pass.",
+    "Do not form a composite score until metric weights are preregistered for a named customer profile."
+  ],
+  "scenarios": [
+    {
+      "id": "crash-between-steps",
+      "workload": "Complete step A, kill the workflow process before step B starts, restart, and finish the run.",
+      "successPredicates": ["step A external effect count equals 1", "step B external effect count equals 1", "run terminates with a declared completion reason"]
+    },
+    {
+      "id": "crash-during-effect",
+      "workload": "Kill execution after an effect is durably elected but before the provider acknowledgement is recorded.",
+      "successPredicates": ["provider-observed effect count equals 1", "attempt history explains the interrupted attempt", "run completes without manual repair"]
+    },
+    {
+      "id": "semantic-repair",
+      "workload": "Return a syntactically valid but quality-gate-failing agent result, then permit bounded repair attempts.",
+      "successPredicates": ["bad result is not promoted", "repair history is durable", "terminal state is either verified success or a declared verification failure"]
+    },
+    {
+      "id": "durable-human-await",
+      "workload": "Reach a human approval, stop all processes for ten minutes, restart, deliver the approval, and finish.",
+      "successPredicates": ["no compute is required while waiting", "one approval resumes one continuation", "prior completed work is not repeated"]
+    },
+    {
+      "id": "concurrent-resume-race",
+      "workload": "Issue 20 concurrent resume requests for one interrupted run.",
+      "successPredicates": ["one replacement attempt receives the lease", "all other requests are no-ops or typed conflicts", "external effects remain exactly once"]
+    },
+    {
+      "id": "budget-accounting-after-crash",
+      "workload": "Crash after a model or memory result is committed and resume through completion.",
+      "successPredicates": ["resumed token total equals one execution per committed step", "resumed provider cost equals one execution per committed step", "journal attributes every charge to one step"]
+    }
+  ],
+  "metrics": {
+    "primary": [
+      "scenario_success_rate",
+      "duplicate_external_effect_rate",
+      "undeclared_terminal_failure_rate",
+      "manual_interventions_per_100_runs"
+    ],
+    "performance": [
+      "p50_wall_time_ms",
+      "p95_wall_time_ms",
+      "p50_recovery_time_ms",
+      "p95_recovery_time_ms",
+      "tokens_per_successful_run",
+      "provider_cost_per_successful_run"
+    ],
+    "buildBurden": [
+      "preregistered_engineering_minutes",
+      "custom_orchestration_source_lines",
+      "custom_persistence_retry_and_dedupe_source_lines",
+      "runtime_dependencies",
+      "operator_minutes_per_100_runs"
+    ]
+  }
+}
diff --git a/scripts/benchmark-protocol.test.mjs b/scripts/benchmark-protocol.test.mjs
new file mode 100644
index 000000000..352670544
--- /dev/null
+++ b/scripts/benchmark-protocol.test.mjs
@@ -0,0 +1,38 @@
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { test } from 'node:test';
+
+const protocol = JSON.parse(
+  readFileSync(new URL('../benchmarks/workflow-reliability/protocol.json', import.meta.url)),
+);
+
+test('competitive claims name the participants required to support them', () => {
+  assert.deepEqual(protocol.claims.buildVsBuy.requiredParticipants, [
+    'relayflows',
+    'diy-reference',
+  ]);
+  assert.deepEqual(protocol.claims.bestInClass.requiredParticipants, [
+    'relayflows',
+    'diy-reference',
+    'temporal',
+    'inngest',
+  ]);
+});
+
+test('every workload has multiple implementation-neutral success predicates', () => {
+  assert.ok(protocol.scenarios.length >= 6);
+  for (const scenario of protocol.scenarios) {
+    assert.match(scenario.id, /^[a-z0-9]+(?:-[a-z0-9]+)*$/u);
+    assert.ok(scenario.workload.length > 20, scenario.id);
+    assert.ok(scenario.successPredicates.length >= 3, scenario.id);
+  }
+});
+
+test('the protocol keeps correctness, performance, and build burden separate', () => {
+  assert.ok(protocol.metrics.primary.includes('duplicate_external_effect_rate'));
+  assert.ok(protocol.metrics.performance.includes('provider_cost_per_successful_run'));
+  assert.ok(protocol.metrics.buildBurden.includes('custom_persistence_retry_and_dedupe_source_lines'));
+  assert.ok(
+    protocol.fairnessControls.some((control) => control.includes('Do not form a composite score')),
+  );
+});
diff --git a/scripts/run-evals.mjs b/scripts/run-evals.mjs
new file mode 100644
index 000000000..3270b114f
--- /dev/null
+++ b/scripts/run-evals.mjs
@@ -0,0 +1,241 @@
+#!/usr/bin/env node
+
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync } from 'node:fs';
+import { cpus, platform, release } from 'node:os';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { performance } from 'node:perf_hooks';
+
+const RESULT_SCHEMA_VERSION = 1;
+
+export function validateEvalSuite(value) {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    value.schemaVersion !== 1 ||
+    value.kind !== 'relayflows-eval' ||
+    typeof value.id !== 'string' ||
+    value.id.length === 0 ||
+    !Number.isInteger(value.defaultRepetitions) ||
+    value.defaultRepetitions < 1 ||
+    !Number.isInteger(value.minimumPublishableRepetitions) ||
+    value.minimumPublishableRepetitions < 1 ||
+    !Array.isArray(value.cases) ||
+    value.cases.length === 0
+  ) {
+    throw new Error('eval suite has an unsupported or incomplete schema');
+  }
+
+  const ids = new Set();
+  for (const testCase of value.cases) {
+    if (
+      !testCase ||
+      typeof testCase.id !== 'string' ||
+      testCase.id.length === 0 ||
+      ids.has(testCase.id) ||
+      typeof testCase.claim !== 'string' ||
+      testCase.claim.length === 0 ||
+      !testCase.command ||
+      typeof testCase.command.file !== 'string' ||
+      testCase.command.file.length === 0 ||
+      !Array.isArray(testCase.command.args) ||
+      !testCase.command.args.every((arg) => typeof arg === 'string') ||
+      typeof testCase.command.cwd !== 'string' ||
+      !Number.isInteger(testCase.command.timeoutMs) ||
+      testCase.command.timeoutMs < 1
+    ) {
+      throw new Error(`eval suite contains an invalid or duplicate case: ${testCase?.id ?? '<missing>'}`);
+    }
+    ids.add(testCase.id);
+  }
+  return value;
+}
+
+export function runEvalSuite(options) {
+  const rootDir = resolve(options.rootDir ?? process.cwd());
+  const suitePath = resolve(rootDir, options.suitePath);
+  const suiteBytes = readFileSync(suitePath);
+  const suite = validateEvalSuite(JSON.parse(suiteBytes.toString('utf8')));
+  const repetitions = options.repetitions ?? suite.defaultRepetitions;
+  if (!Number.isInteger(repetitions) || repetitions < 1) {
+    throw new Error('repetitions must be a positive integer');
+  }
+
+  const execute = options.execute ?? spawnSync;
+  const provenance = options.provenance ?? collectProvenance(rootDir, execute);
+  if (provenance.source.dirty && !options.allowDirty) {
+    throw new Error('refusing to run a publishable eval from a dirty tree; commit it or pass --allow-dirty');
+  }
+
+  const startedAt = new Date().toISOString();
+  const cases = suite.cases.map((testCase) => {
+    const trials = [];
+    for (let repetition = 1; repetition <= repetitions; repetition += 1) {
+      const before = performance.now();
+      const result = execute(testCase.command.file, testCase.command.args, {
+        cwd: resolve(dirname(suitePath), testCase.command.cwd),
+        encoding: 'utf8',
+        timeout: testCase.command.timeoutMs,
+        maxBuffer: 64 * 1024 * 1024,
+        env: { ...process.env, CARGO_TERM_COLOR: 'never' },
+      });
+      const durationMs = Math.round((performance.now() - before) * 1000) / 1000;
+      trials.push({
+        repetition,
+        passed: result.status === 0,
+        exitCode: result.status,
+        signal: result.signal ?? null,
+        durationMs,
+        command: {
+          file: testCase.command.file,
+          args: testCase.command.args,
+          cwd: testCase.command.cwd,
+        },
+        stdout: text(result.stdout),
+        stderr: text(result.stderr),
+        error: result.error ? String(result.error.message ?? result.error) : null,
+      });
+    }
+    const durations = trials.map((trial) => trial.durationMs);
+    const passedTrials = trials.filter((trial) => trial.passed).length;
+    return {
+      id: testCase.id,
+      claim: testCase.claim,
+      passedTrials,
+      totalTrials: trials.length,
+      passRate: passedTrials / trials.length,
+      durationMs: {
+        p50: percentile(durations, 0.5),
+        p95: percentile(durations, 0.95),
+      },
+      trials,
+    };
+  });
+
+  const totalTrials = cases.reduce((sum, testCase) => sum + testCase.totalTrials, 0);
+  const passedTrials = cases.reduce((sum, testCase) => sum + testCase.passedTrials, 0);
+  const blockers = [];
+  if (provenance.source.dirty) blockers.push('source_tree_dirty');
+  if (repetitions < suite.minimumPublishableRepetitions) blockers.push('insufficient_repetitions');
+  if (passedTrials !== totalTrials) blockers.push('trial_failures');
+
+  return {
+    schemaVersion: RESULT_SCHEMA_VERSION,
+    kind: 'relayflows-eval-result',
+    suite: {
+      id: suite.id,
+      path: options.suitePath,
+      sha256: createHash('sha256').update(suiteBytes).digest('hex'),
+    },
+    implementation: options.implementation ?? 'relayflows',
+    startedAt,
+    completedAt: new Date().toISOString(),
+    repetitions,
+    provenance,
+    summary: {
+      passedTrials,
+      totalTrials,
+      passRate: passedTrials / totalTrials,
+      publicationStatus: blockers.length === 0 ? 'eligible' : 'ineligible',
+      blockers,
+    },
+    cases,
+  };
+}
+
+export function collectProvenance(rootDir, execute = spawnSync) {
+  const gitCommit = commandResult(execute, 'git', ['rev-parse', 'HEAD'], rootDir);
+  const gitStatus = commandResult(execute, 'git', ['status', '--porcelain=v1'], rootDir);
+  const cpu = cpus();
+  return {
+    source: {
+      commit: gitCommit.ok ? gitCommit.stdout : null,
+      dirty: !gitStatus.ok || gitStatus.stdout.length > 0,
+      statusAvailable: gitStatus.ok,
+      statusPorcelain: gitStatus.stdout,
+    },
+    runtime: {
+      node: process.version,
+      rustc: commandText(execute, 'rustc', ['--version'], rootDir) || null,
+      cargo: commandText(execute, 'cargo', ['--version'], rootDir) || null,
+    },
+    host: {
+      platform: platform(),
+      release: release(),
+      arch: process.arch,
+      cpuModel: cpu[0]?.model ?? null,
+      logicalCpus: cpu.length,
+    },
+  };
+}
+
+function commandText(execute, file, args, cwd) {
+  const result = execute(file, args, { cwd, encoding: 'utf8' });
+  if (result.status !== 0) return '';
+  return text(result.stdout).trim();
+}
+
+function commandResult(execute, file, args, cwd) {
+  const result = execute(file, args, { cwd, encoding: 'utf8' });
+  return {
+    ok: result.status === 0,
+    stdout: result.status === 0 ? text(result.stdout).trim() : '',
+  };
+}
+
+function percentile(values, quantile) {
+  const sorted = [...values].sort((left, right) => left - right);
+  const index = Math.max(0, Math.ceil(sorted.length * quantile) - 1);
+  return sorted[index];
+}
+
+function text(value) {
+  if (value === undefined || value === null) return '';
+  return Buffer.isBuffer(value) ? value.toString('utf8') : String(value);
+}
+
+function parseArgs(argv) {
+  const options = { rootDir: process.cwd(), allowDirty: false };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === '--allow-dirty') {
+      options.allowDirty = true;
+      continue;
+    }
+    if (!['--suite', '--output', '--implementation', '--repeat', '--root'].includes(arg)) {
+      throw new Error(`unknown option: ${arg}`);
+    }
+    const value = argv[index + 1];
+    if (!value) throw new Error(`missing value for ${arg}`);
+    index += 1;
+    if (arg === '--suite') options.suitePath = value;
+    if (arg === '--output') options.output = value;
+    if (arg === '--implementation') options.implementation = value;
+    if (arg === '--repeat') options.repetitions = Number(value);
+    if (arg === '--root') options.rootDir = value;
+  }
+  if (!options.suitePath) throw new Error('missing required option --suite');
+  return options;
+}
+
+function main() {
+  try {
+    const options = parseArgs(process.argv.slice(2));
+    const report = runEvalSuite(options);
+    const json = `${JSON.stringify(report, null, 2)}\n`;
+    if (options.output) writeFileSync(resolve(options.rootDir, options.output), json);
+    else process.stdout.write(json);
+    process.stderr.write(
+      `${report.suite.id}: ${report.summary.passedTrials}/${report.summary.totalTrials} trials passed; ` +
+        `publication ${report.summary.publicationStatus}\n`,
+    );
+    process.exitCode = report.summary.passedTrials === report.summary.totalTrials ? 0 : 1;
+  } catch (error) {
+    process.stderr.write(`${error.message}\n`);
+    process.exitCode = 1;
+  }
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) main();
diff --git a/scripts/run-evals.test.mjs b/scripts/run-evals.test.mjs
new file mode 100644
index 000000000..e9d4e44fb
--- /dev/null
+++ b/scripts/run-evals.test.mjs
@@ -0,0 +1,109 @@
+import assert from 'node:assert/strict';
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { test } from 'node:test';
+
+import { runEvalSuite, validateEvalSuite } from './run-evals.mjs';
+
+function suite(overrides = {}) {
+  return {
+    schemaVersion: 1,
+    kind: 'relayflows-eval',
+    id: 'fixture-v1',
+    defaultRepetitions: 2,
+    minimumPublishableRepetitions: 2,
+    cases: [
+      {
+        id: 'resume',
+        claim: 'completed work is not repeated',
+        command: { file: 'fixture', args: ['resume'], cwd: '../..', timeoutMs: 1000 },
+      },
+    ],
+    ...overrides,
+  };
+}
+
+function withSuite(value, verify) {
+  const root = mkdtempSync(join(tmpdir(), 'relayflows-eval-'));
+  try {
+    const directory = join(root, 'benchmarks', 'fixture');
+    mkdirSync(directory, { recursive: true });
+    const path = join(directory, 'suite.json');
+    writeFileSync(path, `${JSON.stringify(value)}\n`);
+    verify({ root, path: 'benchmarks/fixture/suite.json' });
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+
+function provenance(dirty = false) {
+  return {
+    source: { commit: 'a'.repeat(40), dirty, statusPorcelain: dirty ? ' M file' : '' },
+    runtime: { node: 'v1', rustc: 'rustc 1', cargo: 'cargo 1' },
+    host: { platform: 'test', release: '1', arch: 'test', cpuModel: 'test', logicalCpus: 1 },
+  };
+}
+
+test('rejects duplicate case ids', () => {
+  const duplicate = suite();
+  duplicate.cases.push({ ...duplicate.cases[0] });
+  assert.throws(() => validateEvalSuite(duplicate), /invalid or duplicate case/u);
+});
+
+test('fails closed on a dirty tree by default', () => {
+  withSuite(suite(), ({ root, path }) => {
+    assert.throws(
+      () => runEvalSuite({ rootDir: root, suitePath: path, provenance: provenance(true) }),
+      /refusing to run a publishable eval from a dirty tree/u,
+    );
+  });
+});
+
+test('captures literal command output and marks an allowed dirty run ineligible', () => {
+  withSuite(suite(), ({ root, path }) => {
+    const report = runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: provenance(true),
+      allowDirty: true,
+      execute: () => ({ status: 0, signal: null, stdout: 'literal stdout\n', stderr: 'literal stderr\n' }),
+    });
+    assert.equal(report.summary.passRate, 1);
+    assert.equal(report.summary.publicationStatus, 'ineligible');
+    assert.deepEqual(report.summary.blockers, ['source_tree_dirty']);
+    assert.equal(report.cases[0].trials[0].stdout, 'literal stdout\n');
+    assert.equal(report.cases[0].trials[0].stderr, 'literal stderr\n');
+    assert.equal(report.cases[0].trials[0].command.cwd, '../..');
+  });
+});
+
+test('reports a failed trial without erasing its exit evidence', () => {
+  withSuite(suite({ defaultRepetitions: 1, minimumPublishableRepetitions: 1 }), ({ root, path }) => {
+    const report = runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: provenance(false),
+      execute: () => ({ status: 9, signal: null, stdout: 'partial\n', stderr: 'failed\n' }),
+    });
+    assert.equal(report.summary.publicationStatus, 'ineligible');
+    assert.deepEqual(report.summary.blockers, ['trial_failures']);
+    assert.equal(report.cases[0].trials[0].exitCode, 9);
+    assert.equal(report.cases[0].trials[0].stderr, 'failed\n');
+  });
+});
+
+test('only clean, passing, sufficiently repeated evidence is publication eligible', () => {
+  withSuite(suite(), ({ root, path }) => {
+    const report = runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: provenance(false),
+      execute: () => ({ status: 0, signal: null, stdout: 'ok\n', stderr: '' }),
+    });
+    assert.equal(report.summary.publicationStatus, 'eligible');
+    assert.deepEqual(report.summary.blockers, []);
+    assert.equal(report.summary.totalTrials, 2);
+    assert.match(report.suite.sha256, /^[a-f0-9]{64}$/u);
+  });
+});

From 2c1b277bcfcfa86bb54e10b96f6ffb5be49277ab Mon Sep 17 00:00:00 2001
From: kjgbot <kjgbot@agentrelay.dev>
Date: Wed, 23 Sep 2026 12:50:03 -0700
Subject: [PATCH 2/8] fix: fail closed on invalid eval evidence

---
 benchmarks/README.md                |  16 ++-
 benchmarks/durability/suite.json    |  59 +++++++--
 scripts/benchmark-protocol.test.mjs |   6 +
 scripts/run-evals.mjs               | 194 +++++++++++++++++++++-------
 scripts/run-evals.test.mjs          | 158 ++++++++++++++++++++--
 5 files changed, 357 insertions(+), 76 deletions(-)

diff --git a/benchmarks/README.md b/benchmarks/README.md
index ca59bd6ae..12096e4a2 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -22,15 +22,17 @@ From the repository root:
 ```bash
 node scripts/run-evals.mjs \
   --suite benchmarks/durability/suite.json \
-  --output evidence/relayflows-durability-v1.json
+  --output /tmp/relayflows-durability-v1.json
 ```
 
-The runner refuses a dirty tree by default. `--allow-dirty` exists for local
-development, but permanently marks the artifact `publicationStatus: ineligible`.
-A publishable result also requires the suite's minimum repetition count and zero
-failed trials. Every trial records its argv, working directory, duration, exit
-code, stdout, and stderr; the report records the suite hash, source commit,
-toolchain, and host class.
+The runner refuses a dirty tree by default and requires report output to live
+outside the source tree, so one result cannot silently dirty the next run.
+`--allow-dirty` exists for local development, but permanently marks the artifact
+`publicationStatus: ineligible` and exits nonzero. A publishable result also
+requires the suite's minimum repetition count, zero failed trials, and explicit
+output witnesses proving each selected test actually ran. Every trial records
+its argv, working directory, duration, outcome, exit code, stdout, and stderr;
+the report records the suite hash, source commit, toolchain, and host class.
 
 ## Claim discipline
 
diff --git a/benchmarks/durability/suite.json b/benchmarks/durability/suite.json
index 8bdab4ba6..eca2cae67 100644
--- a/benchmarks/durability/suite.json
+++ b/benchmarks/durability/suite.json
@@ -2,68 +2,99 @@
   "schemaVersion": 1,
   "kind": "relayflows-eval",
   "id": "relayflows-durability-v1",
-  "defaultRepetitions": 5,
-  "minimumPublishableRepetitions": 5,
+  "defaultRepetitions": 30,
+  "minimumPublishableRepetitions": 30,
+  "environmentFailurePatterns": [
+    "CARGO_NOT_FOUND",
+    "CARGO_BOOTSTRAP_FAILED",
+    "failed to download",
+    "Could not resolve host",
+    "No space left on device"
+  ],
   "cases": [
     {
       "id": "deterministic-boundary-resume",
       "claim": "A deterministic three-step flow survives a kill at every boundary without replaying a completed effect.",
       "command": {
-        "file": "cargo",
+        "file": "../ops/cargo.sh",
         "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "sigkill_sweep_covers_every_hello_step_boundary", "--", "--exact", "--nocapture", "--test-threads=1"],
         "cwd": "../../kernel",
         "timeoutMs": 120000
-      }
+      },
+      "successWitnesses": [
+        {"stream": "stdout", "contains": "test sigkill_sweep_covers_every_hello_step_boundary ... ok"},
+        {"stream": "stdout", "contains": "test result: ok. 1 passed; 0 failed"}
+      ]
     },
     {
       "id": "llm-result-memoization",
       "claim": "A completed llm result and its budget are reused after the server dies during the following step.",
       "command": {
-        "file": "cargo",
+        "file": "../ops/cargo.sh",
         "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step", "--", "--exact", "--nocapture", "--test-threads=1"],
         "cwd": "../../kernel",
         "timeoutMs": 120000
-      }
+      },
+      "successWitnesses": [
+        {"stream": "stdout", "contains": "test llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step ... ok"},
+        {"stream": "stdout", "contains": "test result: ok. 1 passed; 0 failed"}
+      ]
     },
     {
       "id": "agent-effect-exactly-once",
       "claim": "A crash between effect election and provider invocation produces exactly one declared external effect.",
       "command": {
-        "file": "cargo",
+        "file": "../ops/cargo.sh",
         "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once", "--", "--exact", "--nocapture", "--test-threads=1"],
         "cwd": "../../kernel",
         "timeoutMs": 120000
-      }
+      },
+      "successWitnesses": [
+        {"stream": "stdout", "contains": "test agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once ... ok"},
+        {"stream": "stdout", "contains": "test result: ok. 1 passed; 0 failed"}
+      ]
     },
     {
       "id": "durable-channel-redelivery",
       "claim": "An unacknowledged channel message is redelivered after SIGKILL while its external effect remains exactly once.",
       "command": {
-        "file": "cargo",
+        "file": "../ops/cargo.sh",
         "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "channels::channels_sigkill_resume_redelivers_unacked_messages_with_exactly_once_effects", "--", "--exact", "--nocapture", "--test-threads=1"],
         "cwd": "../../kernel",
         "timeoutMs": 120000
-      }
+      },
+      "successWitnesses": [
+        {"stream": "stdout", "contains": "test channels::channels_sigkill_resume_redelivers_unacked_messages_with_exactly_once_effects ... ok"},
+        {"stream": "stdout", "contains": "test result: ok. 1 passed; 0 failed"}
+      ]
     },
     {
       "id": "concurrent-resume-single-lease",
       "claim": "Concurrent resume requests lease exactly one replacement attempt.",
       "command": {
-        "file": "cargo",
+        "file": "../ops/cargo.sh",
         "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "concurrency::concurrent_resumes_lease_exactly_one_attempt", "--", "--exact", "--nocapture", "--test-threads=1"],
         "cwd": "../../kernel",
         "timeoutMs": 120000
-      }
+      },
+      "successWitnesses": [
+        {"stream": "stdout", "contains": "test concurrency::concurrent_resumes_lease_exactly_one_attempt ... ok"},
+        {"stream": "stdout", "contains": "test result: ok. 1 passed; 0 failed"}
+      ]
     },
     {
       "id": "memory-budget-once",
       "claim": "A crash after memory injection reuses the journaled pack and charges its tokens exactly once.",
       "command": {
-        "file": "cargo",
+        "file": "../ops/cargo.sh",
         "args": ["test", "-p", "relayflowd", "--test", "crash_resume", "memory::memory_sigkill_after_injection_replays_pack_and_charges_it_once", "--", "--exact", "--nocapture", "--test-threads=1"],
         "cwd": "../../kernel",
         "timeoutMs": 120000
-      }
+      },
+      "successWitnesses": [
+        {"stream": "stdout", "contains": "test memory::memory_sigkill_after_injection_replays_pack_and_charges_it_once ... ok"},
+        {"stream": "stdout", "contains": "test result: ok. 1 passed; 0 failed"}
+      ]
     }
   ]
 }
diff --git a/scripts/benchmark-protocol.test.mjs b/scripts/benchmark-protocol.test.mjs
index 352670544..dbd8e4909 100644
--- a/scripts/benchmark-protocol.test.mjs
+++ b/scripts/benchmark-protocol.test.mjs
@@ -25,6 +25,12 @@ test('every workload has multiple implementation-neutral success predicates', ()
     assert.match(scenario.id, /^[a-z0-9]+(?:-[a-z0-9]+)*$/u);
     assert.ok(scenario.workload.length > 20, scenario.id);
     assert.ok(scenario.successPredicates.length >= 3, scenario.id);
+    assert.ok(
+      scenario.successPredicates.every(
+        (predicate) => typeof predicate === 'string' && predicate.length > 0,
+      ),
+      scenario.id,
+    );
   }
 });
 
diff --git a/scripts/run-evals.mjs b/scripts/run-evals.mjs
index 3270b114f..5cf4f31b5 100644
--- a/scripts/run-evals.mjs
+++ b/scripts/run-evals.mjs
@@ -4,49 +4,65 @@ import { spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { readFileSync, writeFileSync } from 'node:fs';
 import { cpus, platform, release } from 'node:os';
-import { dirname, resolve } from 'node:path';
+import { dirname, relative, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { performance } from 'node:perf_hooks';
 
 const RESULT_SCHEMA_VERSION = 1;
 
 export function validateEvalSuite(value) {
-  if (
-    !value ||
-    typeof value !== 'object' ||
-    value.schemaVersion !== 1 ||
-    value.kind !== 'relayflows-eval' ||
-    typeof value.id !== 'string' ||
-    value.id.length === 0 ||
-    !Number.isInteger(value.defaultRepetitions) ||
-    value.defaultRepetitions < 1 ||
-    !Number.isInteger(value.minimumPublishableRepetitions) ||
-    value.minimumPublishableRepetitions < 1 ||
-    !Array.isArray(value.cases) ||
-    value.cases.length === 0
-  ) {
-    throw new Error('eval suite has an unsupported or incomplete schema');
+  if (!value || typeof value !== 'object') throw new Error('eval suite must be an object');
+  if (value.schemaVersion !== 1) throw new Error('eval suite schemaVersion must be 1');
+  if (value.kind !== 'relayflows-eval') throw new Error('eval suite kind must be relayflows-eval');
+  requireNonEmptyString(value.id, 'eval suite id');
+  requirePositiveInteger(value.defaultRepetitions, 'defaultRepetitions');
+  requirePositiveInteger(
+    value.minimumPublishableRepetitions,
+    'minimumPublishableRepetitions',
+  );
+  if (!Array.isArray(value.environmentFailurePatterns)) {
+    throw new Error('environmentFailurePatterns must be an array');
+  }
+  value.environmentFailurePatterns.forEach((pattern, index) =>
+    requireNonEmptyString(pattern, `environmentFailurePatterns[${index}]`),
+  );
+  if (!Array.isArray(value.cases) || value.cases.length === 0) {
+    throw new Error('eval suite cases must be a non-empty array');
   }
 
   const ids = new Set();
-  for (const testCase of value.cases) {
+  for (const [caseIndex, testCase] of value.cases.entries()) {
+    if (!testCase || typeof testCase !== 'object') {
+      throw new Error(`cases[${caseIndex}] must be an object`);
+    }
+    requireNonEmptyString(testCase.id, `cases[${caseIndex}].id`);
+    if (ids.has(testCase.id)) throw new Error(`duplicate eval case id: ${testCase.id}`);
+    requireNonEmptyString(testCase.claim, `cases[${caseIndex}].claim`);
+    if (!testCase.command || typeof testCase.command !== 'object') {
+      throw new Error(`cases[${caseIndex}].command must be an object`);
+    }
+    requireNonEmptyString(testCase.command.file, `cases[${caseIndex}].command.file`);
     if (
-      !testCase ||
-      typeof testCase.id !== 'string' ||
-      testCase.id.length === 0 ||
-      ids.has(testCase.id) ||
-      typeof testCase.claim !== 'string' ||
-      testCase.claim.length === 0 ||
-      !testCase.command ||
-      typeof testCase.command.file !== 'string' ||
-      testCase.command.file.length === 0 ||
       !Array.isArray(testCase.command.args) ||
-      !testCase.command.args.every((arg) => typeof arg === 'string') ||
-      typeof testCase.command.cwd !== 'string' ||
-      !Number.isInteger(testCase.command.timeoutMs) ||
-      testCase.command.timeoutMs < 1
+      !testCase.command.args.every((arg) => typeof arg === 'string')
     ) {
-      throw new Error(`eval suite contains an invalid or duplicate case: ${testCase?.id ?? '<missing>'}`);
+      throw new Error(`cases[${caseIndex}].command.args must be an array of strings`);
+    }
+    requireNonEmptyString(testCase.command.cwd, `cases[${caseIndex}].command.cwd`);
+    requirePositiveInteger(testCase.command.timeoutMs, `cases[${caseIndex}].command.timeoutMs`);
+    if (!Array.isArray(testCase.successWitnesses) || testCase.successWitnesses.length === 0) {
+      throw new Error(`cases[${caseIndex}].successWitnesses must be a non-empty array`);
+    }
+    for (const [witnessIndex, witness] of testCase.successWitnesses.entries()) {
+      if (!witness || !['stdout', 'stderr'].includes(witness.stream)) {
+        throw new Error(
+          `cases[${caseIndex}].successWitnesses[${witnessIndex}].stream must be stdout or stderr`,
+        );
+      }
+      requireNonEmptyString(
+        witness.contains,
+        `cases[${caseIndex}].successWitnesses[${witnessIndex}].contains`,
+      );
     }
     ids.add(testCase.id);
   }
@@ -65,26 +81,48 @@ export function runEvalSuite(options) {
 
   const execute = options.execute ?? spawnSync;
   const provenance = options.provenance ?? collectProvenance(rootDir, execute);
+  if (resolve(provenance.source.root) !== rootDir) {
+    throw new Error(
+      `provenance root ${provenance.source.root} does not match requested root ${rootDir}`,
+    );
+  }
   if (provenance.source.dirty && !options.allowDirty) {
     throw new Error('refusing to run a publishable eval from a dirty tree; commit it or pass --allow-dirty');
   }
 
   const startedAt = new Date().toISOString();
   const cases = suite.cases.map((testCase) => {
+    const commandCwd = resolve(dirname(suitePath), testCase.command.cwd);
+    if (!isWithin(rootDir, commandCwd)) {
+      throw new Error(`case ${testCase.id} command cwd escapes the provenance root`);
+    }
     const trials = [];
     for (let repetition = 1; repetition <= repetitions; repetition += 1) {
       const before = performance.now();
       const result = execute(testCase.command.file, testCase.command.args, {
-        cwd: resolve(dirname(suitePath), testCase.command.cwd),
+        cwd: commandCwd,
         encoding: 'utf8',
         timeout: testCase.command.timeoutMs,
         maxBuffer: 64 * 1024 * 1024,
-        env: { ...process.env, CARGO_TERM_COLOR: 'never' },
+        env: process.env,
       });
       const durationMs = Math.round((performance.now() - before) * 1000) / 1000;
+      const stdout = text(result.stdout);
+      const stderr = text(result.stderr);
+      const missingWitnesses = testCase.successWitnesses.filter(
+        (witness) => !{ stdout, stderr }[witness.stream].includes(witness.contains),
+      );
+      const outcome = classifyTrial({
+        result,
+        stdout,
+        stderr,
+        missingWitnesses,
+        environmentFailurePatterns: suite.environmentFailurePatterns,
+      });
       trials.push({
         repetition,
-        passed: result.status === 0,
+        outcome,
+        passed: outcome === 'passed',
         exitCode: result.status,
         signal: result.signal ?? null,
         durationMs,
@@ -93,22 +131,30 @@ export function runEvalSuite(options) {
           args: testCase.command.args,
           cwd: testCase.command.cwd,
         },
-        stdout: text(result.stdout),
-        stderr: text(result.stderr),
+        stdout,
+        stderr,
+        missingWitnesses,
         error: result.error ? String(result.error.message ?? result.error) : null,
       });
     }
-    const durations = trials.map((trial) => trial.durationMs);
+    const durations = trials.filter((trial) => trial.passed).map((trial) => trial.durationMs);
     const passedTrials = trials.filter((trial) => trial.passed).length;
+    const failedTrials = trials.filter((trial) => trial.outcome === 'failed').length;
+    const invalidTrials = trials.filter((trial) => trial.outcome === 'invalid').length;
+    const inconclusiveTrials = trials.filter((trial) => trial.outcome === 'inconclusive').length;
     return {
       id: testCase.id,
       claim: testCase.claim,
       passedTrials,
+      failedTrials,
+      invalidTrials,
+      inconclusiveTrials,
       totalTrials: trials.length,
       passRate: passedTrials / trials.length,
       durationMs: {
-        p50: percentile(durations, 0.5),
-        p95: percentile(durations, 0.95),
+        sampleSize: durations.length,
+        p50: durations.length > 0 ? percentile(durations, 0.5) : null,
+        p95: durations.length >= 20 ? percentile(durations, 0.95) : null,
       },
       trials,
     };
@@ -116,10 +162,18 @@ export function runEvalSuite(options) {
 
   const totalTrials = cases.reduce((sum, testCase) => sum + testCase.totalTrials, 0);
   const passedTrials = cases.reduce((sum, testCase) => sum + testCase.passedTrials, 0);
+  const failedTrials = cases.reduce((sum, testCase) => sum + testCase.failedTrials, 0);
+  const invalidTrials = cases.reduce((sum, testCase) => sum + testCase.invalidTrials, 0);
+  const inconclusiveTrials = cases.reduce(
+    (sum, testCase) => sum + testCase.inconclusiveTrials,
+    0,
+  );
   const blockers = [];
   if (provenance.source.dirty) blockers.push('source_tree_dirty');
   if (repetitions < suite.minimumPublishableRepetitions) blockers.push('insufficient_repetitions');
-  if (passedTrials !== totalTrials) blockers.push('trial_failures');
+  if (failedTrials > 0) blockers.push('trial_failures');
+  if (invalidTrials > 0) blockers.push('execution_witness_missing');
+  if (inconclusiveTrials > 0) blockers.push('environment_failures');
 
   return {
     schemaVersion: RESULT_SCHEMA_VERSION,
@@ -136,6 +190,9 @@ export function runEvalSuite(options) {
     provenance,
     summary: {
       passedTrials,
+      failedTrials,
+      invalidTrials,
+      inconclusiveTrials,
       totalTrials,
       passRate: passedTrials / totalTrials,
       publicationStatus: blockers.length === 0 ? 'eligible' : 'ineligible',
@@ -148,18 +205,20 @@ export function runEvalSuite(options) {
 export function collectProvenance(rootDir, execute = spawnSync) {
   const gitCommit = commandResult(execute, 'git', ['rev-parse', 'HEAD'], rootDir);
   const gitStatus = commandResult(execute, 'git', ['status', '--porcelain=v1'], rootDir);
+  const gitRoot = commandResult(execute, 'git', ['rev-parse', '--show-toplevel'], rootDir);
   const cpu = cpus();
   return {
     source: {
       commit: gitCommit.ok ? gitCommit.stdout : null,
-      dirty: !gitStatus.ok || gitStatus.stdout.length > 0,
+      root: gitRoot.ok ? resolve(gitRoot.stdout) : rootDir,
+      dirty: !gitCommit.ok || !gitStatus.ok || !gitRoot.ok || gitStatus.stdout.length > 0,
       statusAvailable: gitStatus.ok,
       statusPorcelain: gitStatus.stdout,
     },
     runtime: {
       node: process.version,
       rustc: commandText(execute, 'rustc', ['--version'], rootDir) || null,
-      cargo: commandText(execute, 'cargo', ['--version'], rootDir) || null,
+      cargo: commandText(execute, resolve(rootDir, 'ops/cargo.sh'), ['--version'], rootDir) || null,
     },
     host: {
       platform: platform(),
@@ -171,6 +230,25 @@ export function collectProvenance(rootDir, execute = spawnSync) {
   };
 }
 
+function classifyTrial({
+  result,
+  stdout,
+  stderr,
+  missingWitnesses,
+  environmentFailurePatterns,
+}) {
+  if (result.status === 0) return missingWitnesses.length === 0 ? 'passed' : 'invalid';
+  const combined = `${stdout}\n${stderr}\n${result.error?.message ?? ''}`;
+  if (
+    result.error ||
+    result.status === 127 ||
+    environmentFailurePatterns.some((pattern) => combined.includes(pattern))
+  ) {
+    return 'inconclusive';
+  }
+  return 'failed';
+}
+
 function commandText(execute, file, args, cwd) {
   const result = execute(file, args, { cwd, encoding: 'utf8' });
   if (result.status !== 0) return '';
@@ -191,12 +269,29 @@ function percentile(values, quantile) {
   return sorted[index];
 }
 
+function isWithin(parent, child) {
+  const path = relative(parent, child);
+  return path === '' || (!path.startsWith('..') && !path.startsWith('/'));
+}
+
+function requireNonEmptyString(value, field) {
+  if (typeof value !== 'string' || value.length === 0) {
+    throw new Error(`${field} must be a non-empty string`);
+  }
+}
+
+function requirePositiveInteger(value, field) {
+  if (!Number.isInteger(value) || value < 1) {
+    throw new Error(`${field} must be a positive integer`);
+  }
+}
+
 function text(value) {
   if (value === undefined || value === null) return '';
   return Buffer.isBuffer(value) ? value.toString('utf8') : String(value);
 }
 
-function parseArgs(argv) {
+export function parseArgs(argv) {
   const options = { rootDir: process.cwd(), allowDirty: false };
   for (let index = 0; index < argv.length; index += 1) {
     const arg = argv[index];
@@ -220,9 +315,20 @@ function parseArgs(argv) {
   return options;
 }
 
+export function reportExitCode(report) {
+  return report.summary.publicationStatus === 'eligible' ? 0 : 1;
+}
+
+export function validateOutputPath(rootDir, output) {
+  if (output && isWithin(resolve(rootDir), resolve(rootDir, output))) {
+    throw new Error('report output must be outside the source tree so it cannot dirty later runs');
+  }
+}
+
 function main() {
   try {
     const options = parseArgs(process.argv.slice(2));
+    validateOutputPath(options.rootDir, options.output);
     const report = runEvalSuite(options);
     const json = `${JSON.stringify(report, null, 2)}\n`;
     if (options.output) writeFileSync(resolve(options.rootDir, options.output), json);
@@ -231,7 +337,7 @@ function main() {
       `${report.suite.id}: ${report.summary.passedTrials}/${report.summary.totalTrials} trials passed; ` +
         `publication ${report.summary.publicationStatus}\n`,
     );
-    process.exitCode = report.summary.passedTrials === report.summary.totalTrials ? 0 : 1;
+    process.exitCode = reportExitCode(report);
   } catch (error) {
     process.stderr.write(`${error.message}\n`);
     process.exitCode = 1;
diff --git a/scripts/run-evals.test.mjs b/scripts/run-evals.test.mjs
index e9d4e44fb..66b9a5fc7 100644
--- a/scripts/run-evals.test.mjs
+++ b/scripts/run-evals.test.mjs
@@ -4,7 +4,14 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { test } from 'node:test';
 
-import { runEvalSuite, validateEvalSuite } from './run-evals.mjs';
+import {
+  collectProvenance,
+  parseArgs,
+  reportExitCode,
+  runEvalSuite,
+  validateEvalSuite,
+  validateOutputPath,
+} from './run-evals.mjs';
 
 function suite(overrides = {}) {
   return {
@@ -13,11 +20,13 @@ function suite(overrides = {}) {
     id: 'fixture-v1',
     defaultRepetitions: 2,
     minimumPublishableRepetitions: 2,
+    environmentFailurePatterns: ['TOOLCHAIN_UNAVAILABLE'],
     cases: [
       {
         id: 'resume',
         claim: 'completed work is not repeated',
         command: { file: 'fixture', args: ['resume'], cwd: '../..', timeoutMs: 1000 },
+        successWitnesses: [{ stream: 'stdout', contains: 'executed resume' }],
       },
     ],
     ...overrides,
@@ -37,9 +46,15 @@ function withSuite(value, verify) {
   }
 }
 
-function provenance(dirty = false) {
+function provenance(root, dirty = false) {
   return {
-    source: { commit: 'a'.repeat(40), dirty, statusPorcelain: dirty ? ' M file' : '' },
+    source: {
+      commit: 'a'.repeat(40),
+      root,
+      dirty,
+      statusAvailable: true,
+      statusPorcelain: dirty ? ' M file' : '',
+    },
     runtime: { node: 'v1', rustc: 'rustc 1', cargo: 'cargo 1' },
     host: { platform: 'test', release: '1', arch: 'test', cpuModel: 'test', logicalCpus: 1 },
   };
@@ -48,13 +63,22 @@ function provenance(dirty = false) {
 test('rejects duplicate case ids', () => {
   const duplicate = suite();
   duplicate.cases.push({ ...duplicate.cases[0] });
-  assert.throws(() => validateEvalSuite(duplicate), /invalid or duplicate case/u);
+  assert.throws(() => validateEvalSuite(duplicate), /duplicate eval case id/u);
+});
+
+test('validation names the malformed field', () => {
+  const malformed = suite();
+  malformed.cases[0].command.timeoutMs = 0;
+  assert.throws(
+    () => validateEvalSuite(malformed),
+    /cases\[0\]\.command\.timeoutMs must be a positive integer/u,
+  );
 });
 
 test('fails closed on a dirty tree by default', () => {
   withSuite(suite(), ({ root, path }) => {
     assert.throws(
-      () => runEvalSuite({ rootDir: root, suitePath: path, provenance: provenance(true) }),
+      () => runEvalSuite({ rootDir: root, suitePath: path, provenance: provenance(root, true) }),
       /refusing to run a publishable eval from a dirty tree/u,
     );
   });
@@ -65,14 +89,19 @@ test('captures literal command output and marks an allowed dirty run ineligible'
     const report = runEvalSuite({
       rootDir: root,
       suitePath: path,
-      provenance: provenance(true),
+      provenance: provenance(root, true),
       allowDirty: true,
-      execute: () => ({ status: 0, signal: null, stdout: 'literal stdout\n', stderr: 'literal stderr\n' }),
+      execute: () => ({
+        status: 0,
+        signal: null,
+        stdout: 'executed resume\nliteral stdout\n',
+        stderr: 'literal stderr\n',
+      }),
     });
     assert.equal(report.summary.passRate, 1);
     assert.equal(report.summary.publicationStatus, 'ineligible');
     assert.deepEqual(report.summary.blockers, ['source_tree_dirty']);
-    assert.equal(report.cases[0].trials[0].stdout, 'literal stdout\n');
+    assert.equal(report.cases[0].trials[0].stdout, 'executed resume\nliteral stdout\n');
     assert.equal(report.cases[0].trials[0].stderr, 'literal stderr\n');
     assert.equal(report.cases[0].trials[0].command.cwd, '../..');
   });
@@ -83,7 +112,7 @@ test('reports a failed trial without erasing its exit evidence', () => {
     const report = runEvalSuite({
       rootDir: root,
       suitePath: path,
-      provenance: provenance(false),
+      provenance: provenance(root, false),
       execute: () => ({ status: 9, signal: null, stdout: 'partial\n', stderr: 'failed\n' }),
     });
     assert.equal(report.summary.publicationStatus, 'ineligible');
@@ -98,12 +127,119 @@ test('only clean, passing, sufficiently repeated evidence is publication eligibl
     const report = runEvalSuite({
       rootDir: root,
       suitePath: path,
-      provenance: provenance(false),
-      execute: () => ({ status: 0, signal: null, stdout: 'ok\n', stderr: '' }),
+      provenance: provenance(root, false),
+      execute: () => ({ status: 0, signal: null, stdout: 'executed resume\n', stderr: '' }),
     });
     assert.equal(report.summary.publicationStatus, 'eligible');
     assert.deepEqual(report.summary.blockers, []);
     assert.equal(report.summary.totalTrials, 2);
+    assert.equal(report.cases[0].durationMs.p95, null);
     assert.match(report.suite.sha256, /^[a-f0-9]{64}$/u);
   });
 });
+
+test('zero exit without the execution witness is invalid, never passing', () => {
+  withSuite(suite({ defaultRepetitions: 1, minimumPublishableRepetitions: 1 }), ({ root, path }) => {
+    const report = runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: provenance(root, false),
+      execute: () => ({ status: 0, signal: null, stdout: 'running 0 tests\n', stderr: '' }),
+    });
+    assert.equal(report.cases[0].trials[0].outcome, 'invalid');
+    assert.equal(report.summary.invalidTrials, 1);
+    assert.deepEqual(report.summary.blockers, ['execution_witness_missing']);
+  });
+});
+
+test('a recognized toolchain failure is inconclusive rather than a product failure', () => {
+  withSuite(suite({ defaultRepetitions: 1, minimumPublishableRepetitions: 1 }), ({ root, path }) => {
+    const report = runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: provenance(root, false),
+      execute: () => ({
+        status: 127,
+        signal: null,
+        stdout: '',
+        stderr: 'TOOLCHAIN_UNAVAILABLE\n',
+      }),
+    });
+    assert.equal(report.cases[0].trials[0].outcome, 'inconclusive');
+    assert.deepEqual(report.summary.blockers, ['environment_failures']);
+  });
+});
+
+test('case cwd resolves relative to the suite and cannot escape the provenance root', () => {
+  withSuite(suite(), ({ root, path }) => {
+    const observed = [];
+    runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: provenance(root, false),
+      execute: (_file, _args, options) => {
+        observed.push(options.cwd);
+        return { status: 0, signal: null, stdout: 'executed resume\n', stderr: '' };
+      },
+    });
+    assert.deepEqual(observed, [root, root]);
+
+    const escaping = suite();
+    escaping.cases[0].command.cwd = '../../..';
+    writeFileSync(join(root, path), `${JSON.stringify(escaping)}\n`);
+    assert.throws(
+      () =>
+        runEvalSuite({
+          rootDir: root,
+          suitePath: path,
+          provenance: provenance(root, false),
+          execute: () => ({ status: 0, stdout: 'executed resume\n', stderr: '' }),
+        }),
+      /command cwd escapes the provenance root/u,
+    );
+  });
+});
+
+test('collectProvenance exercises clean, dirty, and unavailable git state', () => {
+  const root = '/work/repo';
+  const execute = (_file, args) => {
+    const key = args.join(' ');
+    if (key === 'rev-parse HEAD') return { status: 0, stdout: `${'b'.repeat(40)}\n` };
+    if (key === 'status --porcelain=v1') return { status: 0, stdout: '' };
+    if (key === 'rev-parse --show-toplevel') return { status: 0, stdout: `${root}\n` };
+    return { status: 0, stdout: 'tool 1\n' };
+  };
+  const clean = collectProvenance(root, execute);
+  assert.equal(clean.source.dirty, false);
+  assert.equal(clean.source.commit, 'b'.repeat(40));
+  assert.equal(clean.source.root, root);
+
+  const dirty = collectProvenance(root, (file, args, options) => {
+    if (args.join(' ') === 'status --porcelain=v1') return { status: 0, stdout: ' M file\n' };
+    return execute(file, args, options);
+  });
+  assert.equal(dirty.source.dirty, true);
+  assert.equal(dirty.source.statusPorcelain, 'M file');
+
+  const unavailable = collectProvenance(root, () => ({ status: 128, stdout: '', stderr: 'not a repo' }));
+  assert.equal(unavailable.source.dirty, true);
+  assert.equal(unavailable.source.commit, null);
+  assert.equal(unavailable.source.statusAvailable, false);
+});
+
+test('CLI parsing and exit policy fail closed for nonpublishable evidence', () => {
+  assert.deepEqual(parseArgs(['--suite', 'suite.json', '--repeat', '30', '--allow-dirty']), {
+    rootDir: process.cwd(),
+    allowDirty: true,
+    suitePath: 'suite.json',
+    repetitions: 30,
+  });
+  assert.throws(() => parseArgs(['--unknown']), /unknown option/u);
+  assert.equal(reportExitCode({ summary: { publicationStatus: 'eligible' } }), 0);
+  assert.equal(reportExitCode({ summary: { publicationStatus: 'ineligible' } }), 1);
+  assert.throws(
+    () => validateOutputPath('/work/repo', 'evidence/result.json'),
+    /output must be outside the source tree/u,
+  );
+  assert.doesNotThrow(() => validateOutputPath('/work/repo', '/tmp/result.json'));
+});

From 58a192766e5d56c3a272f2de8b777a1e8ee3aad4 Mon Sep 17 00:00:00 2001
From: kjgbot <kjgbot@agentrelay.dev>
Date: Wed, 23 Sep 2026 12:56:16 -0700
Subject: [PATCH 3/8] test: add black-box SIGKILL durability proof

---
 benchmarks/README.md                      |   5 +-
 benchmarks/durability/black-box-crash.mjs | 181 ++++++++++++++++++++++
 benchmarks/durability/suite.json          |  16 ++
 3 files changed, 200 insertions(+), 2 deletions(-)
 create mode 100644 benchmarks/durability/black-box-crash.mjs

diff --git a/benchmarks/README.md b/benchmarks/README.md
index 12096e4a2..c73ddc48e 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -3,7 +3,8 @@
 This directory separates two claims that must not be blended:
 
 1. `durability/suite.json` is a product conformance eval. It repeatedly executes
-   real crash/resume tests and captures their literal output and provenance. It
+   one independent Node black-box SIGKILL/CLI-resume case plus real Rust
+   crash/resume tests, and captures their literal output and provenance. It
    shows which Relayflows invariants held for the tested commit and host.
 2. `workflow-reliability/protocol.json` is a vendor-neutral competitive protocol.
    It defines the same workloads and success predicates for Relayflows, a
@@ -36,7 +37,7 @@ the report records the suite hash, source commit, toolchain, and host class.
 
 ## Claim discipline
 
-- A passing durability artifact supports only its six named claims. It is not a
+- A passing durability artifact supports only its seven named claims. It is not a
   general proof that every execution path is deterministic.
 - A build-vs-buy claim requires both `relayflows` and `diy-reference` results.
 - A “best in class” claim requires all participants named by the protocol.
diff --git a/benchmarks/durability/black-box-crash.mjs b/benchmarks/durability/black-box-crash.mjs
new file mode 100644
index 000000000..a18c91f71
--- /dev/null
+++ b/benchmarks/durability/black-box-crash.mjs
@@ -0,0 +1,181 @@
+#!/usr/bin/env node
+
+import assert from 'node:assert/strict';
+import { spawn, spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { once } from 'node:events';
+
+class EnvironmentError extends Error {}
+
+export async function runBlackBoxCrash(options = {}) {
+  if (process.platform === 'win32') {
+    throw new EnvironmentError('SIGKILL process semantics are required');
+  }
+  const repoRoot = resolve(options.repoRoot ?? fileURLToPath(new URL('../..', import.meta.url)));
+  const binary = options.binary ?? resolveRelayflowd(repoRoot);
+  const fixture = mkdtempSync(join(tmpdir(), 'relayflows-black-box-'));
+  const dataDir = join(fixture, 'data');
+  const effects = join(fixture, 'effects.txt');
+  const specPath = join(fixture, 'flow.json');
+  const assertions = [];
+
+  try {
+    writeFileSync(specPath, `${JSON.stringify(flowSpec(effects), null, 2)}\n`);
+    const child = spawn(
+      binary,
+      [
+        '--data-dir',
+        dataDir,
+        'run',
+        specPath,
+        '--pause-before-step',
+        'second',
+      ],
+      { detached: true, stdio: ['ignore', 'pipe', 'pipe'] },
+    );
+    let stdout = '';
+    let stderr = '';
+    let spawnError = null;
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+    child.on('error', (error) => {
+      spawnError = error;
+      stderr += String(error);
+    });
+
+    await waitUntil('first effect and durable run receipt', () => {
+      if (spawnError) throw new EnvironmentError(spawnError.message);
+      if (child.exitCode !== null) {
+        throw new Error(`relayflowd exited before the kill boundary: ${stdout}${stderr}`);
+      }
+      return readEffects(effects).join(',') === 'first' && runIds(dataDir).length === 1;
+    });
+    check(assertions, readEffects(effects), ['first'], 'only step one completed before SIGKILL');
+    const ids = runIds(dataDir);
+    check(assertions, ids.length, 1, 'one opaque run receipt exists');
+
+    const exited = once(child, 'exit');
+    process.kill(-child.pid, 'SIGKILL');
+    const [exitCode, signal] = await exited;
+    check(assertions, exitCode, null, 'killed process has no numeric exit code');
+    check(assertions, signal, 'SIGKILL', 'workflow process observed SIGKILL');
+
+    const resumed = spawnSync(binary, ['--data-dir', dataDir, 'resume', ids[0]], {
+      encoding: 'utf8',
+      timeout: 30000,
+    });
+    if (resumed.error) throw new EnvironmentError(resumed.error.message);
+    check(assertions, resumed.status, 0, `resume exited zero: ${resumed.stderr}`);
+    const outcome = JSON.parse(resumed.stdout);
+    check(assertions, outcome.status, 'completed', 'resume completed the run');
+    check(assertions, outcome.completion_reason, 'success', 'completion reason is declared success');
+    check(assertions, outcome.completed_steps, 3, 'all three steps completed');
+    check(
+      assertions,
+      readEffects(effects),
+      ['first', 'second', 'third'],
+      'every external effect occurred exactly once and in dependency order',
+    );
+
+    return {
+      assertions: assertions.length,
+      runStatus: outcome.status,
+      completionReason: outcome.completion_reason,
+      effects: readEffects(effects),
+    };
+  } finally {
+    rmSync(fixture, { recursive: true, force: true });
+  }
+}
+
+function resolveRelayflowd(repoRoot) {
+  const kernel = join(repoRoot, 'kernel');
+  const cargo = join(repoRoot, 'ops', 'cargo.sh');
+  runEnvironmentCommand(cargo, ['build', '-p', 'relayflowd'], kernel);
+  const metadata = runEnvironmentCommand(
+    cargo,
+    ['metadata', '--format-version', '1', '--no-deps'],
+    kernel,
+  );
+  const targetDirectory = JSON.parse(metadata.stdout).target_directory;
+  return join(targetDirectory, 'debug', 'relayflowd');
+}
+
+function runEnvironmentCommand(file, args, cwd) {
+  const result = spawnSync(file, args, { cwd, encoding: 'utf8', timeout: 120000 });
+  if (result.error || result.status !== 0) {
+    throw new EnvironmentError(
+      `${file} ${args.join(' ')} failed: ${result.error?.message ?? result.stderr ?? result.stdout}`,
+    );
+  }
+  return result;
+}
+
+function flowSpec(effects) {
+  const append = "require('node:fs').appendFileSync(process.argv[1], process.argv[2])";
+  const step = (id, value, dependsOn) => ({
+    id,
+    type: 'deterministic',
+    ...(dependsOn ? { depends_on: [dependsOn] } : {}),
+    command: [process.execPath, '-e', append, effects, `${value}\n`],
+  });
+  return {
+    name: 'black-box-crash-resume',
+    steps: [step('first', 'first'), step('second', 'second', 'first'), step('third', 'third', 'second')],
+  };
+}
+
+function runIds(dataDir) {
+  try {
+    return readdirSync(join(dataDir, 'runs'))
+      .filter((name) => name.endsWith('.sqlite3'))
+      .map((name) => name.slice(0, -'.sqlite3'.length));
+  } catch {
+    return [];
+  }
+}
+
+function readEffects(path) {
+  try {
+    return readFileSync(path, 'utf8').trimEnd().split('\n').filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+async function waitUntil(description, predicate) {
+  const deadline = Date.now() + 10000;
+  while (Date.now() < deadline) {
+    if (predicate()) return;
+    await new Promise((resolvePromise) => setTimeout(resolvePromise, 20));
+  }
+  throw new Error(`timed out waiting for ${description}`);
+}
+
+function check(assertions, actual, expected, description) {
+  assert.deepEqual(actual, expected, description);
+  assertions.push(description);
+}
+
+async function main() {
+  try {
+    const result = await runBlackBoxCrash();
+    process.stdout.write(`BLACK_BOX_ASSERTIONS: ${result.assertions}/${result.assertions}\n`);
+    process.stdout.write(`${JSON.stringify(result)}\n`);
+  } catch (error) {
+    const environment = error instanceof EnvironmentError;
+    process.stderr.write(
+      `${environment ? 'BENCHMARK_ENVIRONMENT_FAILURE' : 'BLACK_BOX_ASSERTION_FAILURE'}: ${error.stack ?? error}\n`,
+    );
+    process.exitCode = environment ? 2 : 1;
+  }
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) await main();
diff --git a/benchmarks/durability/suite.json b/benchmarks/durability/suite.json
index eca2cae67..cb52ef167 100644
--- a/benchmarks/durability/suite.json
+++ b/benchmarks/durability/suite.json
@@ -7,11 +7,27 @@
   "environmentFailurePatterns": [
     "CARGO_NOT_FOUND",
     "CARGO_BOOTSTRAP_FAILED",
+    "BENCHMARK_ENVIRONMENT_FAILURE",
     "failed to download",
     "Could not resolve host",
     "No space left on device"
   ],
   "cases": [
+    {
+      "id": "black-box-cli-sigkill-resume",
+      "claim": "An external caller can SIGKILL the CLI after the first effect, resume from its opaque run receipt, and observe each dependent effect exactly once.",
+      "command": {
+        "file": "node",
+        "args": ["black-box-crash.mjs"],
+        "cwd": ".",
+        "timeoutMs": 150000
+      },
+      "successWitnesses": [
+        {"stream": "stdout", "contains": "BLACK_BOX_ASSERTIONS: 9/9"},
+        {"stream": "stdout", "contains": "\"completionReason\":\"success\""},
+        {"stream": "stdout", "contains": "\"effects\":[\"first\",\"second\",\"third\"]"}
+      ]
+    },
     {
       "id": "deterministic-boundary-resume",
       "claim": "A deterministic three-step flow survives a kill at every boundary without replaying a completed effect.",

From ec7464109e93ec75bf63f32ef690aa743d532bf9 Mon Sep 17 00:00:00 2001
From: kjgbot <kjgbot@agentrelay.dev>
Date: Wed, 23 Sep 2026 13:02:55 -0700
Subject: [PATCH 4/8] fix: model complete environment loss in black-box eval

---
 benchmarks/README.md                      |   6 ++
 benchmarks/durability/black-box-crash.mjs | 101 ++++++++++++++++++----
 benchmarks/durability/suite.json          |   5 +-
 3 files changed, 92 insertions(+), 20 deletions(-)

diff --git a/benchmarks/README.md b/benchmarks/README.md
index c73ddc48e..3e938ee8d 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -16,6 +16,12 @@ exactly-once declared effects under the tested faults. It does not mean an LLM
 will emit identical text twice. Quality is measured separately by deterministic
 gates and task-specific evals, matching RFC-0001 settled decision 11.
 
+The black-box case models loss of an execution environment by killing both the
+daemon and its in-flight child process before resume. Killing only the daemon
+while deliberately leaving its child alive is not environment loss: the old
+attempt can continue beside its replacement, so that is a different workload
+with different ownership requirements.
+
 ## Run the durability eval
 
 From the repository root:
diff --git a/benchmarks/durability/black-box-crash.mjs b/benchmarks/durability/black-box-crash.mjs
index a18c91f71..d8f8d15c8 100644
--- a/benchmarks/durability/black-box-crash.mjs
+++ b/benchmarks/durability/black-box-crash.mjs
@@ -19,21 +19,20 @@ export async function runBlackBoxCrash(options = {}) {
   const fixture = mkdtempSync(join(tmpdir(), 'relayflows-black-box-'));
   const dataDir = join(fixture, 'data');
   const effects = join(fixture, 'effects.txt');
+  const attempts = join(fixture, 'attempts.txt');
+  const gate = join(fixture, 'gate');
+  const stepPid = join(fixture, 'step.pid');
   const specPath = join(fixture, 'flow.json');
   const assertions = [];
 
   try {
-    writeFileSync(specPath, `${JSON.stringify(flowSpec(effects), null, 2)}\n`);
+    writeFileSync(
+      specPath,
+      `${JSON.stringify(flowSpec({ effects, attempts, gate, stepPid }), null, 2)}\n`,
+    );
     const child = spawn(
       binary,
-      [
-        '--data-dir',
-        dataDir,
-        'run',
-        specPath,
-        '--pause-before-step',
-        'second',
-      ],
+      ['--data-dir', dataDir, 'run', specPath],
       { detached: true, stdio: ['ignore', 'pipe', 'pipe'] },
     );
     let stdout = '';
@@ -50,14 +49,20 @@ export async function runBlackBoxCrash(options = {}) {
       stderr += String(error);
     });
 
-    await waitUntil('first effect and durable run receipt', () => {
+    await waitUntil('completed first step and blocked second attempt', () => {
       if (spawnError) throw new EnvironmentError(spawnError.message);
       if (child.exitCode !== null) {
         throw new Error(`relayflowd exited before the kill boundary: ${stdout}${stderr}`);
       }
-      return readEffects(effects).join(',') === 'first' && runIds(dataDir).length === 1;
+      return (
+        readLines(effects).join(',') === 'first' &&
+        readLines(attempts).join(',') === 'attempt' &&
+        readPid(stepPid) !== null &&
+        runIds(dataDir).length === 1
+      );
     });
-    check(assertions, readEffects(effects), ['first'], 'only step one completed before SIGKILL');
+    check(assertions, readLines(effects), ['first'], 'only step one completed before SIGKILL');
+    check(assertions, readLines(attempts), ['attempt'], 'step two was in flight before SIGKILL');
     const ids = runIds(dataDir);
     check(assertions, ids.length, 1, 'one opaque run receipt exists');
 
@@ -66,6 +71,9 @@ export async function runBlackBoxCrash(options = {}) {
     const [exitCode, signal] = await exited;
     check(assertions, exitCode, null, 'killed process has no numeric exit code');
     check(assertions, signal, 'SIGKILL', 'workflow process observed SIGKILL');
+    await killAndWait(readPid(stepPid));
+    check(assertions, isProcessAlive(readPid(stepPid)), false, 'in-flight step process was killed');
+    writeFileSync(gate, 'open\n');
 
     const resumed = spawnSync(binary, ['--data-dir', dataDir, 'resume', ids[0]], {
       encoding: 'utf8',
@@ -79,16 +87,23 @@ export async function runBlackBoxCrash(options = {}) {
     check(assertions, outcome.completed_steps, 3, 'all three steps completed');
     check(
       assertions,
-      readEffects(effects),
+      readLines(attempts),
+      ['attempt', 'attempt'],
+      'the interrupted unfinished step executed a replacement attempt',
+    );
+    check(
+      assertions,
+      readLines(effects),
       ['first', 'second', 'third'],
-      'every external effect occurred exactly once and in dependency order',
+      'the completed step was not replayed and final effects occurred once in dependency order',
     );
 
     return {
       assertions: assertions.length,
       runStatus: outcome.status,
       completionReason: outcome.completion_reason,
-      effects: readEffects(effects),
+      attempts: readLines(attempts).length,
+      effects: readLines(effects),
     };
   } finally {
     rmSync(fixture, { recursive: true, force: true });
@@ -118,7 +133,7 @@ function runEnvironmentCommand(file, args, cwd) {
   return result;
 }
 
-function flowSpec(effects) {
+function flowSpec({ effects, attempts, gate, stepPid }) {
   const append = "require('node:fs').appendFileSync(process.argv[1], process.argv[2])";
   const step = (id, value, dependsOn) => ({
     id,
@@ -126,9 +141,29 @@ function flowSpec(effects) {
     ...(dependsOn ? { depends_on: [dependsOn] } : {}),
     command: [process.execPath, '-e', append, effects, `${value}\n`],
   });
+  const blockingSecond = [
+    process.execPath,
+    '-e',
+    [
+      "const fs=require('node:fs')",
+      "fs.appendFileSync(process.argv[3], 'attempt\\n')",
+      "fs.writeFileSync(process.argv[4], String(process.pid))",
+      'const wait=new Int32Array(new SharedArrayBuffer(4))',
+      'while(!fs.existsSync(process.argv[2])) Atomics.wait(wait,0,0,20)',
+      "fs.appendFileSync(process.argv[1], 'second\\n')",
+    ].join(';'),
+    effects,
+    gate,
+    attempts,
+    stepPid,
+  ];
   return {
     name: 'black-box-crash-resume',
-    steps: [step('first', 'first'), step('second', 'second', 'first'), step('third', 'third', 'second')],
+    steps: [
+      step('first', 'first'),
+      { id: 'second', type: 'deterministic', depends_on: ['first'], command: blockingSecond },
+      step('third', 'third', 'second'),
+    ],
   };
 }
 
@@ -142,7 +177,7 @@ function runIds(dataDir) {
   }
 }
 
-function readEffects(path) {
+function readLines(path) {
   try {
     return readFileSync(path, 'utf8').trimEnd().split('\n').filter(Boolean);
   } catch {
@@ -150,6 +185,36 @@ function readEffects(path) {
   }
 }
 
+function readPid(path) {
+  try {
+    const pid = Number(readFileSync(path, 'utf8'));
+    return Number.isInteger(pid) && pid > 0 ? pid : null;
+  } catch {
+    return null;
+  }
+}
+
+async function killAndWait(pid) {
+  if (!pid) throw new Error('in-flight step PID is missing');
+  try {
+    process.kill(pid, 'SIGKILL');
+  } catch (error) {
+    if (error.code !== 'ESRCH') throw error;
+  }
+  await waitUntil('in-flight step process exit', () => !isProcessAlive(pid));
+}
+
+function isProcessAlive(pid) {
+  if (!pid) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    if (error.code === 'ESRCH') return false;
+    throw error;
+  }
+}
+
 async function waitUntil(description, predicate) {
   const deadline = Date.now() + 10000;
   while (Date.now() < deadline) {
diff --git a/benchmarks/durability/suite.json b/benchmarks/durability/suite.json
index cb52ef167..e58e3ddcf 100644
--- a/benchmarks/durability/suite.json
+++ b/benchmarks/durability/suite.json
@@ -15,7 +15,7 @@
   "cases": [
     {
       "id": "black-box-cli-sigkill-resume",
-      "claim": "An external caller can SIGKILL the CLI after the first effect, resume from its opaque run receipt, and observe each dependent effect exactly once.",
+      "claim": "An external caller can SIGKILL the CLI after one completed step while the next is in flight, then resume without replaying completed work while replacing the unfinished attempt.",
       "command": {
         "file": "node",
         "args": ["black-box-crash.mjs"],
@@ -23,8 +23,9 @@
         "timeoutMs": 150000
       },
       "successWitnesses": [
-        {"stream": "stdout", "contains": "BLACK_BOX_ASSERTIONS: 9/9"},
+        {"stream": "stdout", "contains": "BLACK_BOX_ASSERTIONS: 12/12"},
         {"stream": "stdout", "contains": "\"completionReason\":\"success\""},
+        {"stream": "stdout", "contains": "\"attempts\":2"},
         {"stream": "stdout", "contains": "\"effects\":[\"first\",\"second\",\"third\"]"}
       ]
     },

From c4092182870fe148afe673da38358ad7dd20c1e3 Mon Sep 17 00:00:00 2001
From: kjgbot <kjgbot@agentrelay.dev>
Date: Wed, 23 Sep 2026 13:35:51 -0700
Subject: [PATCH 5/8] Harden eval publication evidence

---
 benchmarks/README.md                      |  14 +
 benchmarks/durability/black-box-crash.mjs | 101 ++++---
 benchmarks/durability/suite.json          |   2 +-
 scripts/eval-provenance.mjs               |  51 ++++
 scripts/eval-suite.mjs                    | 318 ++++++++++++++++++++++
 scripts/run-evals.mjs                     | 299 +-------------------
 scripts/run-evals.test.mjs                |  74 ++++-
 7 files changed, 527 insertions(+), 332 deletions(-)
 create mode 100644 scripts/eval-provenance.mjs
 create mode 100644 scripts/eval-suite.mjs

diff --git a/benchmarks/README.md b/benchmarks/README.md
index 3e938ee8d..25af45681 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -41,6 +41,20 @@ output witnesses proving each selected test actually ran. Every trial records
 its argv, working directory, duration, outcome, exit code, stdout, and stderr;
 the report records the suite hash, source commit, toolchain, and host class.
 
+Trial processes receive and record a small allowlist of ambient variables
+(`HOME`, locale, `PATH`, temporary-directory variables, shell, and Relayflows
+toolchain controls), plus any explicit per-case environment. Credentials and
+unrelated ambient state are neither inherited nor written to the report. A
+command timeout is a product failure blocker (`trial_timeouts`), not an
+environmental excuse.
+
+The black-box crash case stays on the supported CLI boundary. It obtains an
+opaque run receipt from `run --stop-after 1`, resumes that run, kills the
+workflow while the next step is in flight, then resumes again using the receipt.
+It does not inspect journal files or storage layout. Its fixed, named assertion
+set is emitted in the report, so the witness cannot pass by merely agreeing
+with its own dynamic count.
+
 ## Claim discipline
 
 - A passing durability artifact supports only its seven named claims. It is not a
diff --git a/benchmarks/durability/black-box-crash.mjs b/benchmarks/durability/black-box-crash.mjs
index d8f8d15c8..ed38c66ca 100644
--- a/benchmarks/durability/black-box-crash.mjs
+++ b/benchmarks/durability/black-box-crash.mjs
@@ -1,12 +1,14 @@
 #!/usr/bin/env node
 
 import assert from 'node:assert/strict';
+import { once } from 'node:events';
 import { spawn, spawnSync } from 'node:child_process';
-import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { dirname, join, resolve } from 'node:path';
+import { join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { once } from 'node:events';
+
+const EXPECTED_ASSERTIONS = 16;
 
 class EnvironmentError extends Error {}
 
@@ -30,9 +32,33 @@ export async function runBlackBoxCrash(options = {}) {
       specPath,
       `${JSON.stringify(flowSpec({ effects, attempts, gate, stepPid }), null, 2)}\n`,
     );
+    const initial = spawnSync(
+      binary,
+      ['--data-dir', dataDir, 'run', specPath, '--stop-after', '1'],
+      { encoding: 'utf8', timeout: 30000 },
+    );
+    if (initial.error) throw new EnvironmentError(initial.error.message);
+    check(assertions, initial.status, 0, `initial run exited zero: ${initial.stderr}`);
+    const receipt = parseOutcome(initial.stdout);
+    check(assertions, receipt.status, 'interrupted', 'public run receipt reports interruption');
+    check(
+      assertions,
+      receipt.completion_reason,
+      null,
+      'interrupted receipt has no completion reason',
+    );
+    check(assertions, receipt.completed_steps, 1, 'public run receipt reports one completed step');
+    check(
+      assertions,
+      typeof receipt.run_id === 'string' && receipt.run_id.length > 0,
+      true,
+      'public run receipt includes a run id',
+    );
+    check(assertions, readLines(effects), ['first'], 'only step one completed before resume');
+
     const child = spawn(
       binary,
-      ['--data-dir', dataDir, 'run', specPath],
+      ['--data-dir', dataDir, 'resume', receipt.run_id],
       { detached: true, stdio: ['ignore', 'pipe', 'pipe'] },
     );
     let stdout = '';
@@ -49,41 +75,38 @@ export async function runBlackBoxCrash(options = {}) {
       stderr += String(error);
     });
 
-    await waitUntil('completed first step and blocked second attempt', () => {
+    await waitUntil('the resumed second step to be in flight', () => {
       if (spawnError) throw new EnvironmentError(spawnError.message);
       if (child.exitCode !== null) {
         throw new Error(`relayflowd exited before the kill boundary: ${stdout}${stderr}`);
       }
-      return (
-        readLines(effects).join(',') === 'first' &&
-        readLines(attempts).join(',') === 'attempt' &&
-        readPid(stepPid) !== null &&
-        runIds(dataDir).length === 1
-      );
+      return readLines(attempts).join(',') === 'attempt' && readPid(stepPid) !== null;
     });
-    check(assertions, readLines(effects), ['first'], 'only step one completed before SIGKILL');
     check(assertions, readLines(attempts), ['attempt'], 'step two was in flight before SIGKILL');
-    const ids = runIds(dataDir);
-    check(assertions, ids.length, 1, 'one opaque run receipt exists');
 
     const exited = once(child, 'exit');
     process.kill(-child.pid, 'SIGKILL');
     const [exitCode, signal] = await exited;
-    check(assertions, exitCode, null, 'killed process has no numeric exit code');
-    check(assertions, signal, 'SIGKILL', 'workflow process observed SIGKILL');
+    check(assertions, exitCode, null, 'killed resume process has no numeric exit code');
+    check(assertions, signal, 'SIGKILL', 'resume process observed SIGKILL');
     await killAndWait(readPid(stepPid));
     check(assertions, isProcessAlive(readPid(stepPid)), false, 'in-flight step process was killed');
     writeFileSync(gate, 'open\n');
 
-    const resumed = spawnSync(binary, ['--data-dir', dataDir, 'resume', ids[0]], {
+    const resumed = spawnSync(binary, ['--data-dir', dataDir, 'resume', receipt.run_id], {
       encoding: 'utf8',
       timeout: 30000,
     });
     if (resumed.error) throw new EnvironmentError(resumed.error.message);
-    check(assertions, resumed.status, 0, `resume exited zero: ${resumed.stderr}`);
-    const outcome = JSON.parse(resumed.stdout);
-    check(assertions, outcome.status, 'completed', 'resume completed the run');
-    check(assertions, outcome.completion_reason, 'success', 'completion reason is declared success');
+    check(assertions, resumed.status, 0, `final resume exited zero: ${resumed.stderr}`);
+    const outcome = parseOutcome(resumed.stdout);
+    check(assertions, outcome.status, 'completed', 'final resume completed the run');
+    check(
+      assertions,
+      outcome.completion_reason,
+      'success',
+      'completion reason is declared success',
+    );
     check(assertions, outcome.completed_steps, 3, 'all three steps completed');
     check(
       assertions,
@@ -97,9 +120,12 @@ export async function runBlackBoxCrash(options = {}) {
       ['first', 'second', 'third'],
       'the completed step was not replayed and final effects occurred once in dependency order',
     );
+    assert.equal(assertions.length, EXPECTED_ASSERTIONS, 'harness assertion count changed');
 
     return {
-      assertions: assertions.length,
+      assertionsPassed: assertions.length,
+      assertionsExpected: EXPECTED_ASSERTIONS,
+      assertionNames: assertions,
       runStatus: outcome.status,
       completionReason: outcome.completion_reason,
       attempts: readLines(attempts).length,
@@ -110,6 +136,21 @@ export async function runBlackBoxCrash(options = {}) {
   }
 }
 
+export function parseOutcome(stdout) {
+  const lines = String(stdout).split('\n').map((line) => line.trim()).filter(Boolean).reverse();
+  for (const line of lines) {
+    try {
+      const outcome = JSON.parse(line);
+      if (outcome && typeof outcome === 'object' && typeof outcome.run_id === 'string') {
+        return outcome;
+      }
+    } catch {
+      // Non-JSON CLI diagnostics are allowed; the last valid receipt is authoritative.
+    }
+  }
+  throw new Error(`relayflowd stdout did not contain a JSON run receipt: ${stdout}`);
+}
+
 function resolveRelayflowd(repoRoot) {
   const kernel = join(repoRoot, 'kernel');
   const cargo = join(repoRoot, 'ops', 'cargo.sh');
@@ -147,7 +188,7 @@ function flowSpec({ effects, attempts, gate, stepPid }) {
     [
       "const fs=require('node:fs')",
       "fs.appendFileSync(process.argv[3], 'attempt\\n')",
-      "fs.writeFileSync(process.argv[4], String(process.pid))",
+      'fs.writeFileSync(process.argv[4], String(process.pid))',
       'const wait=new Int32Array(new SharedArrayBuffer(4))',
       'while(!fs.existsSync(process.argv[2])) Atomics.wait(wait,0,0,20)',
       "fs.appendFileSync(process.argv[1], 'second\\n')",
@@ -167,16 +208,6 @@ function flowSpec({ effects, attempts, gate, stepPid }) {
   };
 }
 
-function runIds(dataDir) {
-  try {
-    return readdirSync(join(dataDir, 'runs'))
-      .filter((name) => name.endsWith('.sqlite3'))
-      .map((name) => name.slice(0, -'.sqlite3'.length));
-  } catch {
-    return [];
-  }
-}
-
 function readLines(path) {
   try {
     return readFileSync(path, 'utf8').trimEnd().split('\n').filter(Boolean);
@@ -232,7 +263,9 @@ function check(assertions, actual, expected, description) {
 async function main() {
   try {
     const result = await runBlackBoxCrash();
-    process.stdout.write(`BLACK_BOX_ASSERTIONS: ${result.assertions}/${result.assertions}\n`);
+    process.stdout.write(
+      `BLACK_BOX_ASSERTIONS: ${result.assertionsPassed}/${result.assertionsExpected}\n`,
+    );
     process.stdout.write(`${JSON.stringify(result)}\n`);
   } catch (error) {
     const environment = error instanceof EnvironmentError;
diff --git a/benchmarks/durability/suite.json b/benchmarks/durability/suite.json
index e58e3ddcf..a65f28c62 100644
--- a/benchmarks/durability/suite.json
+++ b/benchmarks/durability/suite.json
@@ -23,7 +23,7 @@
         "timeoutMs": 150000
       },
       "successWitnesses": [
-        {"stream": "stdout", "contains": "BLACK_BOX_ASSERTIONS: 12/12"},
+        {"stream": "stdout", "contains": "BLACK_BOX_ASSERTIONS: 16/16"},
         {"stream": "stdout", "contains": "\"completionReason\":\"success\""},
         {"stream": "stdout", "contains": "\"attempts\":2"},
         {"stream": "stdout", "contains": "\"effects\":[\"first\",\"second\",\"third\"]"}
diff --git a/scripts/eval-provenance.mjs b/scripts/eval-provenance.mjs
new file mode 100644
index 000000000..2e4e673d5
--- /dev/null
+++ b/scripts/eval-provenance.mjs
@@ -0,0 +1,51 @@
+import { spawnSync } from 'node:child_process';
+import { cpus, platform, release } from 'node:os';
+import { resolve } from 'node:path';
+
+export function collectProvenance(rootDir, execute = spawnSync) {
+  const gitCommit = commandResult(execute, 'git', ['rev-parse', 'HEAD'], rootDir);
+  const gitStatus = commandResult(execute, 'git', ['status', '--porcelain=v1'], rootDir);
+  const gitRoot = commandResult(execute, 'git', ['rev-parse', '--show-toplevel'], rootDir);
+  const cpu = cpus();
+  return {
+    source: {
+      commit: gitCommit.ok ? gitCommit.stdout : null,
+      root: gitRoot.ok ? resolve(gitRoot.stdout) : rootDir,
+      dirty: !gitCommit.ok || !gitStatus.ok || !gitRoot.ok || gitStatus.stdout.length > 0,
+      statusAvailable: gitStatus.ok,
+      statusPorcelain: gitStatus.stdout,
+    },
+    runtime: {
+      node: process.version,
+      nodePath: process.execPath,
+      rustc: commandText(execute, 'rustc', ['--version'], rootDir) || null,
+      cargo: commandText(execute, resolve(rootDir, 'ops/cargo.sh'), ['--version'], rootDir) || null,
+    },
+    host: {
+      platform: platform(),
+      release: release(),
+      arch: process.arch,
+      cpuModel: cpu[0]?.model ?? null,
+      logicalCpus: cpu.length,
+    },
+  };
+}
+
+function commandText(execute, file, args, cwd) {
+  const result = execute(file, args, { cwd, encoding: 'utf8' });
+  if (result.status !== 0) return '';
+  return text(result.stdout).trim();
+}
+
+function commandResult(execute, file, args, cwd) {
+  const result = execute(file, args, { cwd, encoding: 'utf8' });
+  return {
+    ok: result.status === 0,
+    stdout: result.status === 0 ? text(result.stdout).trim() : '',
+  };
+}
+
+function text(value) {
+  if (value === undefined || value === null) return '';
+  return Buffer.isBuffer(value) ? value.toString('utf8') : String(value);
+}
diff --git a/scripts/eval-suite.mjs b/scripts/eval-suite.mjs
new file mode 100644
index 000000000..edd606894
--- /dev/null
+++ b/scripts/eval-suite.mjs
@@ -0,0 +1,318 @@
+import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { dirname, relative, resolve } from 'node:path';
+import { performance } from 'node:perf_hooks';
+
+import { collectProvenance } from './eval-provenance.mjs';
+
+const RESULT_SCHEMA_VERSION = 1;
+const INHERITED_ENV_KEYS = Object.freeze([
+  'CARGO_HOME',
+  'CARGO_TARGET_DIR',
+  'HOME',
+  'LANG',
+  'LC_ALL',
+  'PATH',
+  'RELAYFLOWS_NO_TOOLCHAIN_INSTALL',
+  'RELAYFLOWS_TOOLCHAIN_HOME',
+  'RUSTUP_HOME',
+  'RUSTUP_TOOLCHAIN',
+  'SHELL',
+  'TEMP',
+  'TMP',
+  'TMPDIR',
+]);
+
+export function validateEvalSuite(value) {
+  if (!value || typeof value !== 'object') throw new Error('eval suite must be an object');
+  if (value.schemaVersion !== 1) throw new Error('eval suite schemaVersion must be 1');
+  if (value.kind !== 'relayflows-eval') throw new Error('eval suite kind must be relayflows-eval');
+  requireNonEmptyString(value.id, 'eval suite id');
+  requirePositiveInteger(value.defaultRepetitions, 'defaultRepetitions');
+  requirePositiveInteger(value.minimumPublishableRepetitions, 'minimumPublishableRepetitions');
+  if (!Array.isArray(value.environmentFailurePatterns)) {
+    throw new Error('environmentFailurePatterns must be an array');
+  }
+  value.environmentFailurePatterns.forEach((pattern, index) =>
+    requireNonEmptyString(pattern, `environmentFailurePatterns[${index}]`),
+  );
+  if (!Array.isArray(value.cases) || value.cases.length === 0) {
+    throw new Error('eval suite cases must be a non-empty array');
+  }
+
+  const ids = new Set();
+  for (const [caseIndex, testCase] of value.cases.entries()) {
+    validateCase(testCase, caseIndex, ids);
+    ids.add(testCase.id);
+  }
+  return value;
+}
+
+function validateCase(testCase, caseIndex, ids) {
+  if (!testCase || typeof testCase !== 'object') {
+    throw new Error(`cases[${caseIndex}] must be an object`);
+  }
+  requireNonEmptyString(testCase.id, `cases[${caseIndex}].id`);
+  if (ids.has(testCase.id)) throw new Error(`duplicate eval case id: ${testCase.id}`);
+  requireNonEmptyString(testCase.claim, `cases[${caseIndex}].claim`);
+  if (!testCase.command || typeof testCase.command !== 'object') {
+    throw new Error(`cases[${caseIndex}].command must be an object`);
+  }
+  requireNonEmptyString(testCase.command.file, `cases[${caseIndex}].command.file`);
+  if (!Array.isArray(testCase.command.args) || !testCase.command.args.every(isString)) {
+    throw new Error(`cases[${caseIndex}].command.args must be an array of strings`);
+  }
+  requireNonEmptyString(testCase.command.cwd, `cases[${caseIndex}].command.cwd`);
+  requirePositiveInteger(testCase.command.timeoutMs, `cases[${caseIndex}].command.timeoutMs`);
+  if (testCase.command.env !== undefined) validateCommandEnv(testCase.command.env, caseIndex);
+  if (!Array.isArray(testCase.successWitnesses) || testCase.successWitnesses.length === 0) {
+    throw new Error(`cases[${caseIndex}].successWitnesses must be a non-empty array`);
+  }
+  for (const [witnessIndex, witness] of testCase.successWitnesses.entries()) {
+    if (!witness || !['stdout', 'stderr'].includes(witness.stream)) {
+      throw new Error(
+        `cases[${caseIndex}].successWitnesses[${witnessIndex}].stream must be stdout or stderr`,
+      );
+    }
+    requireNonEmptyString(
+      witness.contains,
+      `cases[${caseIndex}].successWitnesses[${witnessIndex}].contains`,
+    );
+  }
+}
+
+function validateCommandEnv(env, caseIndex) {
+  if (!env || typeof env !== 'object' || Array.isArray(env)) {
+    throw new Error(`cases[${caseIndex}].command.env must be an object`);
+  }
+  for (const [key, value] of Object.entries(env)) {
+    requireNonEmptyString(key, `cases[${caseIndex}].command.env key`);
+    if (typeof value !== 'string') {
+      throw new Error(`cases[${caseIndex}].command.env.${key} must be a string`);
+    }
+  }
+}
+
+export function runEvalSuite(options) {
+  const rootDir = resolve(options.rootDir ?? process.cwd());
+  const suitePath = resolve(rootDir, options.suitePath);
+  const suiteBytes = readFileSync(suitePath);
+  const suite = validateEvalSuite(JSON.parse(suiteBytes.toString('utf8')));
+  const repetitions = options.repetitions ?? suite.defaultRepetitions;
+  if (!Number.isInteger(repetitions) || repetitions < 1) {
+    throw new Error('repetitions must be a positive integer');
+  }
+
+  const execute = options.execute ?? spawnSync;
+  const provenance = options.provenance ?? collectProvenance(rootDir, execute);
+  validateProvenance(provenance, rootDir, options.allowDirty);
+  const environment = trialEnvironment(options.parentEnv ?? process.env);
+  const startedAt = new Date().toISOString();
+  const cases = suite.cases.map((testCase) =>
+    runCase({ testCase, suite, suitePath, rootDir, repetitions, execute, environment }),
+  );
+  return buildReport({ options, suite, suiteBytes, cases, repetitions, provenance, startedAt });
+}
+
+function validateProvenance(provenance, rootDir, allowDirty) {
+  if (resolve(provenance.source.root) !== rootDir) {
+    throw new Error(
+      `provenance root ${provenance.source.root} does not match requested root ${rootDir}`,
+    );
+  }
+  if (provenance.source.dirty && !allowDirty) {
+    throw new Error(
+      'refusing to run a publishable eval from a dirty tree; commit it or pass --allow-dirty',
+    );
+  }
+}
+
+function runCase({ testCase, suite, suitePath, rootDir, repetitions, execute, environment }) {
+  const commandCwd = resolve(dirname(suitePath), testCase.command.cwd);
+  if (!isWithin(rootDir, commandCwd)) {
+    throw new Error(`case ${testCase.id} command cwd escapes the provenance root`);
+  }
+  const commandEnvironment = { ...environment, ...testCase.command.env };
+  const trials = Array.from({ length: repetitions }, (_, index) =>
+    runTrial({
+      testCase,
+      repetition: index + 1,
+      commandCwd,
+      execute,
+      environment: commandEnvironment,
+      environmentFailurePatterns: suite.environmentFailurePatterns,
+    }),
+  );
+  return summarizeCase(testCase, trials);
+}
+
+function runTrial({
+  testCase,
+  repetition,
+  commandCwd,
+  execute,
+  environment,
+  environmentFailurePatterns,
+}) {
+  const before = performance.now();
+  const result = execute(testCase.command.file, testCase.command.args, {
+    cwd: commandCwd,
+    encoding: 'utf8',
+    timeout: testCase.command.timeoutMs,
+    maxBuffer: 64 * 1024 * 1024,
+    env: environment,
+  });
+  const durationMs = Math.round((performance.now() - before) * 1000) / 1000;
+  const stdout = text(result.stdout);
+  const stderr = text(result.stderr);
+  const missingWitnesses = testCase.successWitnesses.filter(
+    (witness) => !{ stdout, stderr }[witness.stream].includes(witness.contains),
+  );
+  const outcome = classifyTrial({
+    result,
+    stdout,
+    stderr,
+    missingWitnesses,
+    environmentFailurePatterns,
+  });
+  return {
+    repetition,
+    outcome,
+    passed: outcome === 'passed',
+    exitCode: result.status,
+    signal: result.signal ?? null,
+    durationMs,
+    command: {
+      file: testCase.command.file,
+      args: testCase.command.args,
+      cwd: testCase.command.cwd,
+      environment,
+    },
+    stdout,
+    stderr,
+    missingWitnesses,
+    error: result.error ? String(result.error.message ?? result.error) : null,
+  };
+}
+
+function classifyTrial({ result, stdout, stderr, missingWitnesses, environmentFailurePatterns }) {
+  if (result.status === 0) return missingWitnesses.length === 0 ? 'passed' : 'invalid';
+  if (result.error?.code === 'ETIMEDOUT') return 'timed_out';
+  const combined = `${stdout}\n${stderr}\n${result.error?.message ?? ''}`;
+  if (
+    result.status === 127 ||
+    environmentFailurePatterns.some((pattern) => combined.includes(pattern))
+  ) {
+    return 'inconclusive';
+  }
+  return 'failed';
+}
+
+function summarizeCase(testCase, trials) {
+  const durations = trials.filter((trial) => trial.passed).map((trial) => trial.durationMs);
+  const count = (outcome) => trials.filter((trial) => trial.outcome === outcome).length;
+  const passedTrials = count('passed');
+  return {
+    id: testCase.id,
+    claim: testCase.claim,
+    passedTrials,
+    failedTrials: count('failed'),
+    timedOutTrials: count('timed_out'),
+    invalidTrials: count('invalid'),
+    inconclusiveTrials: count('inconclusive'),
+    totalTrials: trials.length,
+    passRate: passedTrials / trials.length,
+    durationMs: {
+      sampleSize: durations.length,
+      p50: durations.length > 0 ? percentile(durations, 0.5) : null,
+      p95: durations.length >= 20 ? percentile(durations, 0.95) : null,
+    },
+    trials,
+  };
+}
+
+function buildReport({ options, suite, suiteBytes, cases, repetitions, provenance, startedAt }) {
+  const count = (field) => cases.reduce((sum, testCase) => sum + testCase[field], 0);
+  const summary = {
+    passedTrials: count('passedTrials'),
+    failedTrials: count('failedTrials'),
+    timedOutTrials: count('timedOutTrials'),
+    invalidTrials: count('invalidTrials'),
+    inconclusiveTrials: count('inconclusiveTrials'),
+    totalTrials: count('totalTrials'),
+  };
+  const blockers = publicationBlockers({ ...summary, repetitions, suite, provenance });
+  return {
+    schemaVersion: RESULT_SCHEMA_VERSION,
+    kind: 'relayflows-eval-result',
+    suite: {
+      id: suite.id,
+      path: options.suitePath,
+      sha256: createHash('sha256').update(suiteBytes).digest('hex'),
+    },
+    implementation: options.implementation ?? 'relayflows',
+    startedAt,
+    completedAt: new Date().toISOString(),
+    repetitions,
+    provenance,
+    summary: {
+      ...summary,
+      passRate: summary.passedTrials / summary.totalTrials,
+      publicationStatus: blockers.length === 0 ? 'eligible' : 'ineligible',
+      blockers,
+    },
+    cases,
+  };
+}
+
+function publicationBlockers(summary) {
+  const blockers = [];
+  if (summary.provenance.source.dirty) blockers.push('source_tree_dirty');
+  if (summary.repetitions < summary.suite.minimumPublishableRepetitions) {
+    blockers.push('insufficient_repetitions');
+  }
+  if (summary.failedTrials > 0) blockers.push('trial_failures');
+  if (summary.timedOutTrials > 0) blockers.push('trial_timeouts');
+  if (summary.invalidTrials > 0) blockers.push('execution_witness_missing');
+  if (summary.inconclusiveTrials > 0) blockers.push('environment_failures');
+  return blockers;
+}
+
+export function trialEnvironment(parentEnv, commandEnv = {}) {
+  const environment = {};
+  for (const key of INHERITED_ENV_KEYS) {
+    if (parentEnv[key] !== undefined) environment[key] = String(parentEnv[key]);
+  }
+  return { ...environment, ...commandEnv };
+}
+
+function percentile(values, quantile) {
+  const sorted = [...values].sort((left, right) => left - right);
+  const index = Math.max(0, Math.ceil(sorted.length * quantile) - 1);
+  return sorted[index];
+}
+
+export function isWithin(parent, child) {
+  const path = relative(parent, child);
+  return path === '' || (!path.startsWith('..') && !path.startsWith('/'));
+}
+
+function requireNonEmptyString(value, field) {
+  if (typeof value !== 'string' || value.length === 0) {
+    throw new Error(`${field} must be a non-empty string`);
+  }
+}
+
+function requirePositiveInteger(value, field) {
+  if (!Number.isInteger(value) || value < 1) throw new Error(`${field} must be a positive integer`);
+}
+
+function isString(value) {
+  return typeof value === 'string';
+}
+
+function text(value) {
+  if (value === undefined || value === null) return '';
+  return Buffer.isBuffer(value) ? value.toString('utf8') : String(value);
+}
diff --git a/scripts/run-evals.mjs b/scripts/run-evals.mjs
index 5cf4f31b5..102eafd10 100644
--- a/scripts/run-evals.mjs
+++ b/scripts/run-evals.mjs
@@ -1,295 +1,10 @@
 #!/usr/bin/env node
 
-import { spawnSync } from 'node:child_process';
-import { createHash } from 'node:crypto';
-import { readFileSync, writeFileSync } from 'node:fs';
-import { cpus, platform, release } from 'node:os';
-import { dirname, relative, resolve } from 'node:path';
+import { writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { performance } from 'node:perf_hooks';
 
-const RESULT_SCHEMA_VERSION = 1;
-
-export function validateEvalSuite(value) {
-  if (!value || typeof value !== 'object') throw new Error('eval suite must be an object');
-  if (value.schemaVersion !== 1) throw new Error('eval suite schemaVersion must be 1');
-  if (value.kind !== 'relayflows-eval') throw new Error('eval suite kind must be relayflows-eval');
-  requireNonEmptyString(value.id, 'eval suite id');
-  requirePositiveInteger(value.defaultRepetitions, 'defaultRepetitions');
-  requirePositiveInteger(
-    value.minimumPublishableRepetitions,
-    'minimumPublishableRepetitions',
-  );
-  if (!Array.isArray(value.environmentFailurePatterns)) {
-    throw new Error('environmentFailurePatterns must be an array');
-  }
-  value.environmentFailurePatterns.forEach((pattern, index) =>
-    requireNonEmptyString(pattern, `environmentFailurePatterns[${index}]`),
-  );
-  if (!Array.isArray(value.cases) || value.cases.length === 0) {
-    throw new Error('eval suite cases must be a non-empty array');
-  }
-
-  const ids = new Set();
-  for (const [caseIndex, testCase] of value.cases.entries()) {
-    if (!testCase || typeof testCase !== 'object') {
-      throw new Error(`cases[${caseIndex}] must be an object`);
-    }
-    requireNonEmptyString(testCase.id, `cases[${caseIndex}].id`);
-    if (ids.has(testCase.id)) throw new Error(`duplicate eval case id: ${testCase.id}`);
-    requireNonEmptyString(testCase.claim, `cases[${caseIndex}].claim`);
-    if (!testCase.command || typeof testCase.command !== 'object') {
-      throw new Error(`cases[${caseIndex}].command must be an object`);
-    }
-    requireNonEmptyString(testCase.command.file, `cases[${caseIndex}].command.file`);
-    if (
-      !Array.isArray(testCase.command.args) ||
-      !testCase.command.args.every((arg) => typeof arg === 'string')
-    ) {
-      throw new Error(`cases[${caseIndex}].command.args must be an array of strings`);
-    }
-    requireNonEmptyString(testCase.command.cwd, `cases[${caseIndex}].command.cwd`);
-    requirePositiveInteger(testCase.command.timeoutMs, `cases[${caseIndex}].command.timeoutMs`);
-    if (!Array.isArray(testCase.successWitnesses) || testCase.successWitnesses.length === 0) {
-      throw new Error(`cases[${caseIndex}].successWitnesses must be a non-empty array`);
-    }
-    for (const [witnessIndex, witness] of testCase.successWitnesses.entries()) {
-      if (!witness || !['stdout', 'stderr'].includes(witness.stream)) {
-        throw new Error(
-          `cases[${caseIndex}].successWitnesses[${witnessIndex}].stream must be stdout or stderr`,
-        );
-      }
-      requireNonEmptyString(
-        witness.contains,
-        `cases[${caseIndex}].successWitnesses[${witnessIndex}].contains`,
-      );
-    }
-    ids.add(testCase.id);
-  }
-  return value;
-}
-
-export function runEvalSuite(options) {
-  const rootDir = resolve(options.rootDir ?? process.cwd());
-  const suitePath = resolve(rootDir, options.suitePath);
-  const suiteBytes = readFileSync(suitePath);
-  const suite = validateEvalSuite(JSON.parse(suiteBytes.toString('utf8')));
-  const repetitions = options.repetitions ?? suite.defaultRepetitions;
-  if (!Number.isInteger(repetitions) || repetitions < 1) {
-    throw new Error('repetitions must be a positive integer');
-  }
-
-  const execute = options.execute ?? spawnSync;
-  const provenance = options.provenance ?? collectProvenance(rootDir, execute);
-  if (resolve(provenance.source.root) !== rootDir) {
-    throw new Error(
-      `provenance root ${provenance.source.root} does not match requested root ${rootDir}`,
-    );
-  }
-  if (provenance.source.dirty && !options.allowDirty) {
-    throw new Error('refusing to run a publishable eval from a dirty tree; commit it or pass --allow-dirty');
-  }
-
-  const startedAt = new Date().toISOString();
-  const cases = suite.cases.map((testCase) => {
-    const commandCwd = resolve(dirname(suitePath), testCase.command.cwd);
-    if (!isWithin(rootDir, commandCwd)) {
-      throw new Error(`case ${testCase.id} command cwd escapes the provenance root`);
-    }
-    const trials = [];
-    for (let repetition = 1; repetition <= repetitions; repetition += 1) {
-      const before = performance.now();
-      const result = execute(testCase.command.file, testCase.command.args, {
-        cwd: commandCwd,
-        encoding: 'utf8',
-        timeout: testCase.command.timeoutMs,
-        maxBuffer: 64 * 1024 * 1024,
-        env: process.env,
-      });
-      const durationMs = Math.round((performance.now() - before) * 1000) / 1000;
-      const stdout = text(result.stdout);
-      const stderr = text(result.stderr);
-      const missingWitnesses = testCase.successWitnesses.filter(
-        (witness) => !{ stdout, stderr }[witness.stream].includes(witness.contains),
-      );
-      const outcome = classifyTrial({
-        result,
-        stdout,
-        stderr,
-        missingWitnesses,
-        environmentFailurePatterns: suite.environmentFailurePatterns,
-      });
-      trials.push({
-        repetition,
-        outcome,
-        passed: outcome === 'passed',
-        exitCode: result.status,
-        signal: result.signal ?? null,
-        durationMs,
-        command: {
-          file: testCase.command.file,
-          args: testCase.command.args,
-          cwd: testCase.command.cwd,
-        },
-        stdout,
-        stderr,
-        missingWitnesses,
-        error: result.error ? String(result.error.message ?? result.error) : null,
-      });
-    }
-    const durations = trials.filter((trial) => trial.passed).map((trial) => trial.durationMs);
-    const passedTrials = trials.filter((trial) => trial.passed).length;
-    const failedTrials = trials.filter((trial) => trial.outcome === 'failed').length;
-    const invalidTrials = trials.filter((trial) => trial.outcome === 'invalid').length;
-    const inconclusiveTrials = trials.filter((trial) => trial.outcome === 'inconclusive').length;
-    return {
-      id: testCase.id,
-      claim: testCase.claim,
-      passedTrials,
-      failedTrials,
-      invalidTrials,
-      inconclusiveTrials,
-      totalTrials: trials.length,
-      passRate: passedTrials / trials.length,
-      durationMs: {
-        sampleSize: durations.length,
-        p50: durations.length > 0 ? percentile(durations, 0.5) : null,
-        p95: durations.length >= 20 ? percentile(durations, 0.95) : null,
-      },
-      trials,
-    };
-  });
-
-  const totalTrials = cases.reduce((sum, testCase) => sum + testCase.totalTrials, 0);
-  const passedTrials = cases.reduce((sum, testCase) => sum + testCase.passedTrials, 0);
-  const failedTrials = cases.reduce((sum, testCase) => sum + testCase.failedTrials, 0);
-  const invalidTrials = cases.reduce((sum, testCase) => sum + testCase.invalidTrials, 0);
-  const inconclusiveTrials = cases.reduce(
-    (sum, testCase) => sum + testCase.inconclusiveTrials,
-    0,
-  );
-  const blockers = [];
-  if (provenance.source.dirty) blockers.push('source_tree_dirty');
-  if (repetitions < suite.minimumPublishableRepetitions) blockers.push('insufficient_repetitions');
-  if (failedTrials > 0) blockers.push('trial_failures');
-  if (invalidTrials > 0) blockers.push('execution_witness_missing');
-  if (inconclusiveTrials > 0) blockers.push('environment_failures');
-
-  return {
-    schemaVersion: RESULT_SCHEMA_VERSION,
-    kind: 'relayflows-eval-result',
-    suite: {
-      id: suite.id,
-      path: options.suitePath,
-      sha256: createHash('sha256').update(suiteBytes).digest('hex'),
-    },
-    implementation: options.implementation ?? 'relayflows',
-    startedAt,
-    completedAt: new Date().toISOString(),
-    repetitions,
-    provenance,
-    summary: {
-      passedTrials,
-      failedTrials,
-      invalidTrials,
-      inconclusiveTrials,
-      totalTrials,
-      passRate: passedTrials / totalTrials,
-      publicationStatus: blockers.length === 0 ? 'eligible' : 'ineligible',
-      blockers,
-    },
-    cases,
-  };
-}
-
-export function collectProvenance(rootDir, execute = spawnSync) {
-  const gitCommit = commandResult(execute, 'git', ['rev-parse', 'HEAD'], rootDir);
-  const gitStatus = commandResult(execute, 'git', ['status', '--porcelain=v1'], rootDir);
-  const gitRoot = commandResult(execute, 'git', ['rev-parse', '--show-toplevel'], rootDir);
-  const cpu = cpus();
-  return {
-    source: {
-      commit: gitCommit.ok ? gitCommit.stdout : null,
-      root: gitRoot.ok ? resolve(gitRoot.stdout) : rootDir,
-      dirty: !gitCommit.ok || !gitStatus.ok || !gitRoot.ok || gitStatus.stdout.length > 0,
-      statusAvailable: gitStatus.ok,
-      statusPorcelain: gitStatus.stdout,
-    },
-    runtime: {
-      node: process.version,
-      rustc: commandText(execute, 'rustc', ['--version'], rootDir) || null,
-      cargo: commandText(execute, resolve(rootDir, 'ops/cargo.sh'), ['--version'], rootDir) || null,
-    },
-    host: {
-      platform: platform(),
-      release: release(),
-      arch: process.arch,
-      cpuModel: cpu[0]?.model ?? null,
-      logicalCpus: cpu.length,
-    },
-  };
-}
-
-function classifyTrial({
-  result,
-  stdout,
-  stderr,
-  missingWitnesses,
-  environmentFailurePatterns,
-}) {
-  if (result.status === 0) return missingWitnesses.length === 0 ? 'passed' : 'invalid';
-  const combined = `${stdout}\n${stderr}\n${result.error?.message ?? ''}`;
-  if (
-    result.error ||
-    result.status === 127 ||
-    environmentFailurePatterns.some((pattern) => combined.includes(pattern))
-  ) {
-    return 'inconclusive';
-  }
-  return 'failed';
-}
-
-function commandText(execute, file, args, cwd) {
-  const result = execute(file, args, { cwd, encoding: 'utf8' });
-  if (result.status !== 0) return '';
-  return text(result.stdout).trim();
-}
-
-function commandResult(execute, file, args, cwd) {
-  const result = execute(file, args, { cwd, encoding: 'utf8' });
-  return {
-    ok: result.status === 0,
-    stdout: result.status === 0 ? text(result.stdout).trim() : '',
-  };
-}
-
-function percentile(values, quantile) {
-  const sorted = [...values].sort((left, right) => left - right);
-  const index = Math.max(0, Math.ceil(sorted.length * quantile) - 1);
-  return sorted[index];
-}
-
-function isWithin(parent, child) {
-  const path = relative(parent, child);
-  return path === '' || (!path.startsWith('..') && !path.startsWith('/'));
-}
-
-function requireNonEmptyString(value, field) {
-  if (typeof value !== 'string' || value.length === 0) {
-    throw new Error(`${field} must be a non-empty string`);
-  }
-}
-
-function requirePositiveInteger(value, field) {
-  if (!Number.isInteger(value) || value < 1) {
-    throw new Error(`${field} must be a positive integer`);
-  }
-}
-
-function text(value) {
-  if (value === undefined || value === null) return '';
-  return Buffer.isBuffer(value) ? value.toString('utf8') : String(value);
-}
+import { isWithin, runEvalSuite } from './eval-suite.mjs';
 
 export function parseArgs(argv) {
   const options = { rootDir: process.cwd(), allowDirty: false };
@@ -303,7 +18,7 @@ export function parseArgs(argv) {
       throw new Error(`unknown option: ${arg}`);
     }
     const value = argv[index + 1];
-    if (!value) throw new Error(`missing value for ${arg}`);
+    if (value === undefined) throw new Error(`missing value for ${arg}`);
     index += 1;
     if (arg === '--suite') options.suitePath = value;
     if (arg === '--output') options.output = value;
@@ -312,6 +27,12 @@ export function parseArgs(argv) {
     if (arg === '--root') options.rootDir = value;
   }
   if (!options.suitePath) throw new Error('missing required option --suite');
+  if (
+    options.repetitions !== undefined &&
+    (!Number.isInteger(options.repetitions) || options.repetitions < 1)
+  ) {
+    throw new Error('--repeat must be a positive integer');
+  }
   return options;
 }
 
diff --git a/scripts/run-evals.test.mjs b/scripts/run-evals.test.mjs
index 66b9a5fc7..463062277 100644
--- a/scripts/run-evals.test.mjs
+++ b/scripts/run-evals.test.mjs
@@ -4,14 +4,10 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { test } from 'node:test';
 
-import {
-  collectProvenance,
-  parseArgs,
-  reportExitCode,
-  runEvalSuite,
-  validateEvalSuite,
-  validateOutputPath,
-} from './run-evals.mjs';
+import { parseOutcome } from '../benchmarks/durability/black-box-crash.mjs';
+import { collectProvenance } from './eval-provenance.mjs';
+import { runEvalSuite, trialEnvironment, validateEvalSuite } from './eval-suite.mjs';
+import { parseArgs, reportExitCode, validateOutputPath } from './run-evals.mjs';
 
 function suite(overrides = {}) {
   return {
@@ -170,6 +166,59 @@ test('a recognized toolchain failure is inconclusive rather than a product failu
   });
 });
 
+test('a timed-out command is a product failure blocker, not an environment failure', () => {
+  withSuite(suite({ defaultRepetitions: 1, minimumPublishableRepetitions: 1 }), ({ root, path }) => {
+    const error = Object.assign(new Error('spawnSync fixture ETIMEDOUT'), { code: 'ETIMEDOUT' });
+    const report = runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: provenance(root, false),
+      execute: () => ({ status: null, signal: 'SIGTERM', stdout: '', stderr: '', error }),
+    });
+    assert.equal(report.cases[0].trials[0].outcome, 'timed_out');
+    assert.equal(report.summary.timedOutTrials, 1);
+    assert.deepEqual(report.summary.blockers, ['trial_timeouts']);
+  });
+});
+
+test('trial environments are allowlisted and recorded without ambient secrets', () => {
+  const environment = trialEnvironment(
+    {
+      CARGO_HOME: '/toolchain/cargo',
+      HOME: '/home/eval',
+      PATH: '/bin',
+      RELAY_API_KEY: 'secret',
+      RUSTUP_TOOLCHAIN: 'stable',
+      UNKNOWN: 'not-recorded',
+    },
+    { FIXTURE_MODE: 'deterministic' },
+  );
+  assert.deepEqual(environment, {
+    CARGO_HOME: '/toolchain/cargo',
+    HOME: '/home/eval',
+    PATH: '/bin',
+    RUSTUP_TOOLCHAIN: 'stable',
+    FIXTURE_MODE: 'deterministic',
+  });
+
+  withSuite(suite({ defaultRepetitions: 1, minimumPublishableRepetitions: 1 }), ({ root, path }) => {
+    let observedEnvironment;
+    const report = runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: provenance(root, false),
+      parentEnv: { HOME: '/home/eval', PATH: '/bin', RELAY_API_KEY: 'secret' },
+      execute: (_file, _args, options) => {
+        observedEnvironment = options.env;
+        return { status: 0, signal: null, stdout: 'executed resume\n', stderr: '' };
+      },
+    });
+    assert.deepEqual(observedEnvironment, { HOME: '/home/eval', PATH: '/bin' });
+    assert.deepEqual(report.cases[0].trials[0].command.environment, observedEnvironment);
+    assert.equal('RELAY_API_KEY' in report.cases[0].trials[0].command.environment, false);
+  });
+});
+
 test('case cwd resolves relative to the suite and cannot escape the provenance root', () => {
   withSuite(suite(), ({ root, path }) => {
     const observed = [];
@@ -235,6 +284,7 @@ test('CLI parsing and exit policy fail closed for nonpublishable evidence', () =
     repetitions: 30,
   });
   assert.throws(() => parseArgs(['--unknown']), /unknown option/u);
+  assert.throws(() => parseArgs(['--suite', 'suite.json', '--repeat', '0']), /--repeat must be a positive integer/u);
   assert.equal(reportExitCode({ summary: { publicationStatus: 'eligible' } }), 0);
   assert.equal(reportExitCode({ summary: { publicationStatus: 'ineligible' } }), 1);
   assert.throws(
@@ -243,3 +293,11 @@ test('CLI parsing and exit policy fail closed for nonpublishable evidence', () =
   );
   assert.doesNotThrow(() => validateOutputPath('/work/repo', '/tmp/result.json'));
 });
+
+test('black-box receipt parsing tolerates diagnostics but requires a public run receipt', () => {
+  assert.deepEqual(parseOutcome('diagnostic\n{"run_id":"run-1","status":"completed"}\n'), {
+    run_id: 'run-1',
+    status: 'completed',
+  });
+  assert.throws(() => parseOutcome('diagnostic only\n'), /did not contain a JSON run receipt/u);
+});

From d40219310007a353963b1d763c8890d214ff7a52 Mon Sep 17 00:00:00 2001
From: kjgbot <kjgbot@agentrelay.dev>
Date: Wed, 23 Sep 2026 13:44:31 -0700
Subject: [PATCH 6/8] Close eval harness review gaps

---
 benchmarks/README.md                      |  5 ++--
 benchmarks/durability/black-box-crash.mjs | 34 ++++++++++++++++++++---
 scripts/eval-suite.mjs                    |  4 +--
 scripts/run-evals.test.mjs                | 33 ++++++++++++++++++++--
 4 files changed, 65 insertions(+), 11 deletions(-)

diff --git a/benchmarks/README.md b/benchmarks/README.md
index 25af45681..b581e4e73 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -36,8 +36,9 @@ The runner refuses a dirty tree by default and requires report output to live
 outside the source tree, so one result cannot silently dirty the next run.
 `--allow-dirty` exists for local development, but permanently marks the artifact
 `publicationStatus: ineligible` and exits nonzero. A publishable result also
-requires the suite's minimum repetition count, zero failed trials, and explicit
-output witnesses proving each selected test actually ran. Every trial records
+requires the suite's minimum repetition count, zero failed, timed-out, invalid,
+or inconclusive trials, and explicit output witnesses proving each selected
+test actually ran. Every trial records
 its argv, working directory, duration, outcome, exit code, stdout, and stderr;
 the report records the suite hash, source commit, toolchain, and host class.
 
diff --git a/benchmarks/durability/black-box-crash.mjs b/benchmarks/durability/black-box-crash.mjs
index ed38c66ca..530279ee7 100644
--- a/benchmarks/durability/black-box-crash.mjs
+++ b/benchmarks/durability/black-box-crash.mjs
@@ -26,6 +26,8 @@ export async function runBlackBoxCrash(options = {}) {
   const stepPid = join(fixture, 'step.pid');
   const specPath = join(fixture, 'flow.json');
   const assertions = [];
+  let child = null;
+  let inFlightPid = null;
 
   try {
     writeFileSync(
@@ -56,7 +58,7 @@ export async function runBlackBoxCrash(options = {}) {
     );
     check(assertions, readLines(effects), ['first'], 'only step one completed before resume');
 
-    const child = spawn(
+    child = spawn(
       binary,
       ['--data-dir', dataDir, 'resume', receipt.run_id],
       { detached: true, stdio: ['ignore', 'pipe', 'pipe'] },
@@ -83,14 +85,17 @@ export async function runBlackBoxCrash(options = {}) {
       return readLines(attempts).join(',') === 'attempt' && readPid(stepPid) !== null;
     });
     check(assertions, readLines(attempts), ['attempt'], 'step two was in flight before SIGKILL');
+    inFlightPid = readPid(stepPid);
 
     const exited = once(child, 'exit');
     process.kill(-child.pid, 'SIGKILL');
     const [exitCode, signal] = await exited;
+    child = null;
     check(assertions, exitCode, null, 'killed resume process has no numeric exit code');
     check(assertions, signal, 'SIGKILL', 'resume process observed SIGKILL');
-    await killAndWait(readPid(stepPid));
-    check(assertions, isProcessAlive(readPid(stepPid)), false, 'in-flight step process was killed');
+    await killAndWait(inFlightPid);
+    check(assertions, isProcessAlive(inFlightPid), false, 'in-flight step process was killed');
+    inFlightPid = null;
     writeFileSync(gate, 'open\n');
 
     const resumed = spawnSync(binary, ['--data-dir', dataDir, 'resume', receipt.run_id], {
@@ -132,8 +137,29 @@ export async function runBlackBoxCrash(options = {}) {
       effects: readLines(effects),
     };
   } finally {
-    rmSync(fixture, { recursive: true, force: true });
+    try {
+      await cleanupExecution(child, inFlightPid ?? readPid(stepPid));
+    } finally {
+      rmSync(fixture, { recursive: true, force: true });
+    }
+  }
+}
+
+export async function cleanupExecution(child, stepProcessId) {
+  await terminateProcessTree(child);
+  if (isProcessAlive(stepProcessId)) await killAndWait(stepProcessId);
+}
+
+export async function terminateProcessTree(child) {
+  if (!child?.pid || child.exitCode !== null || child.signalCode !== null) return;
+  const exited = once(child, 'exit');
+  try {
+    process.kill(-child.pid, 'SIGKILL');
+  } catch (error) {
+    if (error.code === 'ESRCH') return;
+    throw error;
   }
+  await exited;
 }
 
 export function parseOutcome(stdout) {
diff --git a/scripts/eval-suite.mjs b/scripts/eval-suite.mjs
index edd606894..af545b21b 100644
--- a/scripts/eval-suite.mjs
+++ b/scripts/eval-suite.mjs
@@ -133,7 +133,7 @@ function runCase({ testCase, suite, suitePath, rootDir, repetitions, execute, en
   if (!isWithin(rootDir, commandCwd)) {
     throw new Error(`case ${testCase.id} command cwd escapes the provenance root`);
   }
-  const commandEnvironment = { ...environment, ...testCase.command.env };
+  const commandEnvironment = trialEnvironment(environment, testCase.command.env);
   const trials = Array.from({ length: repetitions }, (_, index) =>
     runTrial({
       testCase,
@@ -284,7 +284,7 @@ export function trialEnvironment(parentEnv, commandEnv = {}) {
   for (const key of INHERITED_ENV_KEYS) {
     if (parentEnv[key] !== undefined) environment[key] = String(parentEnv[key]);
   }
-  return { ...environment, ...commandEnv };
+  return { ...environment, ...commandEnv, CARGO_TERM_COLOR: 'never' };
 }
 
 function percentile(values, quantile) {
diff --git a/scripts/run-evals.test.mjs b/scripts/run-evals.test.mjs
index 463062277..0d08befa8 100644
--- a/scripts/run-evals.test.mjs
+++ b/scripts/run-evals.test.mjs
@@ -1,10 +1,14 @@
 import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
 import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { test } from 'node:test';
 
-import { parseOutcome } from '../benchmarks/durability/black-box-crash.mjs';
+import {
+  parseOutcome,
+  terminateProcessTree,
+} from '../benchmarks/durability/black-box-crash.mjs';
 import { collectProvenance } from './eval-provenance.mjs';
 import { runEvalSuite, trialEnvironment, validateEvalSuite } from './eval-suite.mjs';
 import { parseArgs, reportExitCode, validateOutputPath } from './run-evals.mjs';
@@ -191,10 +195,11 @@ test('trial environments are allowlisted and recorded without ambient secrets',
       RUSTUP_TOOLCHAIN: 'stable',
       UNKNOWN: 'not-recorded',
     },
-    { FIXTURE_MODE: 'deterministic' },
+    { CARGO_TERM_COLOR: 'always', FIXTURE_MODE: 'deterministic' },
   );
   assert.deepEqual(environment, {
     CARGO_HOME: '/toolchain/cargo',
+    CARGO_TERM_COLOR: 'never',
     HOME: '/home/eval',
     PATH: '/bin',
     RUSTUP_TOOLCHAIN: 'stable',
@@ -213,7 +218,11 @@ test('trial environments are allowlisted and recorded without ambient secrets',
         return { status: 0, signal: null, stdout: 'executed resume\n', stderr: '' };
       },
     });
-    assert.deepEqual(observedEnvironment, { HOME: '/home/eval', PATH: '/bin' });
+    assert.deepEqual(observedEnvironment, {
+      CARGO_TERM_COLOR: 'never',
+      HOME: '/home/eval',
+      PATH: '/bin',
+    });
     assert.deepEqual(report.cases[0].trials[0].command.environment, observedEnvironment);
     assert.equal('RELAY_API_KEY' in report.cases[0].trials[0].command.environment, false);
   });
@@ -301,3 +310,21 @@ test('black-box receipt parsing tolerates diagnostics but requires a public run
   });
   assert.throws(() => parseOutcome('diagnostic only\n'), /did not contain a JSON run receipt/u);
 });
+
+test(
+  'black-box cleanup terminates a detached workflow process',
+  { skip: process.platform === 'win32' },
+  async () => {
+    const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], {
+      detached: true,
+      stdio: 'ignore',
+    });
+    await new Promise((resolvePromise, reject) => {
+      child.once('spawn', resolvePromise);
+      child.once('error', reject);
+    });
+    await terminateProcessTree(child);
+    assert.equal(child.exitCode, null);
+    assert.equal(child.signalCode, 'SIGKILL');
+  },
+);

From 6c09afae52b8700af3c4308d886a346a8dc99fd3 Mon Sep 17 00:00:00 2001
From: kjgbot <kjgbot@agentrelay.dev>
Date: Wed, 23 Sep 2026 13:52:55 -0700
Subject: [PATCH 7/8] Fail closed on eval environment ambiguity

---
 benchmarks/README.md       |  8 +++--
 scripts/eval-suite.mjs     | 22 ++++++++----
 scripts/run-evals.mjs      |  2 +-
 scripts/run-evals.test.mjs | 69 ++++++++++++++++++++++++++++++++++++--
 4 files changed, 88 insertions(+), 13 deletions(-)

diff --git a/benchmarks/README.md b/benchmarks/README.md
index b581e4e73..5ae555ec4 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -41,13 +41,15 @@ or inconclusive trials, and explicit output witnesses proving each selected
 test actually ran. Every trial records
 its argv, working directory, duration, outcome, exit code, stdout, and stderr;
 the report records the suite hash, source commit, toolchain, and host class.
+Missing Rust or Cargo version provenance blocks publication.
 
 Trial processes receive and record a small allowlist of ambient variables
 (`HOME`, locale, `PATH`, temporary-directory variables, shell, and Relayflows
 toolchain controls), plus any explicit per-case environment. Credentials and
-unrelated ambient state are neither inherited nor written to the report. A
-command timeout is a product failure blocker (`trial_timeouts`), not an
-environmental excuse.
+unrelated ambient state are not inherited from the runner. Explicit per-case
+environment values are recorded verbatim for reproducibility and therefore
+must never contain credentials or other secrets. A command timeout is a product
+failure blocker (`trial_timeouts`), not an environmental excuse.
 
 The black-box crash case stays on the supported CLI boundary. It obtains an
 opaque run receipt from `run --stop-after 1`, resumes that run, kills the
diff --git a/scripts/eval-suite.mjs b/scripts/eval-suite.mjs
index af545b21b..5a946cad8 100644
--- a/scripts/eval-suite.mjs
+++ b/scripts/eval-suite.mjs
@@ -1,7 +1,7 @@
 import { spawnSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { readFileSync } from 'node:fs';
-import { dirname, relative, resolve } from 'node:path';
+import { readFileSync, realpathSync } from 'node:fs';
+import { dirname, isAbsolute, relative, resolve } from 'node:path';
 import { performance } from 'node:perf_hooks';
 
 import { collectProvenance } from './eval-provenance.mjs';
@@ -95,8 +95,11 @@ function validateCommandEnv(env, caseIndex) {
 }
 
 export function runEvalSuite(options) {
-  const rootDir = resolve(options.rootDir ?? process.cwd());
-  const suitePath = resolve(rootDir, options.suitePath);
+  const rootDir = realpathSync(resolve(options.rootDir ?? process.cwd()));
+  const suitePath = realpathSync(resolve(rootDir, options.suitePath));
+  if (!isWithin(rootDir, suitePath)) {
+    throw new Error('eval suite path escapes the provenance root');
+  }
   const suiteBytes = readFileSync(suitePath);
   const suite = validateEvalSuite(JSON.parse(suiteBytes.toString('utf8')));
   const repetitions = options.repetitions ?? suite.defaultRepetitions;
@@ -116,7 +119,7 @@ export function runEvalSuite(options) {
 }
 
 function validateProvenance(provenance, rootDir, allowDirty) {
-  if (resolve(provenance.source.root) !== rootDir) {
+  if (realpathSync(resolve(provenance.source.root)) !== rootDir) {
     throw new Error(
       `provenance root ${provenance.source.root} does not match requested root ${rootDir}`,
     );
@@ -129,7 +132,7 @@ function validateProvenance(provenance, rootDir, allowDirty) {
 }
 
 function runCase({ testCase, suite, suitePath, rootDir, repetitions, execute, environment }) {
-  const commandCwd = resolve(dirname(suitePath), testCase.command.cwd);
+  const commandCwd = realpathSync(resolve(dirname(suitePath), testCase.command.cwd));
   if (!isWithin(rootDir, commandCwd)) {
     throw new Error(`case ${testCase.id} command cwd escapes the provenance root`);
   }
@@ -187,6 +190,7 @@ function runTrial({
       file: testCase.command.file,
       args: testCase.command.args,
       cwd: testCase.command.cwd,
+      resolvedCwd: commandCwd,
       environment,
     },
     stdout,
@@ -199,6 +203,7 @@ function runTrial({
 function classifyTrial({ result, stdout, stderr, missingWitnesses, environmentFailurePatterns }) {
   if (result.status === 0) return missingWitnesses.length === 0 ? 'passed' : 'invalid';
   if (result.error?.code === 'ETIMEDOUT') return 'timed_out';
+  if (result.error) return 'inconclusive';
   const combined = `${stdout}\n${stderr}\n${result.error?.message ?? ''}`;
   if (
     result.status === 127 ||
@@ -272,6 +277,9 @@ function publicationBlockers(summary) {
   if (summary.repetitions < summary.suite.minimumPublishableRepetitions) {
     blockers.push('insufficient_repetitions');
   }
+  if (!summary.provenance.runtime.rustc || !summary.provenance.runtime.cargo) {
+    blockers.push('toolchain_provenance_unavailable');
+  }
   if (summary.failedTrials > 0) blockers.push('trial_failures');
   if (summary.timedOutTrials > 0) blockers.push('trial_timeouts');
   if (summary.invalidTrials > 0) blockers.push('execution_witness_missing');
@@ -295,7 +303,7 @@ function percentile(values, quantile) {
 
 export function isWithin(parent, child) {
   const path = relative(parent, child);
-  return path === '' || (!path.startsWith('..') && !path.startsWith('/'));
+  return path === '' || (!path.startsWith('..') && !isAbsolute(path));
 }
 
 function requireNonEmptyString(value, field) {
diff --git a/scripts/run-evals.mjs b/scripts/run-evals.mjs
index 102eafd10..c29977ff3 100644
--- a/scripts/run-evals.mjs
+++ b/scripts/run-evals.mjs
@@ -18,7 +18,7 @@ export function parseArgs(argv) {
       throw new Error(`unknown option: ${arg}`);
     }
     const value = argv[index + 1];
-    if (value === undefined) throw new Error(`missing value for ${arg}`);
+    if (value === undefined || value.length === 0) throw new Error(`missing value for ${arg}`);
     index += 1;
     if (arg === '--suite') options.suitePath = value;
     if (arg === '--output') options.output = value;
diff --git a/scripts/run-evals.test.mjs b/scripts/run-evals.test.mjs
index 0d08befa8..b1233e896 100644
--- a/scripts/run-evals.test.mjs
+++ b/scripts/run-evals.test.mjs
@@ -1,6 +1,13 @@
 import assert from 'node:assert/strict';
 import { spawn } from 'node:child_process';
-import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import {
+  mkdtempSync,
+  mkdirSync,
+  realpathSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { test } from 'node:test';
@@ -104,6 +111,7 @@ test('captures literal command output and marks an allowed dirty run ineligible'
     assert.equal(report.cases[0].trials[0].stdout, 'executed resume\nliteral stdout\n');
     assert.equal(report.cases[0].trials[0].stderr, 'literal stderr\n');
     assert.equal(report.cases[0].trials[0].command.cwd, '../..');
+    assert.equal(report.cases[0].trials[0].command.resolvedCwd, realpathSync(root));
   });
 });
 
@@ -185,6 +193,36 @@ test('a timed-out command is a product failure blocker, not an environment failu
   });
 });
 
+test('a command launch error is inconclusive, not a product failure', () => {
+  withSuite(suite({ defaultRepetitions: 1, minimumPublishableRepetitions: 1 }), ({ root, path }) => {
+    const error = Object.assign(new Error('spawnSync missing ENOENT'), { code: 'ENOENT' });
+    const report = runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: provenance(root, false),
+      execute: () => ({ status: null, signal: null, stdout: '', stderr: '', error }),
+    });
+    assert.equal(report.cases[0].trials[0].outcome, 'inconclusive');
+    assert.equal(report.summary.failedTrials, 0);
+    assert.deepEqual(report.summary.blockers, ['environment_failures']);
+  });
+});
+
+test('missing toolchain provenance blocks publication', () => {
+  withSuite(suite({ defaultRepetitions: 1, minimumPublishableRepetitions: 1 }), ({ root, path }) => {
+    const incomplete = provenance(root, false);
+    incomplete.runtime.rustc = null;
+    const report = runEvalSuite({
+      rootDir: root,
+      suitePath: path,
+      provenance: incomplete,
+      execute: () => ({ status: 0, signal: null, stdout: 'executed resume\n', stderr: '' }),
+    });
+    assert.equal(report.summary.publicationStatus, 'ineligible');
+    assert.deepEqual(report.summary.blockers, ['toolchain_provenance_unavailable']);
+  });
+});
+
 test('trial environments are allowlisted and recorded without ambient secrets', () => {
   const environment = trialEnvironment(
     {
@@ -240,7 +278,7 @@ test('case cwd resolves relative to the suite and cannot escape the provenance r
         return { status: 0, signal: null, stdout: 'executed resume\n', stderr: '' };
       },
     });
-    assert.deepEqual(observed, [root, root]);
+    assert.deepEqual(observed, [realpathSync(root), realpathSync(root)]);
 
     const escaping = suite();
     escaping.cases[0].command.cwd = '../../..';
@@ -258,6 +296,29 @@ test('case cwd resolves relative to the suite and cannot escape the provenance r
   });
 });
 
+test('case cwd cannot escape the provenance root through a symlink', () => {
+  const external = mkdtempSync(join(tmpdir(), 'relayflows-eval-external-'));
+  try {
+    const value = suite({ defaultRepetitions: 1, minimumPublishableRepetitions: 1 });
+    value.cases[0].command.cwd = 'escape';
+    withSuite(value, ({ root, path }) => {
+      symlinkSync(external, join(root, 'benchmarks', 'fixture', 'escape'), 'dir');
+      assert.throws(
+        () =>
+          runEvalSuite({
+            rootDir: root,
+            suitePath: path,
+            provenance: provenance(root, false),
+            execute: () => ({ status: 0, stdout: 'executed resume\n', stderr: '' }),
+          }),
+        /command cwd escapes the provenance root/u,
+      );
+    });
+  } finally {
+    rmSync(external, { recursive: true, force: true });
+  }
+});
+
 test('collectProvenance exercises clean, dirty, and unavailable git state', () => {
   const root = '/work/repo';
   const execute = (_file, args) => {
@@ -293,6 +354,10 @@ test('CLI parsing and exit policy fail closed for nonpublishable evidence', () =
     repetitions: 30,
   });
   assert.throws(() => parseArgs(['--unknown']), /unknown option/u);
+  assert.throws(
+    () => parseArgs(['--suite', 'suite.json', '--implementation', '']),
+    /missing value for --implementation/u,
+  );
   assert.throws(() => parseArgs(['--suite', 'suite.json', '--repeat', '0']), /--repeat must be a positive integer/u);
   assert.equal(reportExitCode({ summary: { publicationStatus: 'eligible' } }), 0);
   assert.equal(reportExitCode({ summary: { publicationStatus: 'ineligible' } }), 1);

From 3f50d89c39dfe67c196138feb401987d0e79cdcd Mon Sep 17 00:00:00 2001
From: kjgbot <kjgbot@agentrelay.dev>
Date: Wed, 23 Sep 2026 14:05:31 -0700
Subject: [PATCH 8/8] Make symlink confinement test portable

---
 scripts/run-evals.test.mjs | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scripts/run-evals.test.mjs b/scripts/run-evals.test.mjs
index b1233e896..2dffcc6d9 100644
--- a/scripts/run-evals.test.mjs
+++ b/scripts/run-evals.test.mjs
@@ -296,13 +296,21 @@ test('case cwd resolves relative to the suite and cannot escape the provenance r
   });
 });
 
-test('case cwd cannot escape the provenance root through a symlink', () => {
+test('case cwd cannot escape the provenance root through a symlink', (testContext) => {
   const external = mkdtempSync(join(tmpdir(), 'relayflows-eval-external-'));
   try {
     const value = suite({ defaultRepetitions: 1, minimumPublishableRepetitions: 1 });
     value.cases[0].command.cwd = 'escape';
     withSuite(value, ({ root, path }) => {
-      symlinkSync(external, join(root, 'benchmarks', 'fixture', 'escape'), 'dir');
+      try {
+        symlinkSync(external, join(root, 'benchmarks', 'fixture', 'escape'), 'dir');
+      } catch (error) {
+        if (['EACCES', 'EPERM'].includes(error.code)) {
+          testContext.skip(`host denied directory symlink creation: ${error.code}`);
+          return;
+        }
+        throw error;
+      }
       assert.throws(
         () =>
           runEvalSuite({