ci(cli): validate installed Eval frameworks (#3188)

* ci(cli): validate installed Eval frameworks

Run Harbor and Pier against the same immutable CLI tarball after cross-platform spec and prerequisite checks. Use deterministic local tasks so the release gate exercises the installed relay, Docker environment, verifier result, artifacts, and cleanup without provider credentials.

Preserve Pier's framework-owned log mounts when adding user mounts; otherwise an explicit empty mount list replaces the paths required for subject scope, rewards, and collected artifacts.

Generated-by: OpenAI Codex

* fix(eval): harden release validation failures

Reserve Pier's framework-owned log subtrees before Docker composition so configured mounts cannot shadow verifier rewards or collected artifacts. Keep framework failures authoritative when summary or diagnostic evidence is malformed or unreadable, and create the deterministic Git fixture under the same isolated environment as the installed candidate.

Generated-by: OpenAI Codex

* test(eval): assert reserved mount errors exactly

Use an exact predicate for the observable Pier mount rejection so path punctuation cannot weaken the regression check.

Author: M4n5ter

.github/workflows/cli-package-validation.yml

@@ -144,3 +144,36 @@ jobs:
           path: packages/cli/release
       - name: Validate the installed tarball
         run: node scripts/smoke-release-cli-package.mjs
+
+  eval:
+    name: Validate installed CLI Eval
+    needs: build
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '24'
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.12'
+      - name: Select the release npm toolchain
+        run: npm install --global --no-audit --no-fund npm@11.12.1
+      - name: Install pinned Eval frameworks
+        run: |
+          python -m venv "$RUNNER_TEMP/maka-harbor"
+          "$RUNNER_TEMP/maka-harbor/bin/python" -m pip install --disable-pip-version-check 'harbor==0.20.0'
+          python -m venv "$RUNNER_TEMP/maka-pier"
+          "$RUNNER_TEMP/maka-pier/bin/python" -m pip install --disable-pip-version-check 'datacurve-pier==0.3.0'
+          echo "MAKA_RELEASE_HARBOR_PYTHON=$RUNNER_TEMP/maka-harbor/bin/python" >> "$GITHUB_ENV"
+          echo "MAKA_RELEASE_PIER_PYTHON=$RUNNER_TEMP/maka-pier/bin/python" >> "$GITHUB_ENV"
+      - name: Download the release candidate
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: cli-release-candidate
+          path: packages/cli/release
+      - name: Validate real Harbor and Pier cells
+        run: npm run release:cli:eval

package.json

@@ -44,9 +44,10 @@
     "check:cli-third-party-notices": "node scripts/generate-third-party-notices.mjs --target cli --check",
     "release:cli:pack": "node scripts/release-cli-package.mjs",
     "release:cli:smoke": "node scripts/smoke-release-cli-package.mjs",
+    "release:cli:eval": "node scripts/release-cli-eval-package.mjs",
     "generate:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs",
     "check:windows-cargo-notices": "node scripts/generate-windows-cargo-notices.mjs --check",
-    "check:release": "npm run check:stale && npm run check:third-party-notices && npm run check:cli-third-party-notices && node --test scripts/release-cli-file-policy.test.mjs scripts/release-cli-artifact-policy.test.mjs",
+    "check:release": "npm run check:stale && npm run check:third-party-notices && npm run check:cli-third-party-notices && node --test scripts/release-cli-file-policy.test.mjs scripts/release-cli-artifact-policy.test.mjs scripts/release-cli-eval-support.test.mjs",
     "package:macos-arm64": "node scripts/package-macos-arm64.mjs",
     "verify:macos-arm64": "node scripts/verify-macos-arm64-dmg.mjs",
     "package:windows-x64": "node scripts/package-windows-x64.mjs",

packages/eval/src/__tests__/lifecycle-boundaries.test.ts

@@ -1295,7 +1295,37 @@ test('pier cannot declare an egress proxy it never enforces', () => {
   );
 });
 
-test('launched trial environment does not inherit MAKA_EVAL_FRAMEWORK', {
+test('Pier rejects configured mounts that collide with framework log ownership', () => {
+  const root = join(tmpdir(), 'maka-test-pier-reserved-mount');
+  const restoreEnvironment = setEnvironment({
+    MAKA_TEST_MOUNT: join(root, 'mount'),
+    MAKA_TEST_PYTHON: join(root, 'python'),
+    MAKA_TEST_TASKS: join(root, 'tasks'),
+    MAKA_TEST_TRIALS: join(root, 'trials'),
+  });
+  try {
+    for (const target of ['/logs/agent/../agent', '/logs/verifier/reward.txt']) {
+      assert.throws(
+        () =>
+          createPierExecutor(
+            {
+              ...executorConfig(),
+              tasksRootEnv: 'MAKA_TEST_TASKS',
+              mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target, readOnly: true }],
+            },
+            'experiment.json',
+          ),
+        (error) =>
+          error instanceof Error &&
+          error.message === `Pier mount target ${target} is reserved for framework logs`,
+      );
+    }
+  } finally {
+    restoreEnvironment();
+  }
+});
+
+test('Pier preserves its log mounts without inheriting MAKA_EVAL_FRAMEWORK', {
   timeout: 10_000,
 }, async () => {
   const root = await mkdtemp(join(tmpdir(), 'maka-eval-framework-env-'));
@@ -1309,6 +1339,8 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises';
 const config = JSON.parse(await readFile(process.argv.at(-1), 'utf8'));
 await writeFile(process.env.MAKA_TEST_ENV, JSON.stringify({
   framework: process.env.MAKA_EVAL_FRAMEWORK ?? null,
+  mounts: config.environment.mounts,
+  trialName: config.trial_name,
 }));
 const socket = connect(config.agent.kwargs.relay_port, config.agent.kwargs.relay_host);
 socket.setEncoding('utf8');
@@ -1344,14 +1376,33 @@ socket.end();
     MAKA_TEST_PYTHON: executable,
     MAKA_TEST_TRIALS: root,
     MAKA_TEST_ENV: envDump,
+    MAKA_TEST_MOUNT: root,
+    MAKA_TEST_TASKS: root,
     MAKA_EVAL_FRAMEWORK: 'pier',
   });
   try {
+    const spec: ExperimentSpec = {
+      ...experiment(),
+      executor: {
+        kind: 'pier',
+        config: {
+          ...executorConfig(),
+          tasksRootEnv: 'MAKA_TEST_TASKS',
+          mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target: '/input', readOnly: true }],
+        },
+      },
+      tasks: [{ id: 'task', input: 'solve', config: { pier: { path: 'task' } } }],
+    };
     const results = await runExperiment({
-      spec: experiment(),
+      spec,
       store: new FileAttemptStore(join(root, 'attempts')),
-      executor: createHarborExecutor(
-        { ...executorConfig(), preparationEnvironment: ['MAKA_TEST_ENV'] },
+      executor: createPierExecutor(
+        {
+          ...executorConfig(),
+          tasksRootEnv: 'MAKA_TEST_TASKS',
+          preparationEnvironment: ['MAKA_TEST_ENV'],
+          mounts: [{ sourceEnv: 'MAKA_TEST_MOUNT', target: '/input', readOnly: true }],
+        },
         join(root, 'experiment.json'),
       ),
       subjects: [
@@ -1372,7 +1423,26 @@ socket.end();
       ],
     });
     assert.equal(results.get('task::1::external')?.result.status, 'completed');
-    assert.deepEqual(JSON.parse(await readFile(envDump, 'utf8')), { framework: null });
+    const launched = JSON.parse(await readFile(envDump, 'utf8')) as {
+      framework: string | null;
+      mounts: Array<{ source: string; target: string }>;
+      trialName: string;
+    };
+    assert.equal(launched.framework, null);
+    assert.deepEqual(launched.mounts, [
+      { type: 'bind', source: root, target: '/input', read_only: true },
+      { type: 'bind', source: join(root, launched.trialName, 'agent'), target: '/logs/agent' },
+      {
+        type: 'bind',
+        source: join(root, launched.trialName, 'verifier'),
+        target: '/logs/verifier',
+      },
+      {
+        type: 'bind',
+        source: join(root, launched.trialName, 'artifacts'),
+        target: '/logs/artifacts',
+      },
+    ]);
   } finally {
     restoreEnvironment();
     await rm(root, { recursive: true, force: true });

packages/eval/src/harness-executor.ts

@@ -4,7 +4,7 @@ import { once } from 'node:events';
 import { createReadStream } from 'node:fs';
 import { chmod, lstat, mkdir, readFile, readdir, unlink, writeFile } from 'node:fs/promises';
 import { createServer, type Server, type Socket } from 'node:net';
-import { basename, dirname, join, relative, resolve, sep } from 'node:path';
+import { basename, dirname, join, posix, relative, resolve, sep } from 'node:path';
 import { createInterface } from 'node:readline';
 import { decodeJsonObject, type ExperimentCell, type JsonObject } from './experiment.js';
 import {
@@ -33,6 +33,12 @@ import type { EvalResult } from './result.js';
 export type HarnessFramework = 'harbor' | 'pier';
 type RelayTransportStage = 'ready' | 'execute' | 'receive' | 'decision';
 
+const PIER_FRAMEWORK_LOG_MOUNTS = Object.freeze([
+  { directory: 'agent', target: '/logs/agent' },
+  { directory: 'verifier', target: '/logs/verifier' },
+  { directory: 'artifacts', target: '/logs/artifacts' },
+]);
+
 interface RelayTransportFailure {
   readonly stage: RelayTransportStage;
   readonly category:
@@ -195,7 +201,12 @@ async function runHarnessAttempt(
           : await waitForTrial(state.child, { phase: 'completion' });
         finalizationEvidence = completed;
         if (!finalizationConfirmed(completed)) throw new Error('Trial did not finalize cleanly');
-        const verification = await readVerification(state, cell, Boolean(options.egressProxy));
+        const verification = await readVerification(
+          state,
+          cell,
+          framework,
+          Boolean(options.egressProxy),
+        );
         verificationConfirmedBeforeCancellation = !hostCancellationObserved;
         return verification;
       },
@@ -381,7 +392,7 @@ async function startTrial(
   const task = decodeTask(framework, options, cell);
   const timeoutMultiplier = positive(cell.budget.timeoutMultiplier, 'budget.timeoutMultiplier');
   const egressPaths = await resolveEgressPaths(options);
-  const environmentConfig = resolveEnvironmentConfig(options, egressPaths);
+  const environmentConfig = resolveEnvironmentConfig(options, egressPaths, framework, trialPath);
   const networkPolicyPath = egressPaths?.networkPolicyPath;
   const executionEnvironment = {
     ...UNATTENDED_EXECUTION_ENVIRONMENT,
@@ -751,6 +762,7 @@ function inspectEgressAudit(audit: Buffer): {
 async function readVerification(
   state: RelayState,
   cell: ExperimentCell,
+  framework: HarnessFramework,
   expectEgressAudit: boolean,
 ): Promise<ExecutorVerification> {
   const result = JSON.parse(await readFile(join(state.trialPath, 'result.json'), 'utf8')) as {
@@ -777,7 +789,7 @@ async function readVerification(
         failureReason: `failed to read egress audit log ${egressAuditPath}${code ? ` (${code})` : ''}`,
         artifacts: [
           { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName },
-          ...(await collectedArtifactInventory(state.trialPath)),
+          ...(await collectedArtifactInventory(state.trialPath, framework)),
           { kind: 'egress-audit-unreadable', path: EGRESS_AUDIT_ARTIFACT_PATH },
         ],
       };
@@ -796,14 +808,20 @@ async function readVerification(
     failureReason: audit.failureReason ?? (score === null ? 'verifier produced no reward' : null),
     artifacts: [
       { kind: 'trial', framework: cell.executor.kind, trialName: state.trialName },
-      ...(await collectedArtifactInventory(state.trialPath)),
+      ...(await collectedArtifactInventory(state.trialPath, framework)),
       ...audit.artifacts,
     ],
   };
 }
 
-async function collectedArtifactInventory(trialPath: string): Promise<JsonObject[]> {
-  const root = join(trialPath, 'artifacts', 'logs', 'artifacts');
+async function collectedArtifactInventory(
+  trialPath: string,
+  framework: HarnessFramework,
+): Promise<JsonObject[]> {
+  const root =
+    framework === 'pier'
+      ? join(trialPath, 'artifacts')
+      : join(trialPath, 'artifacts', 'logs', 'artifacts');
   const files: JsonObject[] = [];
   const targets = [
     join(root, basename(MAKA_RUNTIME_ARTIFACT_PATH)),
@@ -910,6 +928,18 @@ function decodeHarnessOptions(value: JsonObject, framework: HarnessFramework): H
       ? { tasksRootEnv: machinePathEnv(options.tasksRootEnv, 'tasksRootEnv') }
       : {}),
   };
+  if (framework === 'pier') {
+    const reservedTargets = PIER_FRAMEWORK_LOG_MOUNTS.map((mount) => mount.target);
+    const collision = decoded.mounts.find((mount) => {
+      const target = posix.normalize(mount.target);
+      return reservedTargets.some(
+        (reserved) => target === reserved || target.startsWith(`${reserved}/`),
+      );
+    });
+    if (collision) {
+      throw new Error(`Pier mount target ${collision.target} is reserved for framework logs`);
+    }
+  }
   for (const name of [
     decoded.pythonPathEnv,
     decoded.trialsRootEnv,
@@ -980,8 +1010,22 @@ interface ResolvedEgressPaths {
 function resolveEnvironmentConfig(
   options: HarnessOptions,
   egressPaths: ResolvedEgressPaths | undefined,
+  framework: HarnessFramework,
+  trialPath: string,
 ): JsonObject {
-  const base = { ...options.environment, mounts: resolveMounts(options.mounts) };
+  const configuredMounts = resolveMounts(options.mounts);
+  const mounts =
+    framework === 'pier'
+      ? [
+          ...configuredMounts,
+          ...PIER_FRAMEWORK_LOG_MOUNTS.map(({ directory, target }) => ({
+            type: 'bind',
+            source: join(trialPath, directory),
+            target,
+          })),
+        ]
+      : configuredMounts;
+  const base = { ...options.environment, mounts };
   if (!options.egressProxy) return base;
   if (!egressPaths) throw new Error('egress proxy paths are unavailable');
   return { ...base, extra_docker_compose: [egressPaths.composePath] };