diff --git a/mcp/src/lab/gaia/service.ts b/mcp/src/lab/gaia/service.ts index 4d6258980..d84b96aea 100644 --- a/mcp/src/lab/gaia/service.ts +++ b/mcp/src/lab/gaia/service.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, stat, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { ensureGaiaDataset, ensureGaiaPython, SCORER_SHA256 } from "./bootstrap.js"; @@ -238,6 +238,13 @@ export function buildGaiaServices(opts: BuildGaiaOptions = {}): GaiaServices { } const rev = head.stdout.trim(); + // Importing scorer.py compiles bytecode into __pycache__/ — derived + // state, but also a shadowing vector: python prefers a matching .pyc over + // the (sha-pinned) source, so a doctored cache could bypass the pin. + // Delete it rather than tolerate it. score() runs python -B so it + // normally never appears; this also heals checkouts dirtied before -B. + await rm(join(abs, "__pycache__"), { recursive: true, force: true }); + // The dataset must be unmodified (a doctored metadata.jsonl = doctored // gold). scorer.py is expected untracked — it comes from the leaderboard // Space, not this repo. @@ -346,7 +353,9 @@ export function buildGaiaServices(opts: BuildGaiaOptions = {}): GaiaServices { const python = autoSetup ? await ensureGaiaPython(opts.python ? { python: opts.python } : {}) : (opts.python ?? process.env.GAIA_PYTHON ?? "python3"); - const res = await exec(python, ["-c", DRIVER, root, pairsPath], { + // -B: never write __pycache__/ into the checkout (verifyDataset treats a + // dirty tree as doctored gold and refuses to grade). + const res = await exec(python, ["-B", "-c", DRIVER, root, pairsPath], { cwd: root, timeoutMs: o.timeoutMs ?? DEFAULT_SCORE_TIMEOUT_MS, }); diff --git a/mcp/src/lab/gaia/smoke.ts b/mcp/src/lab/gaia/smoke.ts index 9cdacd8ef..5257e9fe3 100644 --- a/mcp/src/lab/gaia/smoke.ts +++ b/mcp/src/lab/gaia/smoke.ts @@ -130,6 +130,14 @@ async function main() { // ── empty pairs refuses ── await assert.rejects(() => gaia.score([]), /no pairs/); + // ── untracked __pycache__/ is healed, not refused (and the driver runs + // with -B, so score() itself must not have created one) ── + assert.ok(!existsSync(join(dir, "__pycache__")), "score() wrote __pycache__ despite -B"); + mkdirSync(join(dir, "__pycache__")); + writeFileSync(join(dir, "__pycache__", "scorer.cpython-311.pyc"), "doctored"); + await gaia.verifyDataset(); + assert.ok(!existsSync(join(dir, "__pycache__")), "verifyDataset left __pycache__ behind"); + // ── dirty tree refuses (tracked file modified) ── writeFileSync(join(dir, "2023", "validation", "metadata.jsonl"), "{}\n"); await assert.rejects(() => gaia.score([{ taskId: T1, answer: "4" }]), /local modifications/);