From 0cbcdb3bd5a7bed9cab263c176aa7b8cc1918f91 Mon Sep 17 00:00:00 2001 From: Evanfeenstra Date: Fri, 28 Aug 2026 21:43:15 -0700 Subject: [PATCH] gaia: stop __pycache__ from dirtying the dataset checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Importing scorer.py compiled bytecode into /__pycache__/, so the first score() dirtied the tree and every later grade refused with 'local modifications'. Run the driver with python -B so bytecode is never written, and have verifyDataset delete any existing __pycache__/ — it's derived state, and a doctored .pyc there could shadow the sha-pinned scorer.py source, so removal (not tolerance) is the right call. Heals already-dirty prod checkouts on the next grade. Co-Authored-By: Claude Fable 5 --- mcp/src/lab/gaia/service.ts | 13 +++++++++++-- mcp/src/lab/gaia/smoke.ts | 8 ++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mcp/src/lab/gaia/service.ts b/mcp/src/lab/gaia/service.ts index 4d6258980..d84b96aea 100644 --- a/mcp/src/lab/gaia/service.ts +++ b/mcp/src/lab/gaia/service.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, stat, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { ensureGaiaDataset, ensureGaiaPython, SCORER_SHA256 } from "./bootstrap.js"; @@ -238,6 +238,13 @@ export function buildGaiaServices(opts: BuildGaiaOptions = {}): GaiaServices { } const rev = head.stdout.trim(); + // Importing scorer.py compiles bytecode into __pycache__/ — derived + // state, but also a shadowing vector: python prefers a matching .pyc over + // the (sha-pinned) source, so a doctored cache could bypass the pin. + // Delete it rather than tolerate it. score() runs python -B so it + // normally never appears; this also heals checkouts dirtied before -B. + await rm(join(abs, "__pycache__"), { recursive: true, force: true }); + // The dataset must be unmodified (a doctored metadata.jsonl = doctored // gold). scorer.py is expected untracked — it comes from the leaderboard // Space, not this repo. @@ -346,7 +353,9 @@ export function buildGaiaServices(opts: BuildGaiaOptions = {}): GaiaServices { const python = autoSetup ? await ensureGaiaPython(opts.python ? { python: opts.python } : {}) : (opts.python ?? process.env.GAIA_PYTHON ?? "python3"); - const res = await exec(python, ["-c", DRIVER, root, pairsPath], { + // -B: never write __pycache__/ into the checkout (verifyDataset treats a + // dirty tree as doctored gold and refuses to grade). + const res = await exec(python, ["-B", "-c", DRIVER, root, pairsPath], { cwd: root, timeoutMs: o.timeoutMs ?? DEFAULT_SCORE_TIMEOUT_MS, }); diff --git a/mcp/src/lab/gaia/smoke.ts b/mcp/src/lab/gaia/smoke.ts index 9cdacd8ef..5257e9fe3 100644 --- a/mcp/src/lab/gaia/smoke.ts +++ b/mcp/src/lab/gaia/smoke.ts @@ -130,6 +130,14 @@ async function main() { // ── empty pairs refuses ── await assert.rejects(() => gaia.score([]), /no pairs/); + // ── untracked __pycache__/ is healed, not refused (and the driver runs + // with -B, so score() itself must not have created one) ── + assert.ok(!existsSync(join(dir, "__pycache__")), "score() wrote __pycache__ despite -B"); + mkdirSync(join(dir, "__pycache__")); + writeFileSync(join(dir, "__pycache__", "scorer.cpython-311.pyc"), "doctored"); + await gaia.verifyDataset(); + assert.ok(!existsSync(join(dir, "__pycache__")), "verifyDataset left __pycache__ behind"); + // ── dirty tree refuses (tracked file modified) ── writeFileSync(join(dir, "2023", "validation", "metadata.jsonl"), "{}\n"); await assert.rejects(() => gaia.score([{ taskId: T1, answer: "4" }]), /local modifications/);