From 77eac1a7b11dd7d8ec8c9053f52b64cd4264d357 Mon Sep 17 00:00:00 2001 From: Evanfeenstra Date: Sat, 29 Aug 2026 15:54:52 -0700 Subject: [PATCH] evolve: guard the author summary channel against filler echoes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observed live (gaia, prod): a schema-mode author ended on a bare text turn and echoed summary '' / 'placeholder' into two generations. The version fallback kept grading honest, but the junk summaries poisoned the briefing channel — the EXPLORE directive's 'pick an approach that is none of the above' is only checkable against real summaries — and left the human report unreadable. - eval/evolve-loop: usableSummary() rejects empty/short/filler echoes (placeholder, todo, n/a, …) and substitutes an honest marker pointing at the version YAML diff, in both the live and journal-replay paths. - gaia/harvey evolve-gen authorSystem: explicit final-message discipline — every schema field filled with real values, never filler, even when the step budget runs out. - gaia evolve-smoke: gen-0 fixture now echoes 'placeholder' and asserts the marker (not the filler) reaches the next briefing and the report. tsc clean; gaia + harvey evolve smokes pass. Co-Authored-By: Claude Fable 5 --- mcp/src/lab/eval/steps/evolve-loop.ts | 23 +++++++++++++++++-- mcp/src/lab/gaia/evolve-smoke.ts | 15 ++++++++++-- .../lab/gaia/workflows/gaia-evolve-gen.yaml | 8 +++++++ .../harvey/workflows/harvey-evolve-gen.yaml | 8 +++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/mcp/src/lab/eval/steps/evolve-loop.ts b/mcp/src/lab/eval/steps/evolve-loop.ts index e879374ca..2b1b3bacf 100644 --- a/mcp/src/lab/eval/steps/evolve-loop.ts +++ b/mcp/src/lab/eval/steps/evolve-loop.ts @@ -73,6 +73,25 @@ function num(v: unknown): number | undefined { return typeof v === "number" && Number.isFinite(v) ? v : undefined; } +/** + * Authors occasionally end schema mode on a bare text turn, echoing filler + * ("placeholder", "") into `summary`. Junk there poisons every later + * briefing — the EXPLORE directive's "pick an approach that is none of the + * above" is only checkable against real summaries — and the report the + * human reads. Replace it with an honest marker instead of passing it + * through. (The version echo has its own fallback in the gen workflows; + * this is the summary-channel counterpart.) + */ +const NO_SUMMARY = + "(no usable approach summary reported by this generation's author — read this version's YAML diff to see what it changed)"; +function usableSummary(v: unknown): string | undefined { + if (typeof v !== "string") return undefined; + const t = v.trim(); + if (t.length < 8) return undefined; + if (/^(placeholder|todo|tbd|n\/?a|none|null|summary|unknown)[.!]?$/i.test(t)) return undefined; + return t; +} + function indent(s: string, pad: string): string { return s .split("\n") @@ -266,7 +285,7 @@ export default defineStep({ genRunId: String((journaled["runs"] as AnyRec[] | undefined)?.[0]?.["runId"] ?? ""), version: typeof journaled["version"] === "string" ? (journaled["version"] as string) : undefined, fitness, - summary: typeof journaled["summary"] === "string" ? (journaled["summary"] as string) : undefined, + summary: usableSummary(journaled["summary"]) ?? NO_SUMMARY, digestText: typeof journaled["digestText"] === "string" ? (journaled["digestText"] as string) : undefined, explore: journaled["directive"] === "explore", }; @@ -350,7 +369,7 @@ export default defineStep({ version: typeof out["version"] === "string" ? (out["version"] as string) : undefined, fitness, allPassCount: num(digest["allPassCount"]), - summary: typeof out["summary"] === "string" ? (out["summary"] as string) : undefined, + summary: usableSummary(out["summary"]) ?? NO_SUMMARY, changes: out["changes"], missingSecrets: out["missingSecrets"], digestText: typeof digest["text"] === "string" ? (digest["text"] as string) : undefined, diff --git a/mcp/src/lab/gaia/evolve-smoke.ts b/mcp/src/lab/gaia/evolve-smoke.ts index 99978edfb..23ea6c687 100644 --- a/mcp/src/lab/gaia/evolve-smoke.ts +++ b/mcp/src/lab/gaia/evolve-smoke.ts @@ -190,7 +190,9 @@ async function main() { output: { candidate: input.candidateName, version: `v${g + 1}`, - summary: `approach ${g}`, + // gen 0 echoes filler (the observed schema-mode failure); the + // loop must replace it with the honest no-summary marker. + summary: g === 0 ? "placeholder" : `approach ${g}`, authorCost: 1, digest: { fitness: rates[g], text: `digest ${g}`, results: [{ cost: 2 }] }, }, @@ -224,7 +226,16 @@ async function main() { assert.ok(genCalls[1].briefing.includes('the seeded produce workflow "gaia-produce"')); // margin 0 semantics: a TIE (0.4 vs baseline 0.4) does not become best assert.ok(genCalls[1].briefing.includes("BEST SO FAR: the baseline itself")); - console.log("✔ eval/evolve-loop: climbs gaia `fitness`, accuracy naming, margin-0 tie handling"); + // junk-summary guard: gen 0's "placeholder" echo must NOT reach the next + // briefing or the report — both carry the honest no-summary marker. + assert.ok(!genCalls[1].briefing.includes("placeholder")); + assert.ok(genCalls[1].briefing.includes("no usable approach summary")); + assert.equal( + loopOut.generations[0].summary.includes("no usable approach summary"), + true, + ); + assert.equal(loopOut.generations[1].summary, "approach 1"); + console.log("✔ eval/evolve-loop: climbs gaia `fitness`, accuracy naming, margin-0 tie handling, junk-summary guard"); console.log("\nALL GAIA EVOLVE VALIDATION CHECKS PASSED"); } finally { diff --git a/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml b/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml index a05bfd7ef..4e814c752 100644 --- a/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml +++ b/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml @@ -279,3 +279,11 @@ params: final healthy publish, a summary of the APPROACH this generation took (the next generation reads it to know what has been tried), the list of changes, and any missing secrets. + + Your structured answer is harvested from your FINAL message — the turn + where you stop calling tools. Fill every field with REAL values then: + never filler like "placeholder" or an empty summary. That summary is + the only record of this generation's approach the next generation and + the human report ever see; a generation whose approach is unreadable + gets retried by later generations, wasting their budget. If you run + out of steps, still describe truthfully whatever you actually shipped. diff --git a/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml b/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml index e86d2c3c3..1ae5ce14f 100644 --- a/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml @@ -295,3 +295,11 @@ params: final healthy publish, a summary of the APPROACH this generation took (the next generation reads it to know what has been tried), the list of changes, and any missing secrets. + + Your structured answer is harvested from your FINAL message — the turn + where you stop calling tools. Fill every field with REAL values then: + never filler like "placeholder" or an empty summary. That summary is + the only record of this generation's approach the next generation and + the human report ever see; a generation whose approach is unreadable + gets retried by later generations, wasting their budget. If you run + out of steps, still describe truthfully whatever you actually shipped.