diff --git a/mcp/src/lab/AGENTS.md b/mcp/src/lab/AGENTS.md index d6bd86691..8e6f2794e 100644 --- a/mcp/src/lab/AGENTS.md +++ b/mcp/src/lab/AGENTS.md @@ -528,6 +528,22 @@ Domain-agnostic eval substrate, shared by every experiment. See domains — harvey 0.02; produce-sampling noise for deterministic scorers — gaia 0). Needs `services.optimizer`. Wired by `harvey-evolve` (pass-rate) and `gaia-evolve` (accuracy). + Three guards keep the climb honest, all learned from live runs: + (a) **no-op generations.** An author can burn its budget and publish + nothing; the version fallback in the `*-evolve-gen` workflows then + resolves to the PREVIOUS generation's publish. The `published` gate + (`vbefore` vs the resolved version) catches that and skips `candeval` + entirely, so the generation reports `noop: true` and costs one author + instead of a whole task set. The loop records it with no fitness — a 0 + there would libel an approach that was never tried. + (b) **re-score guard.** A version this run already graded cannot become + the best on a second, luckier sample (`isNewBest` + the `scored` ledger). + Fitness is resampled, so without this, produce-sampling noise gets + written into the lineage as a hill-climb step. + (c) **budget caps.** `maxCost` / `maxMinutes`, checked BETWEEN generations + (never a mid-generation kill), both null by default. Generation count is + a poor budget on its own: authors reliably evolve toward more expensive + architectures, so per-generation cost and wall-clock GROW over a run. **Naming rule:** `eval/*` = generic. The eval *workflows* that wire these with a rubric/task/dataset belong to the experiment and are named `-…`. diff --git a/mcp/src/lab/eval/steps/evolve-loop.ts b/mcp/src/lab/eval/steps/evolve-loop.ts index 2b1b3bacf..196d697b8 100644 --- a/mcp/src/lab/eval/steps/evolve-loop.ts +++ b/mcp/src/lab/eval/steps/evolve-loop.ts @@ -92,6 +92,30 @@ function usableSummary(v: unknown): string | undefined { return t; } +/** + * A generation only becomes the new best if it BEAT the bar by more than the + * noise margin AND it is a version this run has not already scored. + * + * The second half matters because fitness is resampled: re-running an + * already-graded version can land above its own recorded score by produce- + * sampling luck alone. The gen workflows' `published` gate stops the common + * cause (an author that ships nothing, so the version fallback resolves to + * the previous generation's publish), but a *deliberate* republish of + * identical YAML under a new version string is indistinguishable from here — + * this is the backstop for the case the gate cannot see, and it keeps the + * reported best pinned to the run where that version was first measured. + */ +function isNewBest( + version: string | undefined, + fitness: number, + best: { fitness: number }, + margin: number, + scored: Map, +): boolean { + if (!(fitness > best.fitness + margin)) return false; + return version == null || !scored.has(version); +} + function indent(s: string, pad: string): string { return s .split("\n") @@ -114,6 +138,9 @@ interface GenEntry { produceCost?: number; explore: boolean; error?: string; + /** The author published nothing — nothing was graded, so there is no + * fitness datapoint here (see the gen workflows' `published` gate). */ + noop?: boolean; } export function composeBriefing(args: { @@ -146,6 +173,17 @@ export function composeBriefing(args: { lines.push(`- attempt ${g.gen}: FAILED to complete (${excerpt(g.error, 200)})`); continue; } + // A no-op attempt has no score to compare — saying "mean accuracy 0" + // here would read as a catastrophic approach rather than an author + // that never shipped, and would push later generations to explore + // away from a strategy that was never actually tried. + if (g.noop) { + lines.push( + `- attempt ${g.gen}: NO CANDIDATE PUBLISHED — its author finished without publishing a new ` + + `version, so nothing was graded. Do not read this as evidence about any approach.`, + ); + continue; + } const delta = Math.round((g.fitness - args.baselineFitness) * 1000) / 1000; lines.push( `- attempt ${g.gen} → published ${args.candidateName}@${g.version ?? "?"}: mean ${fitnessName} ${g.fitness} (${delta >= 0 ? "+" : ""}${delta} vs baseline)`, @@ -234,6 +272,23 @@ export default defineStep({ .record(z.any()) .optional() .describe("param overrides for the generation workflow (e.g. { authorModel, authorMaxSteps }) — applied via paramOverrides keyed by genWorkflow"), + // Generation COUNT is a poor budget: each generation costs whatever the + // architecture the authors evolved costs, and authors reliably evolve + // toward more expensive shapes (redundant attempts, reconcilers, extra + // verification passes). A 10-generation run that started at ~1h/gen can + // finish at ~2.5h/gen. These caps bound the run in the units a human + // actually budgets in. Both are checked BETWEEN generations, so the cap + // is a floor on when the loop stops, never a mid-generation kill. + maxCost: z + .number() + .positive() + .nullish() + .describe("stop before starting a generation once totalKnownCost (author + produce) reaches this many dollars — omit for no cost cap"), + maxMinutes: z + .number() + .positive() + .nullish() + .describe("stop before starting a generation once this many minutes of wall-clock have elapsed in the loop — omit for no time cap"), }), output: z.any(), async run(cfg, ctx) { @@ -250,6 +305,10 @@ export default defineStep({ : excerpt(JSON.stringify(baseline), 800); let best = { gen: -1, version: undefined as string | undefined, fitness: baselineFitness, digestText: baselineText }; + // version → the fitness it was FIRST measured at, so a later re-score of + // the same version cannot be promoted as an improvement (see isNewBest). + const scored = new Map(); + const loopStart = Date.now(); let sinceImprove = 0; let consecutiveFailures = 0; let totalKnownCost = 0; @@ -273,12 +332,26 @@ export default defineStep({ // code-step opt-in): pause parks here; cancel stops the loop here. await ctx.control?.checkpoint(); + // Budget gates, checked before spending the next generation. Deliberately + // NOT applied on the journal-replay path below: a resumed run must reach + // the same state it left, and replay spends nothing. + const elapsedMin = (Date.now() - loopStart) / 60000; + if (cfg.maxCost != null && totalKnownCost >= cfg.maxCost) { + stopReason = `maxCost $${cfg.maxCost} reached (spent $${Math.round(totalKnownCost * 100) / 100}) after ${gen} generation(s)`; + break; + } + if (cfg.maxMinutes != null && elapsedMin >= cfg.maxMinutes) { + stopReason = `maxMinutes ${cfg.maxMinutes} reached (elapsed ${Math.round(elapsedMin)}m) after ${gen} generation(s)`; + break; + } + // Durable resume (§5, iterative code steps): a generation whose // synthetic `#gen` step.end is journaled replays — its run is NOT // re-launched. State (best / sinceImprove / stop logic) is rebuilt // from the journaled output so the loop continues where it left off. const journaled = ctx.journal?.[`${ctx.path}#${gen}`] as AnyRec | undefined; if (journaled) { + const noop = journaled["noop"] === true; const fitness = num(journaled["fitness"]) ?? num(journaled["passRate"]) ?? 0; const entry: GenEntry = { gen, @@ -288,18 +361,20 @@ export default defineStep({ summary: usableSummary(journaled["summary"]) ?? NO_SUMMARY, digestText: typeof journaled["digestText"] === "string" ? (journaled["digestText"] as string) : undefined, explore: journaled["directive"] === "explore", + ...(noop ? { noop: true } : {}), }; generations.push(entry); consecutiveFailures = 0; totalKnownCost += num(journaled["knownCost"]) ?? 0; - if (fitness > best.fitness + cfg.improveMargin) { + if (!noop && isNewBest(entry.version, fitness, best, cfg.improveMargin, scored)) { best = { gen, version: entry.version, fitness, digestText: entry.digestText ?? "" }; sinceImprove = 0; } else { sinceImprove++; } + if (!noop && entry.version && !scored.has(entry.version)) scored.set(entry.version, fitness); await emitGen(gen, { type: "step.replayed", output: journaled }); - if (fitness >= cfg.stopFitness) { + if (!noop && fitness >= cfg.stopFitness) { stopReason = `stopFitness ${cfg.stopFitness} reached`; break; } @@ -356,6 +431,43 @@ export default defineStep({ consecutiveFailures = 0; const out = (run.output ?? {}) as AnyRec; + + // NO-OP generation: the gen workflow's `published` gate found that this + // generation's author shipped no new version, so it skipped grading + // rather than re-running an already-scored version over the whole task + // set. Record the wasted author budget, leave `best` alone, and let the + // non-improvement push the directive toward explore — but never write a + // fitness of 0, which would libel an approach that was never tried. + if (out["noop"] === true) { + const authorOnly = num(out["authorCost"]) ?? 0; + totalKnownCost += authorOnly; + generations.push({ + gen, + genRunId: run.runId, + fitness: 0, + noop: true, + explore, + authorCost: authorOnly, + summary: usableSummary(out["summary"]) ?? NO_SUMMARY, + }); + sinceImprove++; + await emitGen(gen, { + type: "step.end", + durationMs: Date.now() - genStart, + output: { + gen, + directive: explore ? "explore" : "exploit", + noop: true, + note: "author published no new candidate version — grading skipped, no fitness recorded", + bestFitness: best.fitness, + bestGen: best.gen, + knownCost: Math.round(authorOnly * 10000) / 10000, + runs: [{ label: `generation ${gen} (no-op)`, workflow: cfg.genWorkflow, runId: run.runId }], + }, + }); + continue; + } + const digest = (out["digest"] ?? {}) as AnyRec; const fitness = num(digest["fitness"]) ?? num(digest["meanPassRate"]) ?? 0; const digestResults = Array.isArray(digest["results"]) ? (digest["results"] as AnyRec[]) : []; @@ -379,12 +491,14 @@ export default defineStep({ }; generations.push(entry); - if (fitness > best.fitness + cfg.improveMargin) { + const rescored = entry.version != null && scored.has(entry.version); + if (isNewBest(entry.version, fitness, best, cfg.improveMargin, scored)) { best = { gen, version: entry.version, fitness, digestText: entry.digestText ?? "" }; sinceImprove = 0; } else { sinceImprove++; } + if (entry.version && !rescored) scored.set(entry.version, fitness); await emitGen(gen, { type: "step.end", @@ -396,6 +510,9 @@ export default defineStep({ fitness, bestFitness: best.fitness, bestGen: best.gen, + // A version this run already scored — its fitness here is a + // resample, not a hill-climb step, and cannot become the best. + ...(rescored ? { rescoredVersion: true } : {}), knownCost: Math.round((authorCost + produceCost) * 10000) / 10000, runs: [{ label: `generation ${gen}`, workflow: cfg.genWorkflow, runId: run.runId }], // Carried so a durable resume can rebuild later generations' diff --git a/mcp/src/lab/gaia/evolve-smoke.ts b/mcp/src/lab/gaia/evolve-smoke.ts index 23ea6c687..36c7474a6 100644 --- a/mcp/src/lab/gaia/evolve-smoke.ts +++ b/mcp/src/lab/gaia/evolve-smoke.ts @@ -70,7 +70,27 @@ async function main() { const fallbackScope = { ...scope, grade: { taskId: "t-1", isCorrect: false, answer: "", level: null, gradeError: "boom" } }; assert.equal((resolveConfig as any)("{{ grade.isCorrect }}", fallbackScope), false); assert.equal((resolveConfig as any)("{{ grade.benchmarkRev }}", fallbackScope), undefined); - console.log("✔ template expressions resolve (fallback, version pin, whole-run pass; no-short-circuit guarded)"); + // gaia-evolve-gen's `published` gate: did this generation's author ship a + // NEW version, or did the version fallback just land back on the previous + // generation's publish? Every access here must stay undefined-safe, since + // meta/get-workflow returns { error } (never undefined) for a miss. + const GATE = + "{{ (vpin.version || vactive.version) && (vpin.version || vactive.version) !== vbefore.version }}"; + const gate = (vbefore: unknown, vpin: unknown, vactive: unknown) => + Boolean((resolveConfig as any)(GATE, { ...scope, vbefore, vpin, vactive })); + // gen 0: candidate does not exist yet, author publishes v1 → shipped + assert.equal(gate({ error: "not found" }, { version: "v1" }, { version: "v1" }), true); + // gen 0: author publishes nothing at all → no-op + assert.equal(gate({ error: "not found" }, {}, { error: "not found" }), false); + // gen N: author publishes a new version → shipped + assert.equal(gate({ version: "v11" }, { version: "v12" }, { version: "v12" }), true); + // gen N: author echoes garbage and published nothing, so the fallback + // resolves to the PREVIOUS generation's publish → no-op (the live bug) + assert.equal(gate({ version: "v11" }, {}, { version: "v11" }), false); + // the no-op flag handed to the loop is the gate's negation + assert.equal((resolveConfig as any)("{{ !published }}", { ...scope, published: true }), false); + assert.equal((resolveConfig as any)("{{ !published }}", { ...scope, published: false }), true); + console.log("✔ template expressions resolve (fallback, version pin, whole-run pass, published gate; no-short-circuit guarded)"); // 3. gaia/evaluate fromRun: unpack-in-code semantics against a faked // scoring service (no python, no dataset). @@ -237,6 +257,117 @@ async function main() { assert.equal(loopOut.generations[1].summary, "approach 1"); console.log("✔ eval/evolve-loop: climbs gaia `fitness`, accuracy naming, margin-0 tie handling, junk-summary guard"); + // 6. NO-OP generation: the gen workflow's `published` gate reports that an + // author shipped nothing, so nothing was graded. The loop must record + // it without a fitness, leave `best` untouched, spend only the author + // cost, and tell the next generation not to read it as evidence. + const noopCalls: any[] = []; + const noopOpt = { + run: async (_name: string, input: any) => { + noopCalls.push(input); + const g = input.generation as number; + return { + runId: `genrun-${g}`, + status: "success", + output: + g === 1 + ? { candidate: input.candidateName, noop: true, authorCost: 1, summary: "ran out of steps" } + : { + candidate: input.candidateName, + version: `v${g + 1}`, + summary: `approach ${g}`, + authorCost: 1, + digest: { fitness: 0.6, text: `digest ${g}`, results: [{ cost: 2 }] }, + }, + }; + }, + }; + const noopBase = { + tasks: ["t-1", "t-2"], + mission: "m", + baseline: { fitness: 0.4, text: "baseline digest" }, + candidateName: "gaia-produce-ai", + baseWorkflow: "gaia-produce", + genWorkflow: "gaia-evolve-gen", + fitnessName: "accuracy", + stopFitness: 1, + improveMargin: 0, + exploreAfter: 2, + }; + const noopOut: any = await loop.run( + loop.input.parse({ ...noopBase, maxGenerations: 3 }), + { ...ctxStub, services: { optimizer: noopOpt } }, + ); + assert.equal(noopOut.generations[1].noop, true); + assert.equal(noopOut.bestGen, 0); // gen 1 did not displace gen 0's v1 + assert.equal(noopOut.bestVersion, "v1"); + assert.equal(noopOut.bestFitness, 0.6); + // no-op spends the author budget only — never the 2 tasks × cost 2 produce + assert.equal(noopOut.totalKnownCost, 3 + 1 + 3); + // the briefing must not libel an approach that was never tried + assert.ok(noopCalls[2].briefing.includes("NO CANDIDATE PUBLISHED")); + assert.ok(!noopCalls[2].briefing.includes("attempt 1 → published")); + console.log("✔ eval/evolve-loop: no-op generation records no fitness, spends only the author budget"); + + // 7. RE-SCORE guard: the same version graded twice cannot be promoted on + // the luckier sample — the run's best stays pinned to first measurement. + const dupOpt = { + run: async (_name: string, input: any) => { + const g = input.generation as number; + return { + runId: `genrun-${g}`, + status: "success", + output: { + candidate: input.candidateName, + version: "v1", // gen 1 re-runs gen 0's version… + summary: `approach ${g}`, + authorCost: 1, + digest: { fitness: g === 0 ? 0.6 : 0.9, text: `digest ${g}`, results: [{ cost: 2 }] }, // …and gets lucky + }, + }; + }, + }; + const dupOut: any = await loop.run( + loop.input.parse({ ...noopBase, maxGenerations: 2 }), + { ...ctxStub, services: { optimizer: dupOpt } }, + ); + assert.equal(dupOut.bestGen, 0); + assert.equal(dupOut.bestFitness, 0.6); // NOT 0.9 — a resample, not a climb + assert.equal(dupOut.generations[1].fitness, 0.9); // still reported honestly + console.log("✔ eval/evolve-loop: a re-scored version cannot become the best on sampling luck"); + + // 8. BUDGET caps stop the loop between generations. + const costOpt = { + run: async (_name: string, input: any) => ({ + runId: `genrun-${input.generation}`, + status: "success", + output: { + candidate: input.candidateName, + version: `v${input.generation + 1}`, + summary: `approach ${input.generation}`, + authorCost: 1, + digest: { fitness: 0.5, text: "d", results: [{ cost: 2 }] }, + }, + }), + }; + const cappedOut: any = await loop.run( + loop.input.parse({ ...noopBase, maxGenerations: 10, maxCost: 8 }), + { ...ctxStub, services: { optimizer: costOpt } }, + ); + // $3/gen (author 1 + produce 2); the gate trips before gen 3, at $9 ≥ $8 + assert.equal(cappedOut.generations.length, 3); + assert.ok(cappedOut.stopReason.includes("maxCost $8 reached")); + const uncappedOut: any = await loop.run( + loop.input.parse({ ...noopBase, maxGenerations: 10 }), + { ...ctxStub, services: { optimizer: costOpt } }, + ); + assert.equal(uncappedOut.generations.length, 10); // no cap = unchanged + // gaia-evolve wires `{{ input.maxCost || params.maxCost }}`, and an unset + // YAML param resolves to null — the schema must read that as "uncapped" + // rather than rejecting the whole step. + assert.equal(loop.input.parse({ ...noopBase, maxGenerations: 1, maxCost: null, maxMinutes: null }).maxCost, null); + console.log("✔ eval/evolve-loop: maxCost stops between generations, absent caps change nothing"); + console.log("\nALL GAIA EVOLVE VALIDATION CHECKS PASSED"); } finally { rmSync(base, { recursive: true, force: true }); diff --git a/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml b/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml index 4e814c752..cc0ed050a 100644 --- a/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml +++ b/mcp/src/lab/gaia/workflows/gaia-evolve-gen.yaml @@ -27,10 +27,20 @@ steps: config: sub: author + # The candidate's active version BEFORE this generation's author runs. + # If the author publishes nothing, the version fallback below resolves to + # this very version — a silent re-run of a version already graded (see the + # `published` gate). Must complete before the author can publish, hence + # the explicit ordering edge into `author`. + - id: vbefore + type: meta/get-workflow + config: + name: "{{ input.candidateName }}" + # ── propose: the authoring agent ─────────────────────────────────────── - id: author type: agent - depends: dir + depends: [dir, vbefore] options: retry: { max: 1, delayMs: 15000 } config: @@ -112,9 +122,30 @@ steps: version: "{{ author.object.version }}" # ── evaluate: the pinned candidate over the task set ─────────────────── + # ── did this generation actually SHIP anything? ──────────────────────── + # An author can burn its whole budget and publish NOTHING — observed live + # when the agent degenerated into a filler loop and echoed a garbage + # version string. The fallback above then resolves to the candidate's + # active version, which is the PREVIOUS generation's publish: the harness + # would re-run an already-graded version over the whole task set, costing + # a full generation's produce budget to learn nothing. Worse, exact-match + # scoring resamples — the re-run can land above its own recorded fitness + # by luck alone and be written into the hill-climb as an improvement. + # + # Gate on it instead: same version in as out ⇒ nothing was published ⇒ + # skip `candeval` entirely and report a no-op generation. At generation 0 + # the candidate does not exist yet, so `vbefore.version` is undefined and + # any real publish trips the gate true. + - id: published + type: if + depends: [vpin, vactive, vbefore] + config: + cond: "{{ (vpin.version || vactive.version) && (vpin.version || vactive.version) !== vbefore.version }}" + - id: candeval type: foreach - depends: [vpin, vactive] + depends: published + when: true config: items: "{{ input.tasks }}" body: @@ -133,15 +164,24 @@ steps: config: results: "{{ candeval }}" + # Depends on the GATE as well as the digest so it runs on both branches: + # vein only skip-propagates when EVERY dep was skipped, and `published` + # always runs. On the no-op branch `canddigest` is skipped, so `digest` is + # undefined and `noop` tells the loop to score nothing. - id: result type: gaia/pack-result - depends: canddigest + depends: [canddigest, published] config: candidate: "{{ input.candidateName }}" generation: "{{ input.generation }}" + # True when this generation's author published NOTHING — the loop must + # record it as a no-op, not as a fitness datapoint (a skipped candeval + # digests to nothing, which would otherwise read as fitness 0). + noop: "{{ !published }}" # The version that was actually GRADED (resolved above) — the loop's # briefing and EXPLOIT anchor read this, so an author's garbage echo - # must not poison the lineage. + # must not poison the lineage. Undefined-safe on the no-op branch, + # where nothing was graded at all. version: "{{ vpin.version || vactive.version }}" summary: "{{ author.object.summary }}" changes: "{{ author.object.changes }}" diff --git a/mcp/src/lab/gaia/workflows/gaia-evolve.yaml b/mcp/src/lab/gaia/workflows/gaia-evolve.yaml index c56a6bc19..6e91f49f7 100644 --- a/mcp/src/lab/gaia/workflows/gaia-evolve.yaml +++ b/mcp/src/lab/gaia/workflows/gaia-evolve.yaml @@ -46,6 +46,12 @@ name: gaia-evolve # author (the digest evidence rides alongside it and, per # the author method, outranks it). # generations: override params.maxGenerations for this run (e.g. 10). +# maxCost: dollar ceiling (author + produce) — the loop stops before +# starting the generation that would cross it. +# maxMinutes: wall-clock ceiling, same between-generations semantics. +# Both default to params (null = uncapped). Prefer these over +# guessing a generation count: cost and time per generation +# GROW as authors evolve more expensive architectures. # Output: { candidate, bestGen, bestVersion, bestAccuracy, # baselineAccuracy, improved, generations, totalKnownCost, # stopReason, baseline, … } @@ -86,6 +92,8 @@ steps: improveMargin: "{{ params.improveMargin }}" exploreAfter: "{{ params.exploreAfter }}" genParams: "{{ params.genParams }}" + maxCost: "{{ input.maxCost || params.maxCost }}" + maxMinutes: "{{ input.maxMinutes || params.maxMinutes }}" # ── promote: the reviewable report ───────────────────────────────────── - id: report @@ -133,3 +141,10 @@ params: # Param overrides for gaia-evolve-gen (e.g. { authorModel: "...", # authorMaxSteps: 120, authorGuidance: "..." }). Empty = its defaults. genParams: {} + # BUDGET CAPS (checked between generations, so they bound when the loop + # STOPS, never kill a generation mid-flight). Generation count alone is a + # poor budget: authors evolve toward more expensive architectures, so a + # run that starts at ~1h and ~$3 per generation can end at 2.5h and $8. + # Null = uncapped (generations remain the only limit). + maxCost: null + maxMinutes: null diff --git a/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml b/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml index 1ae5ce14f..4c4580fc4 100644 --- a/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml @@ -25,10 +25,20 @@ steps: config: sub: author + # The candidate's active version BEFORE this generation's author runs. + # If the author publishes nothing, the version fallback below resolves to + # this very version — a silent re-run of a version already graded (see the + # `published` gate). Must complete before the author can publish, hence + # the explicit ordering edge into `author`. + - id: vbefore + type: meta/get-workflow + config: + name: "{{ input.candidateName }}" + # ── propose: the authoring agent ─────────────────────────────────────── - id: author type: agent - depends: dir + depends: [dir, vbefore] options: retry: { max: 1, delayMs: 15000 } config: @@ -110,9 +120,30 @@ steps: version: "{{ author.object.version }}" # ── evaluate: the pinned candidate over the task set ─────────────────── + # ── did this generation actually SHIP anything? ──────────────────────── + # An author can burn its whole budget and publish NOTHING — observed live + # when the agent degenerated into a filler loop and echoed a garbage + # version string. The fallback above then resolves to the candidate's + # active version, which is the PREVIOUS generation's publish: the harness + # would re-run an already-graded version over the whole task set, costing + # a full generation's produce budget to learn nothing. Worse, exact-match + # scoring resamples — the re-run can land above its own recorded fitness + # by luck alone and be written into the hill-climb as an improvement. + # + # Gate on it instead: same version in as out ⇒ nothing was published ⇒ + # skip `candeval` entirely and report a no-op generation. At generation 0 + # the candidate does not exist yet, so `vbefore.version` is undefined and + # any real publish trips the gate true. + - id: published + type: if + depends: [vpin, vactive, vbefore] + config: + cond: "{{ (vpin.version || vactive.version) && (vpin.version || vactive.version) !== vbefore.version }}" + - id: candeval type: foreach - depends: [vpin, vactive] + depends: published + when: true config: items: "{{ input.tasks }}" body: @@ -132,15 +163,24 @@ steps: results: "{{ candeval }}" maxCriteria: "{{ params.digestMaxCriteria }}" + # Depends on the GATE as well as the digest so it runs on both branches: + # vein only skip-propagates when EVERY dep was skipped, and `published` + # always runs. On the no-op branch `canddigest` is skipped, so `digest` is + # undefined and `noop` tells the loop to score nothing. - id: result type: harvey/pack-result - depends: canddigest + depends: [canddigest, published] config: candidate: "{{ input.candidateName }}" generation: "{{ input.generation }}" + # True when this generation's author published NOTHING — the loop must + # record it as a no-op, not as a fitness datapoint (a skipped candeval + # digests to nothing, which would otherwise read as fitness 0). + noop: "{{ !published }}" # The version that was actually GRADED (resolved above) — the loop's # briefing and EXPLOIT anchor read this, so an author's garbage echo - # must not poison the lineage. + # must not poison the lineage. Undefined-safe on the no-op branch, + # where nothing was graded at all. version: "{{ vpin.version || vactive.version }}" summary: "{{ author.object.summary }}" changes: "{{ author.object.changes }}" diff --git a/mcp/src/lab/harvey/workflows/harvey-evolve.yaml b/mcp/src/lab/harvey/workflows/harvey-evolve.yaml index 140d0ff66..76a6e7919 100644 --- a/mcp/src/lab/harvey/workflows/harvey-evolve.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-evolve.yaml @@ -42,6 +42,12 @@ name: harvey-evolve # author (the digest evidence rides alongside it and, per # the author method, outranks it). # generations: override params.maxGenerations for this run (e.g. 10). +# maxCost: dollar ceiling (author + produce) — the loop stops before +# starting the generation that would cross it. +# maxMinutes: wall-clock ceiling, same between-generations semantics. +# Both default to params (null = uncapped). Prefer these over +# guessing a generation count: cost and time per generation +# GROW as authors evolve more expensive architectures. # Output: { candidate, bestGen, bestVersion, bestPassRate, # baselinePassRate, improved, generations, totalKnownCost, # stopReason, baseline, … } @@ -86,6 +92,8 @@ steps: improveMargin: "{{ params.improveMargin }}" exploreAfter: "{{ params.exploreAfter }}" genParams: "{{ params.genParams }}" + maxCost: "{{ input.maxCost || params.maxCost }}" + maxMinutes: "{{ input.maxMinutes || params.maxMinutes }}" # ── promote: the reviewable report ───────────────────────────────────── - id: report @@ -133,3 +141,10 @@ params: # Param overrides for harvey-evolve-gen (e.g. { authorModel: "...", # authorMaxSteps: 120, authorGuidance: "..." }). Empty = its defaults. genParams: {} + # BUDGET CAPS (checked between generations, so they bound when the loop + # STOPS, never kill a generation mid-flight). Generation count alone is a + # poor budget: authors evolve toward more expensive architectures, so a + # run that starts at ~1h and ~$3 per generation can end at 2.5h and $8. + # Null = uncapped (generations remain the only limit). + maxCost: null + maxMinutes: null