Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions mcp/src/lab/createLabVein.ts
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,20 @@ export async function createLabVein(
serveUi: opts.serveUi ?? true,
});

// Inject the run-sub-workflows capability now that the instance exists. We
// mutate the SAME `services` object createVein holds by reference, so steps
// see it at run time. This is what lets `eval/optimize` loop eval→reflect.
services.optimizer = {
// Inject the run-sub-workflows capability now that the instance exists.
// CRITICAL: mutate `vein.services` — the EFFECTIVE bag createVein built by
// spreading our `services` into a fresh object (standardServices +
// artifacts + ours) — NOT the local `services`, which runs never see
// again. Mutating the local bag here silently broke every consumer of
// `services.optimizer` (eval/optimize, harvey/evolve-loop): steps threw
// "requires a services.optimizer capability" at run time. This is what
// lets the optimize/evolve loops run sub-workflows.
const optimizer: LabServices["optimizer"] = {
run: (name, input, runOpts) => vein.run(name, input, runOpts),
getParams: async (name) => (await vein.workspace.getWorkflow(name)).params ?? {},
};
(vein.services as LabServices).optimizer = optimizer;
services.optimizer = optimizer; // keep the caller's bag consistent too

return vein;
}
60 changes: 56 additions & 4 deletions mcp/src/lab/harvey/evolve-smoke.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { join } from "node:path";
import { WorkspaceManager, buildRegistry, fileArtifactsCapability, resolveConfig } from "vein";
import { seedHarveySteps, seedHarveyWorkflows } from "./seed.js";
import { seedArtifactSteps } from "../artifacts/seed.js";
import { createLabVein } from "../createLabVein.js";

async function main() {
const base = mkdtempSync(join(process.cwd(), ".evolve-validate-"));
Expand Down Expand Up @@ -207,13 +208,64 @@ async function main() {
assert.equal(failOut.improved, false);
console.log("✔ harvey/evolve-loop: aborts after consecutive failures");

// 6. REGRESSION — the optimizer capability must be visible to RUNS.
// createVein SPREADS the caller's services into a fresh bag, so
// createLabVein's post-construction injection must land on
// vein.services (the effective bag), not the local one. This broke
// silently once: eval/optimize and harvey/evolve-loop threw
// "requires a services.optimizer capability" at run time while the
// local bag looked fine. Prove it end to end: boot the real lab
// vein, publish a probe step + workflow, and assert a RUN sees
// services.optimizer.
// Construction-only requirement: concept services demand a provider key
// when the bag is built. The probe never calls an LLM — a dummy keeps
// this smoke offline and keyless.
const hadKey = process.env.ANTHROPIC_API_KEY;
if (!hadKey) process.env.ANTHROPIC_API_KEY = "sk-dummy-offline-smoke";
const labVein = await createLabVein({ workspacePath: join(base, "lab-ws"), serveUi: false });
if (!hadKey) delete process.env.ANTHROPIC_API_KEY;
assert.ok((labVein.services as any).optimizer, "vein.services.optimizer must be set");
await labVein.workspace.publishStep(
"smoke/has-optimizer",
`import { z, defineStep } from "vein";
export default defineStep({
type: "smoke/has-optimizer",
description: "probe: report whether ctx.services.optimizer is present",
input: z.object({}),
output: z.any(),
async run(_cfg, ctx) {
const opt = (ctx.services as any)?.optimizer;
return { hasOptimizer: !!opt && typeof opt.run === "function" };
},
});
`,
undefined,
"smoke",
);
await labVein.rebuildRegistry();
await labVein.workspace.publishWorkflowByContent(
"smoke-optimizer-probe",
"name: smoke-optimizer-probe\nsteps:\n - id: probe\n type: smoke/has-optimizer\n",
"smoke",
"smoke",
);
const probeRun = await labVein.run("smoke-optimizer-probe", {});
assert.equal(probeRun.status, "success", `probe run failed: ${JSON.stringify(probeRun.error)}`);
assert.deepEqual(probeRun.output, { hasOptimizer: true });
console.log("✔ services.optimizer reaches runs (createLabVein wiring)");

console.log("\nALL EVOLVE VALIDATION CHECKS PASSED");
} finally {
rmSync(base, { recursive: true, force: true });
}
}

main().catch((err) => {
console.error(err);
process.exit(1);
});
main().then(
// The lab-vein boot (section 6) leaves live handles (stores, services) —
// exit explicitly so the smoke terminates instead of idling forever.
() => process.exit(0),
(err) => {
console.error(err);
process.exit(1);
},
);
5 changes: 4 additions & 1 deletion mcp/src/lab/harvey/steps/evolve-loop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,10 @@ export function composeBriefing(args: {
lines.push(
`- attempt ${g.gen} → published ${args.candidateName}@${g.version ?? "?"}: mean pass-rate ${g.passRate} (${delta >= 0 ? "+" : ""}${delta} vs baseline)`,
);
if (g.summary) lines.push(` approach: ${excerpt(g.summary, 400)}`);
// The approach summary is the ONLY channel telling the EXPLORE
// directive what has already been tried — keep it roomy enough that
// "pick an approach that is none of the above" stays checkable.
if (g.summary) lines.push(` approach: ${excerpt(g.summary, 1200)}`);
if (g.digestText) lines.push(indent(excerpt(g.digestText, 700), " "));
}
}
Expand Down
33 changes: 29 additions & 4 deletions mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,32 @@ steps:
required: [candidate, version, summary]
additionalProperties: false

# ── resolve: never trust the author's echo ─────────────────────────────
# The candidate NAME is harness-pinned (input.candidateName) — grading the
# string the author returned once zeroed two generations on a literal
# "placeholder" (in schema mode, one no-tool-call text turn ends the agent
# loop and IS the structured output). The VERSION falls back to the
# candidate's active version — the author's own last publish, since
# generations run sequentially — when the echoed pin is empty or bogus.
# meta/get-workflow returns { error } instead of throwing, so `vpin.version`
# is undefined on a bad pin and the `||` falls through to the active one.
- id: vactive
type: meta/get-workflow
depends: author
config:
name: "{{ input.candidateName }}"

- id: vpin
type: meta/get-workflow
depends: author
config:
name: "{{ input.candidateName }}"
version: "{{ author.object.version }}"

# ── evaluate: the pinned candidate over the task set ───────────────────
- id: candeval
type: foreach
depends: author
depends: [vpin, vactive]
config:
items: "{{ input.tasks }}"
body:
Expand All @@ -99,8 +121,8 @@ steps:
config:
workflow: harvey-candidate-run
input:
workflow: "{{ author.object.candidate }}"
version: "{{ author.object.version }}"
workflow: "{{ input.candidateName }}"
version: "{{ vpin.version || vactive.version }}"
task: "{{ $current }}"

- id: canddigest
Expand All @@ -116,7 +138,10 @@ steps:
config:
candidate: "{{ input.candidateName }}"
generation: "{{ input.generation }}"
version: "{{ author.object.version }}"
# The version that was actually GRADED (resolved above) — the loop's
# briefing and EXPLOIT anchor read this, so an author's garbage echo
# must not poison the lineage.
version: "{{ vpin.version || vactive.version }}"
summary: "{{ author.object.summary }}"
changes: "{{ author.object.changes }}"
missingSecrets: "{{ author.object.missingSecrets }}"
Expand Down
Loading
Loading