From 40793ebdacbe92aa7bdc6f69b213ef7c8037fdb9 Mon Sep 17 00:00:00 2001 From: Evanfeenstra Date: Fri, 28 Aug 2026 07:51:31 -0700 Subject: [PATCH 1/4] lab: services.optimizer must land on vein.services (runs never saw it) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit createVein SPREADS the caller's services into a fresh effective bag (standardServices + artifacts + caller's), so createLabVein's post-construction `services.optimizer = …` mutated an object no run ever read again — its comment ("mutate the SAME object createVein holds by reference") described behavior a vein refactor removed. Every consumer of services.optimizer was silently broken at run time: harvey/evolve-loop (seen live: "requires a services.optimizer capability") and eval/optimize (gitsee-optimize / concepts-optimize would hit the same wall). Inject on vein.services (the effective bag) instead, keeping the caller's bag consistent too. evolve-smoke gains the end-to-end regression check: boot the real lab vein (offline, dummy key for construction-only provider checks), publish a probe step + workflow, and assert a RUN sees ctx.services.optimizer — plus an explicit exit, since the booted vein's live handles otherwise keep the smoke process alive forever. Co-Authored-By: Claude Fable 5 --- mcp/src/lab/createLabVein.ts | 15 ++++++-- mcp/src/lab/harvey/evolve-smoke.ts | 60 ++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/mcp/src/lab/createLabVein.ts b/mcp/src/lab/createLabVein.ts index 1a547015f..f370599a2 100644 --- a/mcp/src/lab/createLabVein.ts +++ b/mcp/src/lab/createLabVein.ts @@ -167,13 +167,20 @@ export async function createLabVein( serveUi: opts.serveUi ?? true, }); - // Inject the run-sub-workflows capability now that the instance exists. We - // mutate the SAME `services` object createVein holds by reference, so steps - // see it at run time. This is what lets `eval/optimize` loop eval→reflect. - services.optimizer = { + // Inject the run-sub-workflows capability now that the instance exists. + // CRITICAL: mutate `vein.services` — the EFFECTIVE bag createVein built by + // spreading our `services` into a fresh object (standardServices + + // artifacts + ours) — NOT the local `services`, which runs never see + // again. Mutating the local bag here silently broke every consumer of + // `services.optimizer` (eval/optimize, harvey/evolve-loop): steps threw + // "requires a services.optimizer capability" at run time. This is what + // lets the optimize/evolve loops run sub-workflows. + const optimizer: LabServices["optimizer"] = { run: (name, input, runOpts) => vein.run(name, input, runOpts), getParams: async (name) => (await vein.workspace.getWorkflow(name)).params ?? {}, }; + (vein.services as LabServices).optimizer = optimizer; + services.optimizer = optimizer; // keep the caller's bag consistent too return vein; } diff --git a/mcp/src/lab/harvey/evolve-smoke.ts b/mcp/src/lab/harvey/evolve-smoke.ts index 484910a0c..6ebd9fb23 100644 --- a/mcp/src/lab/harvey/evolve-smoke.ts +++ b/mcp/src/lab/harvey/evolve-smoke.ts @@ -13,6 +13,7 @@ import { join } from "node:path"; import { WorkspaceManager, buildRegistry, fileArtifactsCapability, resolveConfig } from "vein"; import { seedHarveySteps, seedHarveyWorkflows } from "./seed.js"; import { seedArtifactSteps } from "../artifacts/seed.js"; +import { createLabVein } from "../createLabVein.js"; async function main() { const base = mkdtempSync(join(process.cwd(), ".evolve-validate-")); @@ -207,13 +208,64 @@ async function main() { assert.equal(failOut.improved, false); console.log("✔ harvey/evolve-loop: aborts after consecutive failures"); + // 6. REGRESSION — the optimizer capability must be visible to RUNS. + // createVein SPREADS the caller's services into a fresh bag, so + // createLabVein's post-construction injection must land on + // vein.services (the effective bag), not the local one. This broke + // silently once: eval/optimize and harvey/evolve-loop threw + // "requires a services.optimizer capability" at run time while the + // local bag looked fine. Prove it end to end: boot the real lab + // vein, publish a probe step + workflow, and assert a RUN sees + // services.optimizer. + // Construction-only requirement: concept services demand a provider key + // when the bag is built. The probe never calls an LLM — a dummy keeps + // this smoke offline and keyless. + const hadKey = process.env.ANTHROPIC_API_KEY; + if (!hadKey) process.env.ANTHROPIC_API_KEY = "sk-dummy-offline-smoke"; + const labVein = await createLabVein({ workspacePath: join(base, "lab-ws"), serveUi: false }); + if (!hadKey) delete process.env.ANTHROPIC_API_KEY; + assert.ok((labVein.services as any).optimizer, "vein.services.optimizer must be set"); + await labVein.workspace.publishStep( + "smoke/has-optimizer", + `import { z, defineStep } from "vein"; +export default defineStep({ + type: "smoke/has-optimizer", + description: "probe: report whether ctx.services.optimizer is present", + input: z.object({}), + output: z.any(), + async run(_cfg, ctx) { + const opt = (ctx.services as any)?.optimizer; + return { hasOptimizer: !!opt && typeof opt.run === "function" }; + }, +}); +`, + undefined, + "smoke", + ); + await labVein.rebuildRegistry(); + await labVein.workspace.publishWorkflowByContent( + "smoke-optimizer-probe", + "name: smoke-optimizer-probe\nsteps:\n - id: probe\n type: smoke/has-optimizer\n", + "smoke", + "smoke", + ); + const probeRun = await labVein.run("smoke-optimizer-probe", {}); + assert.equal(probeRun.status, "success", `probe run failed: ${JSON.stringify(probeRun.error)}`); + assert.deepEqual(probeRun.output, { hasOptimizer: true }); + console.log("✔ services.optimizer reaches runs (createLabVein wiring)"); + console.log("\nALL EVOLVE VALIDATION CHECKS PASSED"); } finally { rmSync(base, { recursive: true, force: true }); } } -main().catch((err) => { - console.error(err); - process.exit(1); -}); +main().then( + // The lab-vein boot (section 6) leaves live handles (stores, services) — + // exit explicitly so the smoke terminates instead of idling forever. + () => process.exit(0), + (err) => { + console.error(err); + process.exit(1); + }, +); From f1a49382c965a7c9bb8d98ff611a41a9553e173c Mon Sep 17 00:00:00 2001 From: Evanfeenstra Date: Fri, 28 Aug 2026 09:02:19 -0700 Subject: [PATCH 2/4] =?UTF-8?q?harvey:=20never=20trust=20the=20author's=20?= =?UTF-8?q?echo=20=E2=80=94=20resolve=20the=20graded=20candidate=20in=20th?= =?UTF-8?q?e=20harness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two generations zeroed out when the author's schema-mode final turn degenerated to a literal {"candidate": "placeholder", ...} — the harness graded a ghost workflow while the author's real publishes (v5, v6) sat unmeasured. candeval now runs the harness-pinned input.candidateName, and vpin/vactive resolve the version to grade (the echoed pin when it exists, else the candidate's active version — the author's own last publish, since generations run sequentially). The result step reports the resolved version so a garbage echo can't poison the briefing lineage or the EXPLOIT anchor. Also bump the briefing's approach-summary excerpt 400 → 1200 chars: it is the only channel telling the EXPLORE directive what has already been tried. Co-Authored-By: Claude Fable 5 --- mcp/src/lab/harvey/steps/evolve-loop.ts | 5 ++- .../harvey/workflows/harvey-evolve-gen.yaml | 33 ++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/mcp/src/lab/harvey/steps/evolve-loop.ts b/mcp/src/lab/harvey/steps/evolve-loop.ts index bd7cd4774..a687f2120 100644 --- a/mcp/src/lab/harvey/steps/evolve-loop.ts +++ b/mcp/src/lab/harvey/steps/evolve-loop.ts @@ -112,7 +112,10 @@ export function composeBriefing(args: { lines.push( `- attempt ${g.gen} → published ${args.candidateName}@${g.version ?? "?"}: mean pass-rate ${g.passRate} (${delta >= 0 ? "+" : ""}${delta} vs baseline)`, ); - if (g.summary) lines.push(` approach: ${excerpt(g.summary, 400)}`); + // The approach summary is the ONLY channel telling the EXPLORE + // directive what has already been tried — keep it roomy enough that + // "pick an approach that is none of the above" stays checkable. + if (g.summary) lines.push(` approach: ${excerpt(g.summary, 1200)}`); if (g.digestText) lines.push(indent(excerpt(g.digestText, 700), " ")); } } diff --git a/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml b/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml index 2ed3357c0..006404cb1 100644 --- a/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml +++ b/mcp/src/lab/harvey/workflows/harvey-evolve-gen.yaml @@ -87,10 +87,32 @@ steps: required: [candidate, version, summary] additionalProperties: false + # ── resolve: never trust the author's echo ───────────────────────────── + # The candidate NAME is harness-pinned (input.candidateName) — grading the + # string the author returned once zeroed two generations on a literal + # "placeholder" (in schema mode, one no-tool-call text turn ends the agent + # loop and IS the structured output). The VERSION falls back to the + # candidate's active version — the author's own last publish, since + # generations run sequentially — when the echoed pin is empty or bogus. + # meta/get-workflow returns { error } instead of throwing, so `vpin.version` + # is undefined on a bad pin and the `||` falls through to the active one. + - id: vactive + type: meta/get-workflow + depends: author + config: + name: "{{ input.candidateName }}" + + - id: vpin + type: meta/get-workflow + depends: author + config: + name: "{{ input.candidateName }}" + version: "{{ author.object.version }}" + # ── evaluate: the pinned candidate over the task set ─────────────────── - id: candeval type: foreach - depends: author + depends: [vpin, vactive] config: items: "{{ input.tasks }}" body: @@ -99,8 +121,8 @@ steps: config: workflow: harvey-candidate-run input: - workflow: "{{ author.object.candidate }}" - version: "{{ author.object.version }}" + workflow: "{{ input.candidateName }}" + version: "{{ vpin.version || vactive.version }}" task: "{{ $current }}" - id: canddigest @@ -116,7 +138,10 @@ steps: config: candidate: "{{ input.candidateName }}" generation: "{{ input.generation }}" - version: "{{ author.object.version }}" + # The version that was actually GRADED (resolved above) — the loop's + # briefing and EXPLOIT anchor read this, so an author's garbage echo + # must not poison the lineage. + version: "{{ vpin.version || vactive.version }}" summary: "{{ author.object.summary }}" changes: "{{ author.object.changes }}" missingSecrets: "{{ author.object.missingSecrets }}" From 51e5d2d0cac525a00c5a342fac7442f8ed4b6db8 Mon Sep 17 00:00:00 2001 From: Evanfeenstra Date: Fri, 28 Aug 2026 09:02:28 -0700 Subject: [PATCH 3/4] =?UTF-8?q?vein:=20version=20picker=20=E2=80=94=20brow?= =?UTF-8?q?se=20a=20workflow's=20published=20lineage=20in=20the=20UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UI always loaded the active version; agent-evolved workflows (harvey-produce-ai v1…v7) had no way to show how their structure changed across generations. /workflows/:name/flow now takes ?version=, and a topbar dropdown pins the canvas to any published version. Historical views are read-only (Publish/Run/add-step/edge edits gated off, badge shown) — Publish builds on active and Run runs it, so editing the past would mislead. The pin self-invalidates on workflow switch and resets when a run is selected (run events overlay the active structure). Co-Authored-By: Claude Fable 5 --- vein/src/createVein.ts | 7 +++- vein/web/src/api.ts | 8 +++-- vein/web/src/app.tsx | 54 ++++++++++++++++++++++++------ vein/web/src/styles/components.css | 21 ++++++++++++ 4 files changed, 77 insertions(+), 13 deletions(-) diff --git a/vein/src/createVein.ts b/vein/src/createVein.ts index e797a3c5c..b70bd8e3c 100644 --- a/vein/src/createVein.ts +++ b/vein/src/createVein.ts @@ -611,8 +611,13 @@ export async function createVein( app.get("/workflows/:name/flow", async (c) => { const name = c.req.param("name"); + // ?version= pins a historical version (the UI's version picker); + // omitted = the active version, as before. + const version = c.req.query("version"); try { - const flow = await workspace.getWorkflow(name); + const flow = version + ? await workspace.getWorkflowVersion(name, version) + : await workspace.getWorkflow(name); return c.json({ name: flow.name, steps: flow.steps, diff --git a/vein/web/src/api.ts b/vein/web/src/api.ts index eaf47e9ea..058c4efb4 100644 --- a/vein/web/src/api.ts +++ b/vein/web/src/api.ts @@ -75,8 +75,12 @@ export interface FlowDef { promotes?: PromoteSpec[]; } -export const getWorkflowFlow = (name: string) => - fetchJSON(`/workflows/${name}/flow`); +/** Parsed flow of a workflow — the active version, or a pinned historical + * one when `version` is given (the version picker's data source). */ +export const getWorkflowFlow = (name: string, version?: string) => + fetchJSON( + `/workflows/${name}/flow${version ? `?version=${encodeURIComponent(version)}` : ""}`, + ); export interface CreateWorkflowResponse { ok: true; diff --git a/vein/web/src/app.tsx b/vein/web/src/app.tsx index 95617b5d3..de5cff7f2 100644 --- a/vein/web/src/app.tsx +++ b/vein/web/src/app.tsx @@ -118,6 +118,19 @@ export function App() { const activeVersion = workflows.find((w) => w.name === selectedWf)?.activeVersion; const selectedEntry = workflows.find((w) => w.name === selectedWf); + // Version picker: which version of the selected workflow the canvas shows. + // Stored with the workflow it was pinned for, so the pin self-invalidates + // when the selection changes (no effect-ordering games). null = active. + const [versionPin, setVersionPin] = useState<{ wf: string | null; v: string | null }>({ wf: null, v: null }); + const viewVersion = versionPin.wf === selectedWf ? versionPin.v : null; + const setViewVersion = useCallback( + (v: string | null) => setVersionPin({ wf: selectedWf, v }), + [selectedWf], + ); + // Viewing history is read-only: editing/publishing/running only make sense + // against the active version (Publish always builds on active, Run runs it). + const viewingOld = viewVersion != null && activeVersion != null && viewVersion !== activeVersion; + // ── Sidebar grouping ───────────────────────────────────────────────────── // Workflows grouped by category; groups (and workflows within them) are // ordered by most-recent run so the active experiment floats to the top. @@ -272,7 +285,7 @@ export function App() { setFlyoutStepIndex(null); return; } - api.getWorkflowFlow(selectedWf).then((flow) => { + api.getWorkflowFlow(selectedWf, viewVersion ?? undefined).then((flow) => { const steps = flow.steps as StepData[]; setPublishedSteps(steps); setLocalSteps(steps); @@ -286,7 +299,7 @@ export function App() { setLoadError(true); }); refreshRuns(selectedWf); - }, [selectedWf]); + }, [selectedWf, viewVersion]); // Load run events when a run is selected; clear overlay + drill when deselected useEffect(() => { @@ -388,6 +401,7 @@ export function App() { }, []); function updateLocalSteps(steps: StepData[]) { + if (viewingOld) return; // historical versions are read-only setLocalSteps(steps); } @@ -680,7 +694,7 @@ export function App() { {filteredRuns.length === 0 &&
{selectedWf ? "No runs for this workflow" : "No runs yet"}
} {filteredRuns.map((run) => (
{ setSelectedRun(run.runId); closeFlyout(); }}> + onClick={() => { setSelectedRun(run.runId); setViewVersion(null); closeFlyout(); }}>
{run.runId.slice(0, 10)} @@ -734,11 +748,31 @@ export function App() { onSaved={refreshWorkflows} /> )} + {/* Version picker — browse the workflow's published lineage. Only + outside run view (a run overlays the active structure), and only + when there is history to browse. */} + {selectedWf && !selectedRun && selectedEntry && selectedEntry.versions.length > 1 && ( + + )} + {viewingOld && history · read-only} {selectedRun && {selectedRun.slice(0, 10)}} {isDirty && }
- {isDirty && } + {isDirty && !viewingOld && } {isRunView && promotions.length > 0 && ( )} - {selectedWf && ( + {selectedWf && !viewingOld && (
{runBindings && selectedWf && ( @@ -799,11 +833,11 @@ export function App() { )} {canvas ? ( - selectedWf && !isRunView + selectedWf && !isRunView && !viewingOld ? : null )} diff --git a/vein/web/src/styles/components.css b/vein/web/src/styles/components.css index 137d7e0b6..b5c69f3f0 100644 --- a/vein/web/src/styles/components.css +++ b/vein/web/src/styles/components.css @@ -89,6 +89,27 @@ cursor: pointer; } +/* Topbar version picker — browse a workflow's published lineage. */ +.version-select { + margin-left: 10px; + background: var(--surface-2); + color: var(--text); + border: 1px solid var(--border-strong); + border-radius: var(--r-sm); + font-size: 11px; + font-family: var(--mono, monospace); + padding: 2px 4px; + cursor: pointer; + vertical-align: middle; +} +.version-history-badge { + margin-left: 8px; + color: var(--warning, #d9a441); + border: 1px solid currentColor; + background: transparent; + vertical-align: middle; +} + .shell-events { grid-area: events; background: var(--bg); From 074fbcb17c3aa91592f96e7bdf5ece3e73b5a8e7 Mon Sep 17 00:00:00 2001 From: Evanfeenstra Date: Fri, 28 Aug 2026 09:47:40 -0700 Subject: [PATCH 4/4] fix stale run in ui --- vein/RUN_CONTROL_SPEC.md | 350 +++++++++++++++++++++++++++++++++++++++ vein/src/authoring.ts | 25 ++- vein/src/createVein.ts | 46 +++-- 3 files changed, 400 insertions(+), 21 deletions(-) create mode 100644 vein/RUN_CONTROL_SPEC.md diff --git a/vein/RUN_CONTROL_SPEC.md b/vein/RUN_CONTROL_SPEC.md new file mode 100644 index 000000000..bcf092aa4 --- /dev/null +++ b/vein/RUN_CONTROL_SPEC.md @@ -0,0 +1,350 @@ +# Run Control — Spec + +Cancel, pause/resume, and durable resume for runs — including the nested +run TREES the lab produces (an evolve run that launches generation runs +that launch candidate runs), where today the only control is killing the +server and orphaning every in-flight log as "stale". + +Status: DRAFT. Nothing in this document is implemented. + +--- + +## 1. Problem + +A long optimization run (harvey-evolve: hours, tens of dollars, ten+ +nested runs) currently offers zero control after launch: + +- **No cancel.** A run discovered to be misconfigured at generation 1 + burns the remaining budget anyway. +- **No pause.** Server maintenance means choosing between waiting hours + or killing everything. +- **No recovery.** A crash or restart loses ALL in-flight progress. The + event log survives (append-only JSONL, every step's output recorded) + but nothing can consume it: the run can only be started over from + scratch, re-spending everything already spent. +- **No tree awareness.** Even with a per-run kill, the lab's runs are + trees (`harvey-evolve` → `optimizer.run` → `harvey-evolve-gen` → + `meta/run-workflow` → `harvey-produce-ai`). Each nested launch is an + independent `runWorkflow` invocation sharing nothing — stopping a + parent must stop its descendants or it stops nothing. + +The runner's own architecture makes this tractable: `runWorkflow` is the +single choke point every run path goes through, every step boundary is +already instrumented (step.start/step.end events), and completed step +outputs are already durably journaled. Run control is mostly about +exposing seams the engine already has. + +--- + +## 2. Design overview + +Three features, strictly layered — each rung is independently shippable +and each reuses the previous rung's plumbing: + +1. **Cancel** — cooperatively stop a run tree at the next step boundary; + finalize honestly as `cancelled`. +2. **Pause / resume (in-memory)** — park a run tree at the next step + boundary of every active branch; release it later. Does not survive a + restart. +3. **Durable resume (journal replay)** — continue an interrupted run + after a crash/restart by replaying completed steps' outputs from the + event log and executing from the first incomplete step. + +Pause (2) + durable resume (3) compose into the real operational win: +pause, wait for quiescence, restart the server, resume — a long run +survives a deploy. + +### 2.1 One principle: cooperative, at boundaries + +The leaves of a run tree cannot be frozen: an LLM call mid-stream, a +`uv`-spawned grader subprocess, an HTTP request. All control is therefore +**cooperative**: the current unit of work completes (and is paid for), +and the run checks for a control signal before starting the next unit. + +Checkpoints, from coarse to fine: + +| boundary | where | rung | +|-----------------------------|--------------------------------------------|------| +| between DAG steps | `executeFlow.runStep`, before `executeStep`| 1, 2 | +| between loop/foreach iters | `executeLoop` / `executeForeach` | 1, 2 | +| between retry attempts | `executeStep` retry sleep | 1, 2 | +| between agent tool calls | agent step's tool loop | 2 | +| between code-step iters | opt-in via `ctx.control` (e.g. evolve-loop)| 1, 2 | + +A run tree paused at these boundaries quiesces within one step / one +tool call per active branch — bounded, predictable, and the money spent +on the in-flight unit is not wasted (its output lands in the journal). + +### 2.2 One mechanism: the RunController + +```ts +type ControlState = "running" | "pausing" | "paused" | "cancelling"; + +interface RunController { + readonly runId: string; + readonly workflow: string; + readonly parent?: RunController; // tree linkage + readonly children: Set; + state: ControlState; // own state; effective state + // inherits the strictest ancestor + /** The cooperative checkpoint. Resolves immediately when running; + * blocks while (effectively) paused; throws CancelledError when + * (effectively) cancelling. Every boundary in §2.1 awaits this. */ + checkpoint(): Promise; + /** True when this run AND all descendants are parked at a boundary + * (drives "safe to restart now"). */ + quiesced(): boolean; + cancel(): void; // idempotent; propagates to children + pause(): void; // " + resume(): void; // " +} +``` + +- **Registry.** `createVein` holds `controllers: Map` + (superseding today's `activeRuns: Set` — a controller's presence + IS "in-flight", so the `"running" | "stale"` listing fallback reads this + map). Registered/unregistered exactly where `trackRun` is called today: + `launchDetached`, `vein.run`, authoring's `runWorkflow`. +- **Tree linkage.** Nested launches attach to the launching run's + controller. The parent runId travels the same paths the services bag + already does: `RunOptions.parentRunId`, set by `meta/run-workflow` and + the optimizer capability from their calling step's `ctx.runId`. + Subflows/foreach/loop need nothing — they share the parent's runId and + therefore its controller. +- **Effective state.** `checkpoint()` walks ancestors: any ancestor + cancelling → throw; any ancestor pausing/paused → block. Controls + therefore apply to WHOLE SUBTREES: cancelling the evolve run cancels + its generations and candidates; pausing a single candidate pauses only + that candidate. +- **Threading.** The controller rides `RunOptions` into `runWorkflow` and + is exposed to steps as `ctx.control` (optional, like `ctx.registry` — + absent outside the runner, e.g. unit tests). Code steps with long + internal loops (evolve-loop's generations) call + `await ctx.control?.checkpoint()` per iteration; steps that ignore it + simply remain coarse-grained (control applies at their step boundary). + +--- + +## 3. Rung 1 — Cancel + +**Semantics.** `controller.cancel()` flips the subtree to `cancelling`. +Every branch throws `CancelledError` at its next boundary. `runWorkflow` +catches it as a distinct outcome (NOT the generic error path): + +- emit `run.cancelled` (terminal event; `store.tailRun`'s `isTerminal` + gains this type so SSE tails end), +- `store.finalize` with `status: "cancelled"` — run.json exists, the run + is never "stale", partial outputs stay inspectable in the log, +- `onRunEnd` teardown fires as on any other exit (browsers/stacks are + disposed). + +**Ordering.** Children are cancelled by the same effective-state walk — +no separate fan-out message. A parent blocked awaiting a child's result +(optimizer.run, meta/run-workflow) sees the child return +`{ status: "cancelled" }` and then hits its own checkpoint. Steps that +launch children treat a cancelled child like a failed one (the existing +error paths), except the enclosing run is itself cancelling and will not +proceed. + +**In-flight work.** v1 does not kill the current unit: the LLM call or +grader subprocess completes, its output is journaled, THEN the branch +stops. v1.5 (optional): thread an `AbortSignal` (aborted on cancel only, +never pause) into the agent step's AI-SDK calls and `services.http` for +faster, cheaper teardown of the most expensive leaves. Subprocess +graders keep run-to-completion semantics. + +**API/UI.** `POST /workflows/:name/runs/:runId/cancel` (404 unknown, 409 +already terminal). UI: a Cancel button in the topbar of an active run's +view; confirm dialog states the subtree consequence ("cancels N nested +runs"). + +**Status vocabulary.** `RunSummary.status` gains `"cancelled"`. The runs +listing keeps synthesizing `"running" | "stale"` for summary-less dirs +(from the controllers map), and adds `"paused"` (§4). + +--- + +## 4. Rung 2 — Pause / resume (in-memory) + +**Semantics.** `controller.pause()` → subtree state `pausing`; each +branch blocks inside `checkpoint()` at its next boundary; when all +branches are parked, `quiesced()` is true and state reads `paused`. +`resume()` releases every parked checkpoint. No events are lost, no +state is discarded — the run's promises simply stop advancing. + +**Extra boundary: the agent tool loop.** A 200-step author agent must +park between tool calls, not after its whole session. The agent step +checks `ctx.control` between AI-SDK steps (via its per-step hook, e.g. +`prepareStep`/`onStepFinish`) — the in-flight LLM call finishes, the next +one doesn't start. This is the single highest-value checkpoint in the +lab's workloads. + +**Observability.** Emit `run.paused` / `run.resumed` (non-terminal) so +the event log records the gap — otherwise a resumed run's step durations +silently include parked time and poison any timing analysis. The runs +listing reports `paused`; SSE tails stay open (heartbeats already exist). + +**Honest limits.** In-memory only: a paused run dies with the process +(becoming exactly the orphan that §5 resurrects — pause then restart +WITHOUT rung 3 is still data loss). Wall-clock-sensitive steps (`wait`, +external polling) resume where they left off; external systems don't +pause with us. Pause does not release per-run resources (a gitsee +browser/stack stays up while parked) — pausing to save money only stops +NEW spend, chiefly LLM calls. + +--- + +## 5. Rung 3 — Durable resume (journal replay) + +The event log already contains everything a resume needs: every +completed step's `step.end` carries its `output`, keyed by a +deterministic `path` (`wf/stepId`, `wf/stepId#iter/...` for +foreach/loop bodies, synthetic `wf/stepId#gen` rows for iterative code +steps). Resume = re-run the workflow against that journal. + +**Mechanism.** + +``` +resume(workflow, runId): + events = store.getRunEvents(workflow, runId) # the journal + journal = { path → output } for every step.end # completed units + input = the run.start event's recorded input + re-invoke runWorkflow with { runId, journal } # SAME runId +``` + +Execution proceeds normally, except `executeStep` consults the journal +first: a step whose `path` has a journaled output **replays** it (emit +`step.replayed` with the output; resolve the DAG promise; zero cost, no +side effects re-executed) instead of executing. The first path NOT in +the journal executes live, and everything downstream follows. Scope, +templates, and skip/gate logic are reconstructed naturally because they +only ever consume step outputs. + +### 5.1 Hard stops are the same case + +Nothing above requires a graceful shutdown. The journal is written AS +THE RUN HAPPENS — every event is appended (and awaited) before execution +continues — so after a SIGKILL, an OOM kill, or a power cut, the disk +already holds the completed prefix plus at most one dangling +`step.start` per active branch. Resume replays the prefix and re-executes +the interrupted units. Crash recovery is therefore rung 3 verbatim; a +graceful pause-then-restart merely avoids re-spending the in-flight +step. Three crash-specific hardenings: + +- **Torn tail.** A process killed mid-append can leave a truncated final + JSONL line (step outputs are large; single-write atomicity is not + guaranteed). The journal reader must skip an unparseable trailing line + — it belongs to an incomplete unit by definition. +- **Durability policy.** `appendFile` lands in the page cache; a whole- + SYSTEM crash (not a process kill) can lose recently "written" events. + Option: fsync on `step.end` / run-level events only (the events worth + money). Acceptable default: no fsync — losing the cache tail just + rewinds resume a little further back, which is correct, merely less + thrifty. +- **At-least-once side effects.** Re-executed steps REDO their external + effects: an author agent crashed mid-session republishes (an extra + candidate version — benign, versions are append-only), a re-run + meta/run-workflow launches a fresh child run. This is the §6 + re-execution contract doing its job; steps whose effects must not + double need their own idempotency (none in the current lab do). + +Optionally, the server can offer auto-resume of summary-less runs on +boot (off by default — a human choosing Resume on a "stale" run is the +right v1 ergonomics). + +**Same runId, same artifacts.** Resume CONTINUES the original run: it +appends to the same JSONL (after a `run.resumed` marker event) and keeps +the runId — critical because artifact directories are keyed by runId +(`artifacts//...`): a drafted memo written before the crash is +still on disk at the path the journaled outputs reference. A fresh-runId +"resume" would silently dangle every recorded `outputDir`. + +**Nested runs.** A parent step that launched a child run +(meta/run-workflow, optimizer.run) either has a journaled output — child +completed, replayed, child untouched — or doesn't, in which case the +step re-executes and launches a FRESH child run (the interrupted child's +partial log remains on disk as history; v1 does not recursively resume +children — see Non-goals for why not yet). + +**Iterative code steps.** A step like `harvey/evolve-loop` is one DAG +step wrapping N generations; all-or-nothing replay would forfeit +completed generations. Such steps already emit per-iteration synthetic +`step.end` events (`…/evolve#3`) carrying full outputs. Contract: the +runner hands the step its own journal slice (`ctx.journal`: path → +output, scoped under the step's path); the step reconstructs completed +iterations from it and continues from the first missing one. Steps that +ignore `ctx.journal` just replay coarse (whole-step or re-run) — +correct, merely less thrifty. + +**Agent steps.** v1: an agent step with no journaled output re-runs its +whole session (bounded loss: one step). Future: journal the message +transcript per tool call and resume mid-session — deliberately out of +v1, it drags in provider-state questions the rungs below don't need. + +**Validity guards.** Resume refuses when: a live controller exists for +the runId (it's not dead, pause it instead); the workflow's current +content hash differs from the one recorded at `run.start` (the runner +must record it there — replaying outputs into a DIFFERENT DAG is +undefined; power users may override with an explicit flag); or the run +already has a terminal summary. Registry drift (a custom step edited +between crash and resume) is allowed but WARNED — steps re-executing +post-resume use current code, same as any new run. + +**API/UI.** `POST /workflows/:name/runs/:runId/resume` (only valid on +summary-less, controller-less runs — exactly today's "stale"). UI: the +"stale" badge becomes a Resume affordance. This retroactively gives +"stale" a purpose: it is the set of resumable runs. + +--- + +## 6. Step-author contract (additions) + +- `ctx.control?: RunController` — await `checkpoint()` inside long + internal loops; ignore it and your step remains a coarse unit. +- `ctx.journal?: Record` — on resume, your own prior + synthetic `step.end` outputs; consume to skip completed iterations. +- Emit per-iteration synthetic `step.end` events with FULL outputs if + you want thrifty resume (evolve-loop already complies). +- Side-effectful steps must remain safe to re-execute after a crash + mid-step (the journal only protects COMPLETED units). This is already + the implicit contract — retries exist — now it's explicit. + +## 7. Invariants + +- **Measurement discipline untouched.** Control flows through the + runner, not the workflow surface: authors/candidates cannot pause, + cancel, or resume anything (no meta/* exposure; the grader and the + §6-EVOLVE_SPEC firewalls are unaffected). +- **Terminal statuses are honest.** `cancelled` is never conflated with + `error`; replayed steps are `step.replayed`, never fake `step.end` + timings; paused gaps are visible in the log. +- **`runWorkflow` stays the single choke point.** All three rungs live + in the runner + the launch sites; no per-step-type special cases + beyond the opt-in contract in §6. + +## 8. Non-goals (v1) + +- Preemptive freezing of in-flight LLM calls/subprocesses, or process + snapshotting (CRIU-style). +- Recursive resume of interrupted CHILD runs (parent re-launches fresh; + a child's completed work is recoverable in a later iteration by + making optimizer.run/meta-run-workflow journal-aware). +- Mid-session agent resume (transcript journaling). +- Cross-machine migration of paused runs. +- Scheduling/quotas ("pause when spend exceeds $X") — trivially layered + on `pause()` later, out of scope here. + +## 9. Rollout + +1. RunController + registry + tree linkage; cancel; `cancelled` status; + UI button. (Subsumes today's `activeRuns`/`trackRun`.) +2. Pause/resume + agent tool-loop checkpoint + paused/quiesced surfacing. +3. Content-hash recording at `run.start` (ship early — resume needs + history to exist); journal replay; `ctx.journal`; evolve-loop + iteration resume; UI resume-from-stale. + +Each rung lands with runner tests (cancel/pause mid-DAG, mid-foreach, +mid-retry; resume replay correctness incl. skip/gate reconstruction) and +one lab-level integration: cancel a nested optimizer run; pause/resume +an agent step between tool calls; resume a killed two-generation evolve +run and verify generation 0 replays for free. diff --git a/vein/src/authoring.ts b/vein/src/authoring.ts index d4eb6f7a5..c1a45478d 100644 --- a/vein/src/authoring.ts +++ b/vein/src/authoring.ts @@ -403,6 +403,11 @@ export interface AuthoringDeps extends StepPublishDeps { /** Read-only view of the deployment's secret store (NAMES only — never * values). Optional: `listSecrets` degrades gracefully when absent. */ secrets?: { list(): Promise }; + /** Register a nested run as in-flight (returns the untrack fn) so the + * server's runs listing reports it "running" rather than "stale" while + * its run.json doesn't exist yet. Optional: embedders without a live + * server need not care. */ + trackRun?: (workflow: string, runId: string) => () => void; } export function buildAuthoringCapability(deps: AuthoringDeps): AuthoringCapability { @@ -571,13 +576,19 @@ export function buildAuthoringCapability(deps: AuthoringDeps): AuthoringCapabili // FRESH registry, same reason as runStep: steps published mid-run are // invisible to the enclosing run's registry snapshot. const registry = await deps.getRegistry(); - return runWorkflow(flow, coerceJsonArg(input) ?? {}, registry, { - runId: generateRunId(), - store, - workspace, - services: deps.services, - params: coerceJsonArg(params) as Record | undefined, - }); + const runId = generateRunId(); + const untrack = deps.trackRun?.(flow.name, runId); + try { + return await runWorkflow(flow, coerceJsonArg(input) ?? {}, registry, { + runId, + store, + workspace, + services: deps.services, + params: coerceJsonArg(params) as Record | undefined, + }); + } finally { + untrack?.(); + } }, async listRuns(name, limit = 20) { diff --git a/vein/src/createVein.ts b/vein/src/createVein.ts index b70bd8e3c..1b810ed25 100644 --- a/vein/src/createVein.ts +++ b/vein/src/createVein.ts @@ -290,6 +290,17 @@ export async function createVein( // restart / crash / cancellation) and is reported as "stale" rather than a // perpetual "running". const activeRuns = new Set(); + /** Register a run as in-flight for the runs-listing status fallback (a run + * dir with events but no run.json yet is "running" if registered here, + * "stale" — i.e. orphaned by a crash/restart — otherwise). Every launch + * path must register: HTTP (launchDetached), programmatic (vein.run), and + * in-process nested runs (authoring's meta/run-workflow). Returns the + * untrack fn for the caller's finally. */ + const trackRun = (workflow: string, runId: string) => { + const key = `${workflow}/${runId}`; + activeRuns.add(key); + return () => activeRuns.delete(key); + }; // Deployment-scoped secret store backing the `secrets` capability + the // `/secrets` admin endpoints. Mirrors the run/chat store defaults: encrypted // file store for the standard server, in-memory when runs are in-memory. @@ -372,6 +383,7 @@ export async function createVein( workspace, store, services, + trackRun, publishingEnabled: !registryWasInjected, getRegistry: async () => { await rebuildRegistry(); @@ -958,8 +970,7 @@ export async function createVein( */ function launchDetached(flow: Flow, body: RunBody): string { const runId = body.runId ?? generateRunId(); - const key = `${flow.name}/${runId}`; - activeRuns.add(key); + const untrack = trackRun(flow.name, runId); void runWorkflow(flow, body.input ?? {}, registry, { runId, store, @@ -971,9 +982,7 @@ export async function createVein( .catch((err) => { console.error(`[run ${runId}] launch failed:`, err); }) - .finally(() => { - activeRuns.delete(key); - }); + .finally(untrack); return runId; } @@ -1338,15 +1347,24 @@ export async function createVein( : await workspace.getWorkflow(workflow) : workflow; - return runWorkflow(flow, input, registry, { - runId: runOpts?.runId, - store, - workspace, - services: runOpts?.services ?? services, - params: runOpts?.params, - paramOverrides: runOpts?.paramOverrides, - onEvent: runOpts?.onEvent, - }); + // Generate the runId here (rather than letting runWorkflow default it) so + // the run can be registered as in-flight — otherwise nested runs (e.g. the + // optimizer capability's generation runs) list as "stale" while running. + const runId = runOpts?.runId ?? generateRunId(); + const untrack = trackRun(flow.name, runId); + try { + return await runWorkflow(flow, input, registry, { + runId, + store, + workspace, + services: runOpts?.services ?? services, + params: runOpts?.params, + paramOverrides: runOpts?.paramOverrides, + onEvent: runOpts?.onEvent, + }); + } finally { + untrack(); + } } // ── Listener ─────────────────────────────────────────────────────────────