From fd5d599019133fbf83c099214b0fe92bf18a2689 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Fri, 2 Oct 2026 00:01:31 -0700 Subject: [PATCH 1/4] test(flows): Garden runs must fit their own time budget (cloud#4108) Replays run bda21b91's measured step times against the kernel's budget rule: on the 2h header the run is refused before the step after its passing checks, as in production (120.2m used before "run-15"). Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 3b31cc74-0c43-4890-9650-f2be3566b7ff --- web/lib/test/flow-budget.test.ts | 181 +++++++++++++++++++++++++++ web/lib/test/flow-local.test.ts | 2 +- web/lib/test/flow-onboarding.test.ts | 2 +- 3 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 web/lib/test/flow-budget.test.ts diff --git a/web/lib/test/flow-budget.test.ts b/web/lib/test/flow-budget.test.ts new file mode 100644 index 00000000..d260bc2d --- /dev/null +++ b/web/lib/test/flow-budget.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from 'vitest'; +import ts from 'typescript'; +import { + FLOW_BASE_CHECK_COMMAND, + FLOW_CHECK_RUN_COMMAND, + FLOW_OPEN_CHANGE_COMMAND, + FLOW_PUBLISH_CHECK_COMMAND, + FLOW_TIME, + FLOW_TIME_STOP_COMMAND, + FLOW_VALIDATE_CHANGE_METADATA_COMMAND, +} from '../flow-workflows'; +import { factorySource, type FactoryDraft } from '../flow-onboarding'; + +// AgentWorkforce/cloud#4108: Garden runs died on their own 2h budget before +// they could publish (bda21b91: 120.2m used before step "run-15", with the +// checks passing at step 14), because one check run takes 11-14 min on that +// repository and two repair passes took 57 min. + +const issue = { source: 'github', title: 'Fix login', body: 'Login fails', labels: ['ready'], repository: 'acme/app', identifier: '#507', url: 'https://github.com/acme/app/issues/507' }; +const draft: FactoryDraft = { version: 4, sources: ['github'], sourceSettings: { github: { repository: 'acme/app', labels: 'ready' } }, agents: ['claude', 'codex'], otherAgent: '', task: 'Add a test', workflow: 'traditional', step: 3 }; + +const MINUTE = 60_000; +const UNITS: Record = { ms: 1, s: 1000, m: MINUTE, h: 60 * MINUTE }; + +/** The wallclock the generated header declares, in ms. */ +function declaredWallclockMs(source: string): number { + const match = /budget: \{ wallclock: "(\d+)(ms|s|m|h)" \}/.exec(source); + if (!match) throw new Error('no wallclock budget in the generated header'); + return Number(match[1]) * UNITS[match[2]!]!; +} + +type Timing = { + /** Minutes each named agent step takes (prefix match: "check-repair-1" matches "check-repair"). */ + agents: Record; + /** Minutes and verdict of each check run, in order; later runs repeat the last. */ + checks: { minutes: number; verdict: string }[]; + baseline?: { minutes: number; verdict: string }; +}; + +/** + * Runs the generated flow against a simulated clock and the kernel's budget + * rule (flows `AuthoredBudget`): every step's wallclock is charged, and a step + * is refused once the charged total exceeds the header's wallclock. Every + * other command takes 5 seconds. + */ +async function runTimed(timing: Timing, workflow: FactoryDraft['workflow'] = 'traditional') { + const source = factorySource({ ...draft, workflow }); + const budgetMs = declaredWallclockMs(source); + const compiled = ts.transpileModule(source.replace('import { flow } from "@relayflows/surface";', ''), { + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.CommonJS }, + }); + const exports: { default?: (ctx: unknown, input: unknown) => Promise } = {}; + new Function('exports', 'flow', compiled.outputText)(exports, (_n: string, _o: unknown, body: unknown) => body); + const epoch = 1_759_000_000; + let charged = 0; + let checkIndex = 0; + const calls: { name: string; atMinute: number }[] = []; + const errors: string[] = []; + let finish = ''; + let refused: string | null = null; + const step = (name: string, ms: number) => { + if (charged > budgetMs) { + refused = name; + throw new Error(`Flow budget exceeded before step "${name}"`); + } + calls.push({ name, atMinute: charged / MINUTE }); + charged += ms; + }; + const agentMinutes = (name: string) => { + const key = Object.keys(timing.agents).find(prefix => name.startsWith(prefix)); + return key === undefined ? 5 : timing.agents[key]!; + }; + const originalError = console.error; + console.error = (message: string) => { errors.push(String(message)); }; + try { + await exports.default!({ + agent: async (name: string) => { step(name, agentMinutes(name) * MINUTE); }, + run: async (command: string) => { + if (command === FLOW_CHECK_RUN_COMMAND) { + const check = timing.checks[Math.min(checkIndex++, timing.checks.length - 1)]!; + step('check', check.minutes * MINUTE); + return check.verdict; + } + if (command.endsWith(FLOW_BASE_CHECK_COMMAND)) { + const base = timing.baseline ?? { minutes: 15, verdict: 'fail' }; + step('base-check', base.minutes * MINUTE); + return base.verdict; + } + step(command.startsWith(FLOW_OPEN_CHANGE_COMMAND) ? 'open-change' : command === FLOW_TIME_STOP_COMMAND ? 'time-stop' : 'run', 5000); + // The flow reads the clock through a journaled step: the simulated + // wall clock is the charged step time. + if (command === 'date +%s') return String(epoch + Math.floor(charged / 1000)); + if (command.endsWith(FLOW_PUBLISH_CHECK_COMMAND)) return 'publish'; + if (command.endsWith(FLOW_VALIDATE_CHANGE_METADATA_COMMAND)) return 'valid'; + if (command === 'git rev-parse HEAD') return 'abc123'; + if (command.startsWith('test -f review.clean')) return 'no'; + if (command.startsWith('test -s')) return 'yes'; + return ''; + }, + human: async () => { throw new Error('unsupported'); }, + done: (reason: string) => { finish = reason; }, + }, { issue, approver: 'owner' }); + } catch (error) { + if (refused === null) throw error; + } finally { + console.error = originalError; + } + const opened = calls.find(call => call.name === 'open-change'); + return { calls, errors, finish, refused, budgetMs, chargedMinutes: charged / MINUTE, opened }; +} + +describe('Garden flow time budget (cloud#4108)', () => { + it('states the worst-case arithmetic the generated flow is built on', () => { + const t = FLOW_TIME; + // The header's wallclock is the Cloud run-budget maximum (180m), less the + // setup Cloud spends before the flow body starts. + expect(t.headerMinutes).toBe(180); + expect(t.bodyMinutes).toBe(t.headerMinutes - t.setupMinutes); + // A repair starts only when it, the re-check, the base-commit check that a + // still-failing check triggers, and publishing all fit. + expect(t.repairStartMinutes).toBe(t.repairMinutes + t.checkMinutes + t.checkMinutes + t.publishMinutes); + // A review round starts only when the review and a time-stop publish fit. + expect(t.reviewStartMinutes).toBe(t.reviewMinutes + t.publishMinutes); + // A fix round is the fixer, its check, the next review, and publishing. + expect(t.fixRoundStartMinutes).toBe(t.fixerMinutes + t.checkMinutes + t.reviewMinutes + t.publishMinutes); + // The allowances cover the longest steps measured on 2026-10-01: a check + // run of 14m00s (15m lease), repair agents of 37m18s and 44m09s. + expect(t.checkMinutes).toBeGreaterThanOrEqual(15); + expect(t.repairMinutes).toBeGreaterThanOrEqual(45); + }); + + it('declares the 3h wallclock in the header, from the same constant', () => { + expect(factorySource(draft)).toContain('{ budget: { wallclock: "3h" } }'); + }); + + it('publishes run bda21b91 (measured step times) instead of dying on the budget', async () => { + // bda21b91: planning/discovery agents 4m47s / 9m24s / 5m57s / 6m22s, + // checks 11m30s fail, 11m21s fail, 14m00s pass, repairs 19m27s and 37m18s. + const run = await runTimed({ + agents: { 'planner': 4.8, 'plan-reviewer': 9.4, 'check-discovery': 6, 'implementer': 6.4, 'check-repair-1': 19.5, 'check-repair-2': 37.3, 'adversary': 8, 'fixer': 15 }, + checks: [{ minutes: 11.5, verdict: 'fail' }, { minutes: 11.4, verdict: 'fail' }, { minutes: 14, verdict: 'pass' }], + }); + expect(run.refused).toBeNull(); + expect(run.opened).toBeDefined(); + expect(run.chargedMinutes).toBeLessThanOrEqual(run.budgetMs / MINUTE); + }); + + it('still publishes when every check times out and every repair and review takes its full allowance', async () => { + const run = await runTimed({ + agents: { 'planner': 10, 'plan-reviewer': 10, 'check-discovery': 10, 'implementer': 30, 'check-repair': FLOW_TIME.repairMinutes, 'adversary': FLOW_TIME.reviewMinutes, 'fixer': FLOW_TIME.fixerMinutes }, + checks: [{ minutes: FLOW_TIME.checkMinutes, verdict: 'timeout' }], + baseline: { minutes: FLOW_TIME.checkMinutes, verdict: 'fail' }, + }); + expect(run.refused).toBeNull(); + expect(run.opened).toBeDefined(); + expect(run.chargedMinutes).toBeLessThanOrEqual(FLOW_TIME.bodyMinutes); + // Out of time, the work is kept as a draft and the run asks for a person. + expect(run.calls.map(call => call.name)).toContain('time-stop'); + expect(run.finish).toBe('needs_human'); + expect(run.errors.join('\n')).toMatch(/out of time/i); + }); + + it('skips a repair it cannot afford and publishes the draft with time to spare', async () => { + const run = await runTimed({ + agents: { 'implementer': 120, 'check-repair': FLOW_TIME.repairMinutes }, + checks: [{ minutes: FLOW_TIME.checkMinutes, verdict: 'fail' }], + baseline: { minutes: FLOW_TIME.checkMinutes, verdict: 'fail' }, + }, 'simple'); + expect(run.refused).toBeNull(); + expect(run.calls.filter(call => call.name.startsWith('check-repair'))).toHaveLength(0); + expect(run.opened).toBeDefined(); + expect(run.errors.join('\n')).toMatch(/skipped the repair/i); + }); + + it('changes nothing for a run with time to spare', async () => { + const run = await runTimed({ agents: {}, checks: [{ minutes: 5, verdict: 'fail' }, { minutes: 5, verdict: 'pass' }] }); + expect(run.calls.filter(call => call.name.startsWith('check-repair'))).toHaveLength(1); + expect(run.calls.map(call => call.name)).not.toContain('time-stop'); + expect(run.calls.filter(call => call.name.startsWith('adversary'))).toHaveLength(2); + }); +}); diff --git a/web/lib/test/flow-local.test.ts b/web/lib/test/flow-local.test.ts index 7dad3505..ea989cf7 100644 --- a/web/lib/test/flow-local.test.ts +++ b/web/lib/test/flow-local.test.ts @@ -360,7 +360,7 @@ describe('local flow starter kit', () => { const source = factorySource({ ...draft, workflow }, 'local'); const file = ts.createSourceFile('local.flow.ts', source, ts.ScriptTarget.ES2022, true); expect((file as unknown as { parseDiagnostics: unknown[] }).parseDiagnostics).toEqual([]); - expect(source).toContain('wallclock: "2h"'); + expect(source).toContain('wallclock: "3h"'); expect(source).not.toContain('$8/run'); expect(source).not.toContain('pr merge'); } diff --git a/web/lib/test/flow-onboarding.test.ts b/web/lib/test/flow-onboarding.test.ts index 2b3b8172..60c3e61f 100644 --- a/web/lib/test/flow-onboarding.test.ts +++ b/web/lib/test/flow-onboarding.test.ts @@ -245,7 +245,7 @@ describe('software factory onboarding', () => { it('gives Cloud flows a wall-clock budget so unpriced agents are never refused', () => { for (const agents of [['claude', 'codex'], ['codex'], ['claude']] as FactoryDraft['agents'][]) { const source = factorySource({ ...completed, agents }); - expect(source).toContain('{ budget: { wallclock: "2h" } }'); + expect(source).toContain('{ budget: { wallclock: "3h" } }'); expect(source).not.toMatch(/budget: "\$\d/); } }); From 913c9d6f9af90befc8743fdb9f041cd2d24c97ec Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Fri, 2 Oct 2026 00:01:31 -0700 Subject: [PATCH 2/4] fix(flows): give Garden a 3h budget and start only steps that still fit (cloud#4108) - Header wallclock 2h -> 3h (Cloud's maximum run budget), from FLOW_TIME. - The flow reads the clock through a journaled step and starts a repair, a fix round or a review only when that step's allowance and publishing still fit. Agent steps take no time limit of their own, so this is the only cap a flow body can apply. - Out of time after the pull request is open: push stays, the PR goes to draft with a note, and the run asks for a person. - A repair it cannot afford is skipped; the existing path then opens the draft with the check report. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 3b31cc74-0c43-4890-9650-f2be3566b7ff --- web/lib/flow-onboarding.ts | 8 +++-- web/lib/flow-workflows.ts | 69 +++++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/web/lib/flow-onboarding.ts b/web/lib/flow-onboarding.ts index c4e6f8dc..f82aee38 100644 --- a/web/lib/flow-onboarding.ts +++ b/web/lib/flow-onboarding.ts @@ -1,6 +1,6 @@ import { validFlowAgentSettings, type FlowAgentSettings } from './flow-agent-settings'; import { flowPreview } from './flow-preview'; -import { WORKFLOWS, workflowCode, workflowAgents, type WorkflowId } from './flow-workflows'; +import { FLOW_TIME, WORKFLOWS, workflowCode, workflowAgents, type WorkflowId } from './flow-workflows'; import { ISSUE_SOURCES, issueSourceCode, validSourcePreferences, type IssueSourceId, type SourcePreferences } from './flow-sources'; import { CODING_AGENTS, isCodingAgent, type AgentId, type CodingAgent } from './flow-agents'; @@ -107,8 +107,10 @@ export function factoryCodeSections(draft: FactoryDraft, target: 'cloud' | 'loca // budget no longer refuses a model-less Codex step (AgentWorkforce/flows#421); // such a step runs unmetered, so a dollar cap cannot bound it. Wall-clock is // enforced on every step regardless of pricing, which is why it stays the - // default here; `{ dollars, wallclock }` together is also valid. - const budget = '{ wallclock: "2h" }'; + // default here; `{ dollars, wallclock }` together is also valid. Its length + // is the flow's time plan (FLOW_TIME, AgentWorkforce/cloud#4108): 2h could + // not hold a run whose checks needed repair on a ~12 minute test suite. + const budget = `{ wallclock: "${FLOW_TIME.headerMinutes / 60}h" }`; if (!draft.sources.length) return [{ id: 'empty', code: `import { flow } from "@relayflows/surface"; export default flow("software-factory", diff --git a/web/lib/flow-workflows.ts b/web/lib/flow-workflows.ts index aa6562f4..01991965 100644 --- a/web/lib/flow-workflows.ts +++ b/web/lib/flow-workflows.ts @@ -254,6 +254,49 @@ export const FLOW_CHECK_BLOCKED_COMMAND = [ 'if gh pr comment --body-file .relayflow/check-report.md >/dev/null 2>&1; then echo "relayflow: posted the check report to the pull request."; else echo "relayflow: could not comment on the pull request; .relayflow/check-report.md still holds the report." >&2; fi', ].join('; '); +/** + * The generated flow's time plan (AgentWorkforce/cloud#4108). The header's + * wallclock is charged per step and checked before each one, so once it is + * spent no step can run, not even the push that keeps the work. Agent steps + * take no time limit of their own, so the flow reads the clock (a journaled + * `date +%s`) before each optional, expensive step and starts it only when + * that step's allowance and publishing still fit. The allowances cover the + * longest steps measured on 2026-10-01: a 14m check run (its lease is 15m) + * and repair agents of 37m and 44m. + */ +export const FLOW_TIME = (() => { + const headerMinutes = 180; // Cloud's maximum run budget (RELAYFLOW_V2_RUN_BUDGET_MAX_MINUTES). + const setupMinutes = 5; // Cloud's run budget also counts the setup before the body. + const checkMinutes = 15; + const repairMinutes = 45; + const reviewMinutes = 20; + const fixerMinutes = 45; + const publishMinutes = 10; + return Object.freeze({ + headerMinutes, setupMinutes, checkMinutes, repairMinutes, reviewMinutes, fixerMinutes, publishMinutes, + bodyMinutes: headerMinutes - setupMinutes, + // A repair, its re-check, the base-commit check a still-failing check + // triggers, and publishing. + repairStartMinutes: repairMinutes + checkMinutes + checkMinutes + publishMinutes, + // A review, then publishing a time stop. + reviewStartMinutes: reviewMinutes + publishMinutes, + // The fixer, its check, the next review, and publishing. + fixRoundStartMinutes: fixerMinutes + checkMinutes + reviewMinutes + publishMinutes, + }); +})(); + +/** + * Marks the pull request as not ready when the run stops for time before its + * reviews are done, and says so on it. Every branch exits 0, like + * FLOW_CHECK_BLOCKED_COMMAND. + */ +export const FLOW_TIME_STOP_COMMAND = [ + 'mkdir -p .relayflow', + `printf '%s\\n' "**This run ran out of time before its reviews were done** (the flow's ${FLOW_TIME.headerMinutes / 60}h budget). The work is pushed and this pull request is a draft so it is not lost; review it by hand or run the flow again." > .relayflow/time-stop.md`, + 'if gh pr ready --undo >/dev/null 2>&1; then echo "relayflow: converted the pull request to a draft."; else echo "relayflow: could not convert the pull request to a draft." >&2; fi', + 'if gh pr comment --body-file .relayflow/time-stop.md >/dev/null 2>&1; then echo "relayflow: posted the time stop to the pull request."; else echo "relayflow: could not comment on the pull request." >&2; fi', +].join('; '); + /** * Files the flow and its agents write for each other, which are never part of * the change. Paths are relative to the repository root. @@ -627,7 +670,14 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType Number((await f.run("date +%s")).trim()); + const startedAt = await clock(); + const minutesLeft = async () => ${FLOW_TIME.bodyMinutes} - (await clock() - startedAt) / 60;` }]; if (workflow === 'traditional') sections.push({ id: 'plan', code: ` // Read the ticket and agree on a plan before changing code. await f.agent("planner", { ${options('planner', 'builder')} @@ -703,6 +753,11 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType { let result = verdict(await f.run(runChecks, { timeout: "15m" })); for (let attempt = 0; attempt < 2 && broken(result); attempt++) { + // A repair, its re-check and publishing must fit in what is left. + if (await minutesLeft() < ${FLOW_TIME.repairStartMinutes}) { + console.error("Skipped the repair: too little of the flow's time budget is left to repair and still publish. The checks stand as they are."); + break; + } await f.agent("check-repair-" + (++repairs), { ${options('check-repair', 'builder')} }); @@ -787,12 +842,24 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType Date: Fri, 2 Oct 2026 00:36:21 -0700 Subject: [PATCH 3/4] fix(flows): count parallel prototypes, gate the base check, keep review findings on a time stop Review on #132 (cubic, Cursor Bugbot): - the budget charges each parallel prototype in full, so the clock adds the extra two charges - the base-commit check is skipped (reported as not checked) when it and publishing no longer fit - no time for a fix round now stops as an unresolved review, so the first review's findings go on the pull request - the test harness gives the body the budget less Cloud's setup, runs check-discovery, and models parallel agents on separate clocks Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 3b31cc74-0c43-4890-9650-f2be3566b7ff --- web/lib/flow-workflows.ts | 21 +++++++--- web/lib/test/flow-budget.test.ts | 71 ++++++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/web/lib/flow-workflows.ts b/web/lib/flow-workflows.ts index 01991965..9ae1bf06 100644 --- a/web/lib/flow-workflows.ts +++ b/web/lib/flow-workflows.ts @@ -677,7 +677,9 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType Number((await f.run("date +%s")).trim()); const startedAt = await clock(); - const minutesLeft = async () => ${FLOW_TIME.bodyMinutes} - (await clock() - startedAt) / 60;` }]; + // Parallel steps are charged once each, so they count more than the clock. + let parallelMinutes = 0; + const minutesLeft = async () => ${FLOW_TIME.bodyMinutes} - (await clock() - startedAt) / 60 - parallelMinutes;` }]; if (workflow === 'traditional') sections.push({ id: 'plan', code: ` // Read the ticket and agree on a plan before changing code. await f.agent("planner", { ${options('planner', 'builder')} @@ -697,12 +699,16 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType f.agent("prototype-" + (index + 1), { cli: prototypeAgents[index], cwd, model: prototypeSettings[index].model, task: task + "\\n" + prototypeSettings[index].prompt + " Assigned approach: " + approaches[index], }))); + parallelMinutes += 2 * (await clock() - prototypesStartedAt) / 60; // All three implementations are finished before comparison begins. await f.agent("comparator", { ${options('comparator', 'reviewer', ' + " Prototype worktrees: " + paths.join(", ")')} @@ -774,7 +780,11 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType Promise } = {}; new Function('exports', 'flow', compiled.outputText)(exports, (_n: string, _o: unknown, body: unknown) => body); const epoch = 1_759_000_000; + // Two clocks: `charged` is what the kernel bills (every step's own time), + // `wallMs` is what `date` reads. They differ only for parallel agents. let charged = 0; + let wallMs = 0; + let groupStart: number | null = null; let checkIndex = 0; const calls: { name: string; atMinute: number }[] = []; const errors: string[] = []; let finish = ''; let refused: string | null = null; - const step = (name: string, ms: number) => { + const step = (name: string, ms: number, parallel = false) => { + // Strictly greater, as the kernel compares (flows authored-budget.ts, + // machine/budget.rs): a step is refused once the charged time exceeds the limit. if (charged > budgetMs) { refused = name; throw new Error(`Flow budget exceeded before step "${name}"`); } calls.push({ name, atMinute: charged / MINUTE }); charged += ms; + if (parallel) { + groupStart ??= wallMs; + wallMs = Math.max(wallMs, groupStart + ms); + } else { + groupStart = null; + wallMs += ms; + } }; const agentMinutes = (name: string) => { const key = Object.keys(timing.agents).find(prefix => name.startsWith(prefix)); @@ -74,7 +89,11 @@ async function runTimed(timing: Timing, workflow: FactoryDraft['workflow'] = 'tr console.error = (message: string) => { errors.push(String(message)); }; try { await exports.default!({ - agent: async (name: string) => { step(name, agentMinutes(name) * MINUTE); }, + agent: async (name: string) => { + // Agents started together (prototypes) run at once: each is charged + // in full, but the wall clock moves once for the whole group. + step(name, agentMinutes(name) * MINUTE, options.parallel?.some(prefix => name.startsWith(prefix)) ?? false); + }, run: async (command: string) => { if (command === FLOW_CHECK_RUN_COMMAND) { const check = timing.checks[Math.min(checkIndex++, timing.checks.length - 1)]!; @@ -86,14 +105,15 @@ async function runTimed(timing: Timing, workflow: FactoryDraft['workflow'] = 'tr step('base-check', base.minutes * MINUTE); return base.verdict; } - step(command.startsWith(FLOW_OPEN_CHANGE_COMMAND) ? 'open-change' : command === FLOW_TIME_STOP_COMMAND ? 'time-stop' : 'run', 5000); - // The flow reads the clock through a journaled step: the simulated - // wall clock is the charged step time. - if (command === 'date +%s') return String(epoch + Math.floor(charged / 1000)); + step(command.startsWith(FLOW_OPEN_CHANGE_COMMAND) ? 'open-change' : command === FLOW_TIME_STOP_COMMAND ? 'time-stop' : command === FLOW_REVIEW_BLOCKED_COMMAND ? 'review-blocked' : 'run', 5000); + // The flow reads the clock through a journaled step. + if (command === 'date +%s') return String(epoch + Math.floor(wallMs / 1000)); if (command.endsWith(FLOW_PUBLISH_CHECK_COMMAND)) return 'publish'; if (command.endsWith(FLOW_VALIDATE_CHANGE_METADATA_COMMAND)) return 'valid'; if (command === 'git rev-parse HEAD') return 'abc123'; if (command.startsWith('test -f review.clean')) return 'no'; + // No committed check script, so check-discovery runs (and is charged). + if (command.startsWith('test -s .relayflow/check.sh')) return 'no'; if (command.startsWith('test -s')) return 'yes'; return ''; }, @@ -178,4 +198,39 @@ describe('Garden flow time budget (cloud#4108)', () => { expect(run.calls.map(call => call.name)).not.toContain('time-stop'); expect(run.calls.filter(call => call.name.startsWith('adversary'))).toHaveLength(2); }); + + it('counts parallel prototypes as the budget does, three charges for one stretch of clock', async () => { + // The kernel charges each prototype's own time; three 30m prototypes in + // parallel cost 90m of budget in 30m of wall clock. + const timing: Timing = { + agents: { 'prototype': 30, 'comparator': 10, 'implementer': 30, 'check-repair': FLOW_TIME.repairMinutes, 'adversary': FLOW_TIME.reviewMinutes }, + checks: [{ minutes: FLOW_TIME.checkMinutes, verdict: 'fail' }], + baseline: { minutes: FLOW_TIME.checkMinutes, verdict: 'fail' }, + }; + const run = await runTimed(timing, 'prototype', { parallel: ['prototype'] }); + expect(run.refused).toBeNull(); + expect(run.opened).toBeDefined(); + }); + + it('reports the first review\'s findings when there is no time for a fix round', async () => { + const run = await runTimed({ + agents: { 'planner': 10, 'plan-reviewer': 10, 'implementer': 60, 'adversary': FLOW_TIME.reviewMinutes }, + checks: [{ minutes: FLOW_TIME.checkMinutes, verdict: 'pass' }], + }); + expect(run.refused).toBeNull(); + expect(run.calls.map(call => call.name)).not.toContain('fixer'); + expect(run.calls.map(call => call.name)).toContain('review-blocked'); + expect(run.errors.join('\n')).toMatch(/out of time for a fix round/i); + }); + + it('skips the base-commit check rather than run out of time before publishing', async () => { + const run = await runTimed({ + agents: { 'implementer': 150 }, + checks: [{ minutes: FLOW_TIME.checkMinutes, verdict: 'fail' }], + baseline: { minutes: FLOW_TIME.checkMinutes, verdict: 'fail' }, + }, 'simple'); + expect(run.refused).toBeNull(); + expect(run.calls.map(call => call.name)).not.toContain('base-check'); + expect(run.opened).toBeDefined(); + }); }); From c8db5ce24332969d1cd600cd639483980f9963f5 Mon Sep 17 00:00:00 2001 From: agentrelaybot Date: Fri, 2 Oct 2026 02:57:00 -0700 Subject: [PATCH 4/4] fix(flows): say so when the base check was skipped for time; pin the prototype charge test Review round 2 on #132 (cubic): - a time-skipped base check is "skipped", and the run says there was no time to check the base commit instead of claiming it fails too - the prototype test now fails if parallel time stops counting (it asserts no repair starts); each fix was mutation-checked Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: 3b31cc74-0c43-4890-9650-f2be3566b7ff --- web/lib/flow-workflows.ts | 6 ++++-- web/lib/test/flow-budget.test.ts | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/web/lib/flow-workflows.ts b/web/lib/flow-workflows.ts index 9ae1bf06..864602af 100644 --- a/web/lib/flow-workflows.ts +++ b/web/lib/flow-workflows.ts @@ -783,7 +783,7 @@ export function workflowCode(workflow: WorkflowId, agents: ReturnType { }); it('counts parallel prototypes as the budget does, three charges for one stretch of clock', async () => { - // The kernel charges each prototype's own time; three 30m prototypes in - // parallel cost 90m of budget in 30m of wall clock. - const timing: Timing = { - agents: { 'prototype': 30, 'comparator': 10, 'implementer': 30, 'check-repair': FLOW_TIME.repairMinutes, 'adversary': FLOW_TIME.reviewMinutes }, + // Three 20m prototypes in parallel take 20m of wall clock but 60m of + // budget. By the first failed check the clock shows about 80m used, which + // would leave room for a repair; the budget has spent about 120m, which + // does not. Counting only the clock starts the repair and is refused + // before publishing. + const run = await runTimed({ + agents: { 'prototype': 20, 'comparator': 10, 'implementer': 30, 'check-repair': FLOW_TIME.repairMinutes, 'adversary': FLOW_TIME.reviewMinutes }, checks: [{ minutes: FLOW_TIME.checkMinutes, verdict: 'fail' }], baseline: { minutes: FLOW_TIME.checkMinutes, verdict: 'fail' }, - }; - const run = await runTimed(timing, 'prototype', { parallel: ['prototype'] }); + }, 'prototype', { parallel: ['prototype'] }); + expect(run.calls.filter(call => call.name.startsWith('check-repair'))).toHaveLength(0); expect(run.refused).toBeNull(); expect(run.opened).toBeDefined(); }); @@ -232,5 +235,8 @@ describe('Garden flow time budget (cloud#4108)', () => { expect(run.refused).toBeNull(); expect(run.calls.map(call => call.name)).not.toContain('base-check'); expect(run.opened).toBeDefined(); + // Nothing was checked, so the run must not claim the base commit fails too. + expect(run.errors.join('\n')).toMatch(/no time left to check the base commit/); + expect(run.errors.join('\n')).not.toMatch(/as far as the base commit shows/); }); });