From 75332293dbaa376edcc6c37b8d523696dbcb60c9 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Thu, 24 Sep 2026 06:12:34 +0000 Subject: [PATCH 01/11] test(sdk): reproduce repeated authored CLI probes and expired leases --- .../sdk/tests/authored-probe-cache.test.ts | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 packages/sdk/tests/authored-probe-cache.test.ts diff --git a/packages/sdk/tests/authored-probe-cache.test.ts b/packages/sdk/tests/authored-probe-cache.test.ts new file mode 100644 index 00000000..d9349a21 --- /dev/null +++ b/packages/sdk/tests/authored-probe-cache.test.ts @@ -0,0 +1,136 @@ +import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { flow } from '@relayflows/surface'; +import { executeAuthoredFlow } from '../src/authored-flow-executor.js'; +import { attachLocalAgent } from '../src/local-agent.js'; +import { JournalClient } from '../src/journal-client.js'; +import { LlmWorker } from '../src/llm-worker.js'; +import { socketPathFor } from '../src/daemon-connection.js'; +import { onWorkerFailure } from '../src/worker-lease.js'; +import { DEFAULT_LOCAL_AGENT_CAPACITY } from '../src/worker-slots.js'; +import { chainFixture } from './flow-chain-fixture.js'; + +const closes: Array<() => Promise> = []; +afterEach(async () => { + for (const close of closes.splice(0).reverse()) await close(); +}); + +async function slowProbe(capacity: number, probeMs: number, fail = false) { + const fixture = chainFixture(); + closes.push(() => fixture.close()); + const log = join(fixture.root, 'probes.jsonl'); + writeFileSync(fixture.wrapper, `#!/usr/bin/env node +import { receiveWrapperRequest } from ${JSON.stringify(resolve('../../testdata/preflight/wrapper-session.mjs'))}; +import { appendFileSync } from 'node:fs'; +const log = value => appendFileSync(${JSON.stringify(log)}, JSON.stringify(value) + '\\n'); +if (process.argv[2] === 'auth') { + log({ kind: 'auth' }); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ${probeMs}); + ${fail ? "process.kill(process.pid, 'SIGTERM');" : 'process.exit(0);'} +} +// Identify-only probes exit inside receiveWrapperRequest; executions also +// identify, so subtract the recorded sessions to count preflight probes. +log({ kind: 'identify' }); +const request = await receiveWrapperRequest(); +if (request) { + const start = Date.now(); + await new Promise(done => setTimeout(done, 200)); + log({ kind: 'session', start, end: Date.now() }); + process.stdout.write('{"x":4}'); +} +`); + const client = await fixture.connect(); + const agent = await attachLocalAgent(client, undefined, undefined, undefined, capacity, fixture.root); + closes.push(() => agent.close()); + const llmClient = new JournalClient(socketPathFor(fixture.data)); + await llmClient.connect(); + await llmClient.hello('probe-cache-llm'); + closes.push(async () => { llmClient.close(); }); + const worker = new LlmWorker(llmClient, `${agent.stream}-llm`, capacity); + const failures: unknown[] = []; + worker.on('error', onWorkerFailure('test-llm', error => { failures.push(error); client.close(); })); + await worker.attach(); + closes.push(() => worker.close()); + const logs = () => readFileSync(log, 'utf8').trim().split('\n') + .map(line => JSON.parse(line) as { kind: string; start: number; end: number }); + return { fixture, client, agent, failures, logs, + options: { flowPath: fixture.flowPath, localAgentStream: agent.stream, workerCapacity: capacity } }; +} + +const nine = flow('nine-llms', async f => { + const xs = await Promise.all(Array.from({ length: 9 }, (_, i) => f.llm(`Return JSON for ${i}`, { + output: { type: 'object', properties: { x: { type: 'number' } }, required: ['x'] }, model: 'test-model', + }))); + expect(xs).toEqual(Array.from({ length: 9 }, () => ({ x: 4 }))); + f.done('success'); +}); + +function expectOneProbe(logs: ReturnType>['logs']>) { + expect(logs.filter(l => l.kind === 'auth')).toHaveLength(1); + expect(logs.filter(l => l.kind === 'identify').length - logs.filter(l => l.kind === 'session').length).toBe(1); +} + +describe('authored run CLI probe cache', () => { + it.each([1, DEFAULT_LOCAL_AGENT_CAPACITY])('nine calls have no expired attempts at capacity %i', async capacity => { + const { fixture, client, options, failures, logs } = await slowProbe(capacity, 4_000); + const result = await executeAuthoredFlow(nine, client, undefined, options); + expect(result.completionReason).toBe('success'); + expect(failures).toEqual([]); + const runs = readdirSync(join(fixture.data, 'runs')).filter(name => name.endsWith('.sqlite3')); + expect(runs.length).toBeGreaterThanOrEqual(9); + for (const file of runs) { + const { entries } = await client.journalRead(file.slice(0, -8), 1); + expect(JSON.stringify(entries), file).not.toContain('lease_expired'); + expect(entries.filter(e => e.entry_type === 'step.attempt_started'), file).toHaveLength(1); + } + const sessions = logs().filter(l => l.kind === 'session'); + expect(sessions).toHaveLength(9); + const peak = Math.max(...sessions.map(s => sessions.filter(o => o.start <= s.start && s.start < o.end).length)); + expect(peak).toBeLessThanOrEqual(capacity); + expectOneProbe(logs()); + }, 120_000); + + it('does not serialize nine starts behind nine probes, and probes again on a new run', async () => { + const { client, options, logs } = await slowProbe(1, 300); + const starts: number[] = []; + await executeAuthoredFlow(nine, client, undefined, { ...options, onProgress: event => { + if (event.type === 'step.started' && event.stepType === 'llm') starts.push(performance.now()); + } }); + expect(starts).toHaveLength(9); + // F1 permits one synchronous probe (~300ms), not nine (~3s). + expect(starts.at(-1)! - starts[0]!).toBeLessThan(450); + expectOneProbe(logs()); + await executeAuthoredFlow(nine, client, undefined, options); + expect(logs().filter(l => l.kind === 'auth')).toHaveLength(2); + }, 20_000); + + it('caches probe failures while refusing all nine calls', async () => { + const { client, options, logs, fixture } = await slowProbe(1, 300, true); + const refusals: PromiseSettledResult[] = []; + const failing = flow('failed-probes', async f => { + refusals.push(...await Promise.allSettled(Array.from({ length: 9 }, () => f.llm`hello`))); + f.done('success'); + }); + await expect(executeAuthoredFlow(failing, client, undefined, options)).rejects.toThrow('probe'); + expect(refusals).toHaveLength(9); + for (const refusal of refusals) { + expect(refusal.status).toBe('rejected'); + if (refusal.status === 'rejected') expect(String(refusal.reason)).toContain('probe'); + } + expectOneProbe(logs()); + // Refused LLMs never acquired a lease. + expect(readdirSync(join(fixture.data, 'runs')).filter(n => n.endsWith('.sqlite3'))).toHaveLength(0); + }, 20_000); + + it('shares probe results across agent calls too', async () => { + const { client, options, logs, fixture } = await slowProbe(1, 300); + mkdirSync(join(fixture.root, 'work')); + const agents = flow('three-agents', async f => { + await Promise.all(['a', 'b', 'c'].map(name => f.agent(name, { task: 'work', cwd: 'work' }))); + f.done('success'); + }); + expect((await executeAuthoredFlow(agents, client, undefined, options)).completionReason).toBe('success'); + expectOneProbe(logs()); + }, 20_000); +}); From 55caf19008567b833ba3c71b7c38fc8983ed87c7 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Thu, 24 Sep 2026 06:17:40 +0000 Subject: [PATCH 02/11] fix(sdk): reuse authored CLI probes for each run before worker admission --- packages/sdk/src/authored-worker-step.ts | 13 +++++++--- packages/sdk/src/cli/check.ts | 3 +++ packages/sdk/src/preflight.ts | 6 +++-- .../sdk/tests/authored-probe-cache.test.ts | 16 +++++++----- .../sdk/tests/preflight-run-cache.test.ts | 26 +++++++++++++++++++ 5 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 packages/sdk/tests/preflight-run-cache.test.ts diff --git a/packages/sdk/src/authored-worker-step.ts b/packages/sdk/src/authored-worker-step.ts index ac936fec..8f14a6fe 100644 --- a/packages/sdk/src/authored-worker-step.ts +++ b/packages/sdk/src/authored-worker-step.ts @@ -1,11 +1,11 @@ -import { isAbsolute, relative, sep } from 'node:path'; +import { dirname, resolve, isAbsolute, relative, sep } from 'node:path'; import type { AuthoredBudget } from './authored-budget.js'; import { parseBudget } from './budget.js'; import type { AgentOptions, AgentResult, LlmOptions, NamedGate } from '@relayflows/surface'; import { compileSpec, toKernelSpec } from './compile.js'; -import { checkAuthoredFlow } from './cli/check.js'; +import { checkAuthoredFlow, readProjectConfig, type ProjectConfig } from './cli/check.js'; import { classifyOutcome, type RunLifecycleOptions, type RunReport } from './cli/run.js'; -import type { PreflightDiagnostic } from './preflight.js'; +import type { CliProbeOutcome, PreflightDiagnostic } from './preflight.js'; import { AuthoredFlowExecutionError } from './authored-flow-error.js'; import { agentCwdDeclarationError, agentCwdTransportError } from './agent-cwd.js'; import type { JournalClient } from './journal-client.js'; @@ -31,6 +31,10 @@ export function authoredWorkerRunner( // slot instead of being admitted and parked for want of a free worker. const slots = workerCapacity === undefined ? undefined : { agent: new WorkerSlots(workerCapacity), llm: new WorkerSlots(workerCapacity) }; + // Repeated synchronous probes starve dispatch and heartbeats before admission. + // Match declarative preflight: cache both pass and refusal for this run only. + const cliProbeCache = new Map(); + let projectConfig: ProjectConfig | undefined; const context: AuthoredStepContext = { ...(rootRunId === undefined ? {} : { rootRunId }), ...(waitOptions.dataDir === undefined ? {} : { dataDir: waitOptions.dataDir }), @@ -44,7 +48,8 @@ export function authoredWorkerRunner( // (cli/check.ts), searching for the nearest flows.json from `flowPath` // and real-probing auth/model readiness. An authored agent step gets // nothing for free just because it was declared in TS instead of YAML. - const { report, flow: resolved } = checkAuthoredFlow(authoring, flowPath); + const { report, flow: resolved } = checkAuthoredFlow(authoring, flowPath, + projectConfig ??= readProjectConfig(dirname(resolve(flowPath))), {}, cliProbeCache); if (!report.ok || resolved === undefined) { const refusal = report.diagnostics.find( (diagnostic): diagnostic is PreflightDiagnostic & { severity: 'refusal' } => diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index 12d9df0c..c0352f1b 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -29,6 +29,7 @@ import { CliProbeError, type CliResolution, type CliProbeResult, + type CliProbeOutcome, type PreflightDiagnostic, type PreflightProbes, } from '../preflight.js'; @@ -188,12 +189,14 @@ export function checkAuthoredFlow( path: string, projectConfig?: ProjectConfig, invocation: CheckInvocation = {}, + cliProbeCache?: Map, ): CheckExecution { const absolutePath = resolve(path); try { const config = projectConfig ?? readProjectConfig(dirname(absolutePath)); const probes = systemProbes(dirname(absolutePath), config); const result = preflight(authoring, { + ...(cliProbeCache === undefined ? {} : { cliProbeCache }), projectCli: config.cli, projectConfigPath: config.path, projectSearchStart: dirname(absolutePath), diff --git a/packages/sdk/src/preflight.ts b/packages/sdk/src/preflight.ts index 360b5251..739a2eba 100644 --- a/packages/sdk/src/preflight.ts +++ b/packages/sdk/src/preflight.ts @@ -65,7 +65,7 @@ export class CliProbeError extends Error { } } -type CliProbeOutcome = +export type CliProbeOutcome = | { result: CliProbeResult } | { failure: CliProbeFailureDetail | null }; @@ -89,6 +89,8 @@ export interface PreflightProbes { } export interface PreflightOptions { + /** Reuse only within one run and one CLI resolution environment; failures are cached too. */ + cliProbeCache?: Map; pluginSearchStart?: string; /** Validated tools.mcp header and nearest flows.json connections. */ mcpServers?: readonly string[]; @@ -235,7 +237,7 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul const cliResolutionDiagnostics: PreflightDiagnostic[] = []; const resolutions: CliResolution[] = []; const resolutionByStep = new Map(); - const cliProbeResults = new Map(); + const cliProbeResults = options.cliProbeCache ?? new Map(); // A pure fact about the compiled snapshot, collected before anything that // can return early. An unresolved CLI, an unknown model or a bad scope all diff --git a/packages/sdk/tests/authored-probe-cache.test.ts b/packages/sdk/tests/authored-probe-cache.test.ts index d9349a21..b554e7f9 100644 --- a/packages/sdk/tests/authored-probe-cache.test.ts +++ b/packages/sdk/tests/authored-probe-cache.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; import { flow } from '@relayflows/surface'; @@ -74,16 +74,17 @@ function expectOneProbe(logs: ReturnType>[' describe('authored run CLI probe cache', () => { it.each([1, DEFAULT_LOCAL_AGENT_CAPACITY])('nine calls have no expired attempts at capacity %i', async capacity => { const { fixture, client, options, failures, logs } = await slowProbe(capacity, 4_000); - const result = await executeAuthoredFlow(nine, client, undefined, options); - expect(result.completionReason).toBe('success'); + const result = await executeAuthoredFlow(nine, client, undefined, options).catch((error: unknown) => error); expect(failures).toEqual([]); const runs = readdirSync(join(fixture.data, 'runs')).filter(name => name.endsWith('.sqlite3')); - expect(runs.length).toBeGreaterThanOrEqual(9); for (const file of runs) { const { entries } = await client.journalRead(file.slice(0, -8), 1); expect(JSON.stringify(entries), file).not.toContain('lease_expired'); - expect(entries.filter(e => e.entry_type === 'step.attempt_started'), file).toHaveLength(1); + expect(entries.filter(e => typeof e === 'object' && e !== null + && 'entry_type' in e && e.entry_type === 'step.attempt.started'), file).toHaveLength(1); } + expect(result).toMatchObject({ completionReason: 'success' }); + expect(runs.length).toBeGreaterThanOrEqual(9); const sessions = logs().filter(l => l.kind === 'session'); expect(sessions).toHaveLength(9); const peak = Math.max(...sessions.map(s => sessions.filter(o => o.start <= s.start && s.start < o.end).length)); @@ -99,7 +100,8 @@ describe('authored run CLI probe cache', () => { } }); expect(starts).toHaveLength(9); // F1 permits one synchronous probe (~300ms), not nine (~3s). - expect(starts.at(-1)! - starts[0]!).toBeLessThan(450); + // Leave process-startup headroom for loaded CI; exact counts below pin caching. + expect(starts.at(-1)! - starts[0]!).toBeLessThan(1_000); expectOneProbe(logs()); await executeAuthoredFlow(nine, client, undefined, options); expect(logs().filter(l => l.kind === 'auth')).toHaveLength(2); @@ -120,7 +122,7 @@ describe('authored run CLI probe cache', () => { } expectOneProbe(logs()); // Refused LLMs never acquired a lease. - expect(readdirSync(join(fixture.data, 'runs')).filter(n => n.endsWith('.sqlite3'))).toHaveLength(0); + expect(existsSync(join(fixture.data, 'runs'))).toBe(false); }, 20_000); it('shares probe results across agent calls too', async () => { diff --git a/packages/sdk/tests/preflight-run-cache.test.ts b/packages/sdk/tests/preflight-run-cache.test.ts new file mode 100644 index 00000000..e3a1b88b --- /dev/null +++ b/packages/sdk/tests/preflight-run-cache.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest'; +import { preflight, type CliProbeOutcome, type PreflightProbes } from '../src/preflight.js'; +import type { FlowSpec } from '../src/spec.js'; + +describe('preflight shared run cache', () => { + it('keeps CLI, resolution source and model in the cache key', () => { + const cliProbeCache = new Map(); + const calls: unknown[][] = []; + const probes: PreflightProbes = { + cli: (...args) => { calls.push(args); return { exists: true, authenticated: true, modelAvailable: true }; }, + executor: () => true, command: () => true, + }; + function check(cli: string, model: string, source: 'step' | 'project') { + const spec: FlowSpec = { version: '0.1.0', name: 'cache', steps: [{ + id: 'call', type: 'llm', prompt: 'hello', model, ...(source === 'step' ? { cli } : {}), + }] }; + return preflight(spec, { probes, cliProbeCache, projectCli: cli, models: ['a', 'b'] }); + } + expect(check('claude', 'a', 'step').ok).toBe(true); + expect(check('claude', 'a', 'step').ok).toBe(true); + expect(check('claude', 'b', 'step').ok).toBe(true); + expect(check('codex', 'a', 'step').ok).toBe(true); + expect(check('claude', 'a', 'project').ok).toBe(true); + expect(calls).toHaveLength(4); + }); +}); From 3572bf8fe48270a98f4534408c23121af4f6ab77 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Thu, 24 Sep 2026 06:21:35 +0000 Subject: [PATCH 03/11] docs: capture lease regression mutation and live repro evidence --- evidence/561/README.md | 89 ++ evidence/561/baseline.txt | 115 ++ evidence/561/cache-keys.txt | 10 + evidence/561/check-live-journals.py | 23 + evidence/561/fixed.txt | 15 + evidence/561/flows.json | 1 + evidence/561/kernel.txt | 458 ++++++ evidence/561/live-default.txt | 41 + evidence/561/live-journals.txt | 20 + evidence/561/live-one.txt | 41 + evidence/561/mutation-green.txt | 15 + evidence/561/mutation-red.txt | 109 ++ evidence/561/mutation.patch | 13 + evidence/561/package.json | 1 + evidence/561/prompt-lab.txt | 2 + evidence/561/regression-typecheck.txt | 1 + evidence/561/restore.txt | 5 + .../561/runtime-parallel-llm-repro.flow.ts | 8 + evidence/561/sdk-suite.txt | 1373 +++++++++++++++++ evidence/561/tsconfig.tests.json | 14 + evidence/561/typecheck.txt | 4 + summary.md | 222 +-- 22 files changed, 2392 insertions(+), 188 deletions(-) create mode 100644 evidence/561/README.md create mode 100644 evidence/561/baseline.txt create mode 100644 evidence/561/cache-keys.txt create mode 100644 evidence/561/check-live-journals.py create mode 100644 evidence/561/fixed.txt create mode 100644 evidence/561/flows.json create mode 100644 evidence/561/kernel.txt create mode 100644 evidence/561/live-default.txt create mode 100644 evidence/561/live-journals.txt create mode 100644 evidence/561/live-one.txt create mode 100644 evidence/561/mutation-green.txt create mode 100644 evidence/561/mutation-red.txt create mode 100644 evidence/561/mutation.patch create mode 100644 evidence/561/package.json create mode 100644 evidence/561/prompt-lab.txt create mode 100644 evidence/561/regression-typecheck.txt create mode 100644 evidence/561/restore.txt create mode 100644 evidence/561/runtime-parallel-llm-repro.flow.ts create mode 100644 evidence/561/sdk-suite.txt create mode 100644 evidence/561/tsconfig.tests.json create mode 100644 evidence/561/typecheck.txt diff --git a/evidence/561/README.md b/evidence/561/README.md new file mode 100644 index 00000000..084d328d --- /dev/null +++ b/evidence/561/README.md @@ -0,0 +1,89 @@ +# #561 verification commands and captured output + +Commands ran from the repository root unless a `cd` is shown. Output files are literal stdout/stderr captures, not rewritten summaries. The initial pre-fix tests were committed in `7533229`; the fix and corrected assertions are in `55caf19`. The first baseline predates the corrected journal entry spelling and timing headroom; the mutation transcript uses the final assertions. + +## Regression and mutation + +For `baseline.txt`, `fixed.txt`, `mutation-red.txt`, and `mutation-green.txt`: + +```sh +cd packages/sdk +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npx vitest run tests/authored-probe-cache.test.ts +``` + +[Baseline](baseline.txt), [fixed](fixed.txt), [mutation failure](mutation-red.txt), [restored pass](mutation-green.txt). + +To reproduce the mutation from the fixed checkout: + +```sh +git apply evidence/561/mutation.patch +# Run the regression command above (exit 1). +git restore -- packages/sdk/src/authored-worker-step.ts +git diff --exit-code -- packages/sdk/src/authored-worker-step.ts +# Run the regression command above again (exit 0). +``` + +The mutation removes only the fifth `checkAuthoredFlow` argument, as captured in [mutation.patch](mutation.patch). After the failure, `git restore -- packages/sdk/src/authored-worker-step.ts` restores the committed bytes. [restore.txt](restore.txt) captures `git diff --exit-code` and the SHA-256 comparison with HEAD. + +The spread bound allows one 300 ms synchronous probe plus process startup under load (1,000 ms total); the original nine probes take over 3 seconds. Exact auth and identify-only counts additionally pin the cache independently of timing. Worker identification sessions are counted separately from preflight probes. Tests also cover failures, agents, run isolation, capacity, and every child journal. + +## Broader checks + +```sh +cd packages/sdk +npm test +``` + +[sdk-suite.txt](sdk-suite.txt) is the full output, including failures. It includes production and existing test typechecks and the SDK build. It is **not green**. No gates or existing tests were weakened to accommodate this environment. + +```sh +cd kernel +sh ../ops/cargo.sh test +``` + +[kernel.txt](kernel.txt) contains the full kernel result. + +```sh +cd packages/sdk +npm run typecheck +``` + +[typecheck.txt](typecheck.txt). + +```sh +cd packages/sdk +npx vitest run tests/preflight-run-cache.test.ts +``` + +[cache-keys.txt](cache-keys.txt). + +## Live reproduction + +The [repro source](runtime-parallel-llm-repro.flow.ts) is copied verbatim from `origin/feat/examples-prompt-lab:examples/prompt-lab/evidence/runtime-findings/runtime-parallel-llm-repro.flow.ts`. Adjacent `flows.json` chooses Claude and allows its declared model; `package.json` declares ESM. + +```sh +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd node packages/sdk/dist/cli.js run --local-agent --no-observer-link --data-dir /tmp/flows-561-live-default evidence/561/runtime-parallel-llm-repro.flow.ts --input '{"n":3}' +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd node packages/sdk/dist/cli.js run --local-agent --agent-capacity 1 --no-observer-link --data-dir /tmp/flows-561-live-one evidence/561/runtime-parallel-llm-repro.flow.ts --input '{"n":3}' +python3 evidence/561/check-live-journals.py +``` + +[Default capacity](live-default.txt), [capacity 1](live-one.txt), [journal assertions and outputs](live-journals.txt). Both CLI invocations exited 0. Journals live in the named `/tmp` directories in this environment; they are not committed. Reproduction needs authenticated Claude. `npm run build` in `packages/sdk` regenerates `dist` if the test runner has pruned it. + +```sh +python3 - <<'PY' +from pathlib import Path +for p in ['examples/prompt-lab/jobs/shared.ts', 'examples/prompt-lab/jobs/new-agency.ts']: + print(p + ': ' + ('present' if Path(p).exists() else 'absent')) +PY +``` + +[prompt-lab.txt](prompt-lab.txt) records why the example restoration remains open. + +## Typecheck the new regression files + +```sh +cd packages/sdk +npx tsc -p ../../evidence/561/tsconfig.tests.json +``` + +[regression-typecheck.txt](regression-typecheck.txt) captures output and exit status. This separate config includes the new tests without changing the repository's check configuration. diff --git a/evidence/561/baseline.txt b/evidence/561/baseline.txt new file mode 100644 index 00000000..33be0a47 --- /dev/null +++ b/evidence/561/baseline.txt @@ -0,0 +1,115 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +(node:16742) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M390MBFVS1E4P7XHG40XXVT3 step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M390MBFVS1E4P7XHG40XXVT3/llm-1. +(Use `node --trace-warnings ...` to show where the warning was created) +(node:16742) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M390NG2Q7NEX3EPDJJCAE5T7 step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M390NG2Q7NEX3EPDJJCAE5T7/llm-1. + ❯ tests/authored-probe-cache.test.ts (5 tests | 5 failed) 86829ms + × authored run CLI probe cache > nine calls have no expired attempts at capacity 1 37494ms + → step_failed: journal step "llm-1" completed with lease_expired +Inspect: flows replay 01M390MBFVS1E4P7XHG40XXVT3 --at llm-1 + × authored run CLI probe cache > nine calls have no expired attempts at capacity 4 37529ms + → step_failed: journal step "llm-1" completed with lease_expired +Inspect: flows replay 01M390NG2Q7NEX3EPDJJCAE5T7 --at llm-1 + × authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 6453ms + → expected 3250.5002949999907 to be less than 450 + × authored run CLI probe cache > caches probe failures while refusing all nine calls 3299ms + → expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9 + × authored run CLI probe cache > shares probe results across agent calls too 2052ms + → expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 5 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 1 +AuthoredFlowExecutionError: step_failed: journal step "llm-1" completed with lease_expired +Inspect: flows replay 01M390MBFVS1E4P7XHG40XXVT3 --at llm-1 + ❯ stepFailure src/authored-step-output.ts:123:10 + 121| ...edges, + 122| }); + 123| return new AuthoredFlowExecutionError('step_failed', message, reason… + | ^ + 124| } + 125| + ❯ Module.readCompletedStepOutput src/authored-step-output.ts:70:11 + ❯ WorkerSlots.run src/worker-slots.ts:44:14 + ❯ Object.llm src/authored-worker-step.ts:233:22 + ❯ Module.observeStep src/progress.ts:48:20 + ❯ AuthoredFlowOperation.begin src/authored-flow-operation.ts:174:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 4 +AuthoredFlowExecutionError: step_failed: journal step "llm-1" completed with lease_expired +Inspect: flows replay 01M390NG2Q7NEX3EPDJJCAE5T7 --at llm-1 + ❯ stepFailure src/authored-step-output.ts:123:10 + 121| ...edges, + 122| }); + 123| return new AuthoredFlowExecutionError('step_failed', message, reason… + | ^ + 124| } + 125| + ❯ Module.readCompletedStepOutput src/authored-step-output.ts:70:11 + ❯ WorkerSlots.run src/worker-slots.ts:44:14 + ❯ Object.llm src/authored-worker-step.ts:233:22 + ❯ Module.observeStep src/progress.ts:48:20 + ❯ AuthoredFlowOperation.begin src/authored-flow-operation.ts:174:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run +AssertionError: expected 3250.5002949999907 to be less than 450 + ❯ tests/authored-probe-cache.test.ts:102:41 + 100| expect(starts).toHaveLength(9); + 101| // F1 permits one synchronous probe (~300ms), not nine (~3s). + 102| expect(starts.at(-1)! - starts[0]!).toBeLessThan(450); + | ^ + 103| expectOneProbe(logs()); + 104| await executeAuthoredFlow(nine, client, undefined, options); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > caches probe failures while refusing all nine calls +AssertionError: expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9 + +- Expected ++ Received + +- 1 ++ 9 + + ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47 + 68| + 69| function expectOneProbe(logs: ReturnType l.kind === 'auth')).toHaveLength(1); + | ^ + 71| expect(logs.filter(l => l.kind === 'identify').length - logs.filter(… + 72| } + ❯ tests/authored-probe-cache.test.ts:121:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > shares probe results across agent calls too +AssertionError: expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3 + +- Expected ++ Received + +- 1 ++ 3 + + ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47 + 68| + 69| function expectOneProbe(logs: ReturnType l.kind === 'auth')).toHaveLength(1); + | ^ + 71| expect(logs.filter(l => l.kind === 'identify').length - logs.filter(… + 72| } + ❯ tests/authored-probe-cache.test.ts:134:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/5]⎯ + + Test Files 1 failed (1) + Tests 5 failed (5) + Start at 06:11:11 + Duration 88.74s (transform 1.05s, setup 0ms, collect 1.73s, tests 86.83s, environment 0ms, prepare 44ms) + diff --git a/evidence/561/cache-keys.txt b/evidence/561/cache-keys.txt new file mode 100644 index 00000000..f538b5c1 --- /dev/null +++ b/evidence/561/cache-keys.txt @@ -0,0 +1,10 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/preflight-run-cache.test.ts (1 test) 12ms + + Test Files 1 passed (1) + Tests 1 passed (1) + Start at 06:15:07 + Duration 1.02s (transform 390ms, setup 0ms, collect 836ms, tests 12ms, environment 0ms, prepare 43ms) + diff --git a/evidence/561/check-live-journals.py b/evidence/561/check-live-journals.py new file mode 100644 index 00000000..69e79621 --- /dev/null +++ b/evidence/561/check-live-journals.py @@ -0,0 +1,23 @@ +import glob +import json +import sqlite3 + +for directory in ['/tmp/flows-561-live-default', '/tmp/flows-561-live-one']: + journals = sorted(glob.glob(directory + '/runs/*.sqlite3')) + assert journals + llms = 0 + for path in journals: + with sqlite3.connect('file:' + path + '?mode=ro', uri=True) as db: + rows = db.execute('select entry_type, step_id, payload from entries').fetchall() + assert all('lease_expired' not in payload for _, _, payload in rows), path + starts = [step for kind, step, _ in rows if kind == 'step.attempt.started' and step.startswith('llm-')] + if starts: + assert len(starts) == 1, (path, starts) + llms += 1 + for kind, step, payload in rows: + if kind == 'step.completed' and step and step.startswith('llm-'): + value = json.loads(payload) + assert value['completionReason'] == 'success', value + print(step, value['output']) + assert llms == 9, llms + print(directory, f'{len(journals)} journals, {llms} LLM children, one attempt each, no lease_expired') diff --git a/evidence/561/fixed.txt b/evidence/561/fixed.txt new file mode 100644 index 00000000..bdc7a301 --- /dev/null +++ b/evidence/561/fixed.txt @@ -0,0 +1,15 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-probe-cache.test.ts (5 tests) 22896ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 1 7405ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 4 5425ms + ✓ authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 8357ms + ✓ authored run CLI probe cache > caches probe failures while refusing all nine calls 401ms + ✓ authored run CLI probe cache > shares probe results across agent calls too 1306ms + + Test Files 1 passed (1) + Tests 5 passed (5) + Start at 06:13:43 + Duration 24.51s (transform 872ms, setup 0ms, collect 1.45s, tests 22.90s, environment 0ms, prepare 46ms) + diff --git a/evidence/561/flows.json b/evidence/561/flows.json new file mode 100644 index 00000000..139ab56b --- /dev/null +++ b/evidence/561/flows.json @@ -0,0 +1 @@ +{"cli":"claude","models":["claude-haiku-4-5-20251001"]} diff --git a/evidence/561/kernel.txt b/evidence/561/kernel.txt new file mode 100644 index 00000000..fdd2f414 --- /dev/null +++ b/evidence/561/kernel.txt @@ -0,0 +1,458 @@ + Downloading crates ... + Downloaded errno v0.3.14 + Downloaded fastrand v2.5.0 + Downloaded getrandom v0.4.3 + Downloaded tempfile v3.27.0 + Downloaded rustix v1.1.4 + Downloaded linux-raw-sys v0.12.1 + Compiling bitflags v2.13.1 + Compiling getrandom v0.4.3 + Compiling rustix v1.1.4 + Compiling linux-raw-sys v0.12.1 + Compiling rusqlite v0.37.0 + Compiling fastrand v2.5.0 + Compiling relayflowd-core v0.1.0 (/home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/relayflowd-journal) + Compiling tempfile v3.27.0 + Compiling relayflowd v0.1.0 (/home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 13.92s + Running unittests src/lib.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/relayflowd-f043db0bb3534a16) + +running 55 tests +test engine::remote::worker_failure_detail_tests::a_non_string_output_is_rendered_rather_than_dropped ... ok +test engine::remote::worker_failure_detail_tests::a_null_or_blank_output_yields_no_detail ... ok +test engine::boot_identity_tests::every_engine_in_this_process_shares_one_boot_id ... ok +test engine::remote::worker_failure_detail_tests::a_string_output_is_carried_verbatim_and_trimmed ... ok +test engine::remote::worker_failure_detail_tests::an_output_at_the_boundary_is_not_truncated ... ok +test engine::remote::worker_failure_detail_tests::render_is_bounded_before_allocation_for_hostile_string ... ok +test engine::remote::worker_failure_detail_tests::truncation_does_not_split_a_multi_byte_char ... ok +test engine::wake::claim_guard_tests::a_disarmed_guard_leaves_the_claim_alone ... ok +test engine::remote::worker_failure_detail_tests::render_is_bounded_before_allocation_for_hostile_json ... ok +test engine::wake::claim_guard_tests::a_guard_only_releases_its_own_run ... ok +test engine::wake::claim_guard_tests::a_panic_between_claim_and_register_still_releases ... ok +test exec_det::tests::captures_deterministic_output ... ok +test exec_det::tests::failed_command_evidence_survives_completion ... ok +test engine::wake::claim_guard_tests::an_armed_guard_releases_the_claim_when_dropped ... ok +test engine::boot_identity_tests::prior_boot_registered_undriven_admission_is_recovered_by_start_retry ... ok +test exec_det::tests::timeout_has_an_explicit_completion_reason ... ok +test server::channels::tests::unknown_verb_never_falls_through_to_receive ... ok +test server::client::tests::resume_waits_while_the_heartbeat_renewed_lease_is_live ... ok +test server::liveness::tests::sweep_id_buckets_by_the_interval ... ok +test exec_det::tests::lease_override_bounds_execution_and_preserves_command_timeout ... ok +test engine::boot_identity_tests::failure_after_workspace_binding_releases_admission_without_exposing_effects ... ok +test server::liveness::tests::sweep_pass_healthy_subscription_is_a_noop ... ok +test server::liveness::tests::sweep_pass_latches_after_journaling_and_next_bucket_is_empty ... ok +test server::tests::a_lone_surrogate_in_a_request_is_refused_with_no_request_id ... ok +test server::tests::a_failed_disconnect_journal_append_is_retained_and_retried_not_dropped ... ok +test server::tests::agent::contract::a_transcript_digest_at_its_budget_rides_trajectory_tail_verbatim ... ok +test server::tests::agent::contract::an_agent_worker_attaching_without_pins_is_refused_at_attach ... ok +test server::tests::agent::contract::a_replacement_worker_that_never_reported_the_pinned_surface_is_not_dispatched_to ... ok +test server::tests::agent::contract::agent_without_a_compatible_worker_parks_without_starting ... ok +test server::tests::agent::contract::an_oversized_trajectory_tail_is_refused_at_step_complete ... ok +test server::tests::agent::contract::an_agent_worker_missing_a_declared_surface_parks_the_run_instead_of_erroring ... ok +test server::tests::agent::contract::an_llm_completion_claiming_an_effect_fails_closed_with_the_reason_journaled ... ok +test server::tests::agent::eligibility::required_streams_must_be_held_before_worker_registration ... ok +test server::tests::agent::eligibility::required_streams_keep_ordinary_steps_off_conversation_workers ... ok +test server::tests::agent::contract::human_intervention_is_durable_and_resume_requires_explicit_override ... ok +test server::tests::agent::pins::reset_worker_reporting_a_revision_other_than_its_pin_fails_closed_as_worker_error ... ok +test server::tests::agent::pins::consecutive_agent_steps_on_different_surfaces_each_start_from_their_own_pins ... ok +test server::tests::deterministic_marker_carrying_json_survives_reopen_and_refuses_changed_detail ... ok +test server::tests::hello_enforces_protocol_version ... ok +test server::tests::run_resume_adopts_a_real_journal_whose_registry_row_is_missing ... ok +test server::tests::run_resume_asks_the_registry_instead_of_treating_an_orphan_file_as_a_run ... ok +test server::tests::run_resume_refuses_a_journal_that_never_recorded_its_run ... ok +test server::tests::run_resume_refuses_a_valid_journal_that_belongs_to_another_run ... ok +test server::tests::run_start_admission_key_recovers_the_same_run_and_refuses_spec_drift ... ok +test server::tests::run_start_fails_closed_on_an_unknown_verification_key ... ok +test server::tests::run_start_refuses_invalid_admission_keys ... ok +test server::tests::step_wait_parks_the_attempt_and_a_human_answer_redispatches_it ... ok +test server::tests::stopped_heartbeats_past_the_deadline_journal_lease_expired_and_release_the_step ... ok +test socket_path::tests::deep_data_dir_produces_short_socket_path ... ok +test socket_path::tests::different_data_dirs_yield_different_sockets ... ok +test socket_path::tests::relative_and_absolute_data_dirs_agree ... ok +test socket_path::tests::same_data_dir_yields_same_socket ... ok +test exec_det::tests::timeout_kills_the_whole_process_group ... ok +test server::tests::agent::pins::a_replacement_worker_at_a_different_revision_is_not_dispatched_the_stale_pins ... ok +test server::tests::an_entry_appended_during_watch_registration_is_delivered_exactly_once ... ok + +test result: ok. 55 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.64s + + Running unittests src/main.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/relayflowd-6e3681176306c99e) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/advisory_deterministic.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/advisory_deterministic-5c10aa9bf9dc5692) + +running 2 tests +test the_same_flow_without_the_declaration_still_fails_and_never_reaches_the_dependent ... ok +test a_recorded_red_step_completes_and_hands_its_exit_code_to_the_next_step ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.09s + + Running tests/budget_gate.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/budget_gate-9607ae7c2db11b74) + +running 10 tests +test daily_windows_reset_and_exact_limits_do_not_refuse ... ok +test carried_metered_dollars_still_stop_the_continuing_run ... ok +test metering_flag_is_additive_on_the_wire ... ok +test prior_spend_metering_flag_is_additive_and_fails_closed_for_older_kernels ... ok +test carried_prior_spend_keeps_unknown_dollar_cost_unmetered ... ok +test crossing_completion_is_durable_and_next_step_is_refused ... ok +test deterministic_spend_and_wallclock_limit_gate_parallel_batch_starts ... ok +test unmetered_spend_is_journaled_as_unknown_and_never_crosses_a_dollar_ceiling ... ok +test unmetered_tokens_still_cross_a_token_ceiling ... ok +test unmetered_usage_may_not_claim_priced_dollars ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/crash_resume.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/crash_resume-d2d102ec3e89b705) + +running 40 tests +test agent::resume_without_a_worker_parks_immediately_instead_of_timing_out ... ok +test agent::rung_c_sigkill_after_final_effect_replays_results_without_redispatch ... ok +test agent::rung_c_reset_sigkill_mid_edit_restores_pins_dedupes_effect_and_explains_attempts ... ok +test agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once ... ok +test agent::rung_c_sigkill_between_agent_completion_and_final_effect_memoizes_the_agent ... ok +test channels::channels_reject_foreign_workers_stale_attempts_and_invalid_acknowledgements ... ok +test concurrency::cancel_and_completion_race_has_one_terminal_fact ... ok +test agent::rung_c_sigkill_boundaries_resume_only_unfinished_steps_via_real_cli ... ok +test concurrency::cancel_closes_the_lease_and_rejects_a_late_completion ... ok +test concurrency::concurrent_resumes_lease_exactly_one_attempt ... ok +test concurrency::run_start_dispatches_every_independent_lane_before_any_completion ... ok +test concurrency::live_resume_leaves_an_active_lease_running ... ok +test concurrency::server_restart_recovers_every_parallel_lease_without_duplicate_success ... ok +test llm::failing_llm_verification_schedules_a_durable_retry_and_succeeds ... ok +test llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step ... ok +test llm::serve_plumbs_watch_events_and_replayable_stream_verbs ... ok +test llm::llm_verification_exhaustion_is_a_declared_failure_kind ... ok +test llm::sigkill_after_the_final_rung_b_effect_resumes_without_redispatching_llm ... ok +test llm::sigkill_under_serve_mid_llm_releases_the_lease_and_finishes_via_cli_resume ... ok +test channels::channels_sigkill_resume_redelivers_unacked_messages_with_exactly_once_effects ... ok +test llm::worker_killed_while_holding_a_lease_is_explained_and_released_on_cli_resume ... ok +test memory::memory_sigkill_after_injection_replays_pack_and_charges_it_once ... ok +test llm::sigkill_sweep_covers_before_and_between_the_rung_b_steps ... ok +test parallel_lifecycle::terminal_failure_drains_or_explains_every_live_sibling ... ok +test parallel_lifecycle::overlapping_agent_conflict_survives_server_crash_and_resume ... ok +test pin_projection::rejected_completion_cannot_forge_inspect_retry_pins_over_the_real_socket ... ok +test placement::declared_placement_keeps_one_source_tree_across_resume ... ok +test parallel_lifecycle::overlapping_agent_lanes_serialize_while_disjoint_lanes_merge_in_either_order ... ok +test placement::sigkill_before_first_step_preserves_the_submitted_workspace ... ok +test protocol_admission::every_mutating_run_verb_refuses_terminal_before_changing_state ... ok +test sigkill_after_cancel_request_resumes_to_one_canceled_fact ... ok +test placement::sigkill_mid_step_keeps_the_route_and_source_tree ... ok +test sigkill_mid_step_replaces_and_explains_the_dead_attempt ... ok +test sigkill_under_serve_resumes_the_socket_started_run ... ok +test surface_identity::aliases_are_rejected_and_external_ancestors_serialize_over_real_sockets ... ok +test sigkill_sweep_covers_every_hello_step_boundary ... ok +test worker_capacity::two_workers_receive_a_deterministic_fair_capacity_bounded_batch ... ok +test worker_capacity::default_capacity_one_reopens_only_after_durable_completion_or_crash ... ok +test workspace_identity::workspace_aliases_are_refused_and_canonical_subtrees_serialize_over_real_sockets ... ok +test parallel_lifecycle::renewed_parallel_leases_survive_the_original_grant_and_remain_distinct ... ok + +test result: ok. 40 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 37.22s + + Running tests/daemon_lifecycle.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/daemon_lifecycle-b705da9761b2a254) + +running 6 tests +test clean_shutdown_removes_advertisement_and_socket ... ok +test connection_file_is_published_only_after_the_socket_is_live ... ok +test a_second_serve_on_a_served_data_dir_refuses_and_the_first_keeps_serving ... ok +test a_sigkilled_daemons_successor_starts_cleanly ... ok +test sigkill_leaves_a_stale_file_with_a_dead_pid ... ok +test deep_data_dir_still_binds ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/event_wake.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/event_wake-b98778c1ad4de872) + +running 3 tests +test two_racing_deliveries_of_one_event_produce_exactly_one_run ... ok +test matching_event_wakes_once_with_fresh_context ... ok +test a_resumed_run_dispatches_the_original_wake_context ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/hn_monitor_integration.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/hn_monitor_integration-059c38eb4828898f) + +running 1 test +test hn_story_event_wakes_monitor_once_with_story_context ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + + Running tests/input_binding.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/input_binding-4f132e6302508de8) + +running 2 tests +test binding_schema_is_additive_and_fails_closed ... ok +test sigkill_before_consumer_resolves_original_journal_output_without_reexecuting_source ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.10s + + Running tests/invalid_schema_preflight.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/invalid_schema_preflight-9c08869ea58d283f) + +running 3 tests +test invalid_json_schema_is_refused_before_journal_or_command ... ok +test unbounded_json_schema_is_refused_before_journal_or_command ... ok +test legitimately_recursive_json_schema_still_starts ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 3.66s + + Running tests/memoization.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/memoization-e5537edfeea8b814) + +running 3 tests +test refuses_missing_wrong_flow_and_unreadable_journal_before_creating_run ... ok +test reused_prefix_survives_restart_without_source_and_never_mutates_prior ... ok +test actual_changed_input_invalidates_consumer_even_with_identical_consumer_spec ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.08s + + Running tests/memory.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/memory-fe5dbc6738ee15ff) + +running 5 tests +test rejected_journal_fact_releases_reservation_and_never_dispatches ... ok +test llm_dispatch_receives_same_pack_after_resume_without_provider ... ok +test replay_and_resume_need_no_provider_and_script_receives_recorded_pack ... ok +test over_budget_and_provider_errors_fail_without_dispatch_or_charge ... ok +test semantic_retry_reuses_memory_without_a_second_charge ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/memory_epoch.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/memory_epoch-627e50d1c86c39e4) + +running 1 test +test epoch_carries_pack_and_exact_charge_and_refuses_duplicate_injection ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/parallel_driver.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/parallel_driver-7a554ae88aa529f9) + +running 4 tests +test stop_after_one_holds_for_an_independent_deterministic_batch ... ok +test pause_before_second_independent_step_holds_the_driver_boundary ... ok +test backpressured_or_mismatched_lane_does_not_drop_a_later_dispatch ... ok +test crash_boundaries_resume_the_real_driver_with_one_effect_per_lane ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.09s + + Running tests/placement_pins.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/placement_pins-839e136752b4e5af) + +running 3 tests +test unsupported_local_pty_is_refused_before_an_earlier_step_can_run ... ok +test default_worker_pins_the_declared_worktree_base_commit_and_refuses_missing_source ... ok +test a_resumed_attempt_keeps_the_original_pin_after_the_worktree_head_moves ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/placement_routing.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/placement_routing-9cb694324a7d7bf1) + +running 3 tests +test a_failed_routing_append_never_starts_or_dispatches_work ... ok +test crash_between_routing_and_start_does_not_redecide ... ok +test worker_retry_consumes_the_original_routing_fact ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/routing_diagnostics.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/routing_diagnostics-7807262e1a1167ec) + +running 2 tests +test duplicate_routes_have_a_distinct_diagnostic_and_leave_the_original_fact_intact ... ok +test malformed_routes_name_the_same_field_at_append_replay_and_epoch_replay ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/spec_review_routing.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/spec_review_routing-032df51688a72750) + +running 4 tests +test attempt_scoped_route_is_rejected_at_append_and_replay ... ok +test malformed_epoch_routes_are_rejected_before_commit ... ok +test epoch_cannot_drop_or_replace_a_durable_route ... ok +test workspace_pin_peels_tags_and_refuses_non_commit_objects ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/subscription_liveness.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/subscription_liveness-c3bbfd58dcafb336) + +running 3 tests +test submit_event_upserts_subscription_row_and_sweep_flags_it_stale_after_budget ... ok +test stale_transition_is_journaled_as_subscription_stale_entry_in_the_last_known_run ... ok +test a_fresh_arrival_re_arms_the_latch_and_the_next_silence_can_stale_again ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/trigger_watcher.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/trigger_watcher-0504d7ed791030e6) + +running 3 tests +test retains_bad_and_unregistered_events_while_consuming_filter_nonmatches ... ok +test failed_archive_retries_the_same_durable_run ... ok +test journals_payload_and_filename_key_then_archives_and_dedupes_replay ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running unittests src/lib.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/relayflowd_core-e734d1d7b8cb7b12) + +running 73 tests +test channel::tests::send_retry_is_stable_and_conflicting_content_is_rejected ... ok +test channel::tests::forged_deliveries_and_acknowledgements_fail_closed ... ok +test channel::tests::malformed_payloads_and_invalid_new_channel_appends_leave_state_unchanged ... ok +test channel::tests::delivery_replay_and_independent_acknowledged_offsets ... ok +test clock::tests::simulated_clock_is_explicitly_advanced ... ok +test entry::completion_reason_tests::every_journal_label_matches_serialized ... ok +test entry::completion_reason_tests::all_covers_every_serialized_label ... ok +test journal::tests::memory_journal_assigns_sequences_and_rolls_epochs ... ok +test machine::parallel_tests::every_declared_mutable_surface_participates_in_conflict_selection ... ok +test machine::parallel_tests::external_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::parallel_tests::crash_resume_preserves_each_parallel_lease_exactly_once ... ok +test machine::parallel_tests::disjoint_agent_lanes_merge_pins_in_either_completion_order ... ok +test machine::parallel_tests::machine_starts_every_runnable_step_in_authored_order ... ok +test machine::parallel_tests::failed_run_drains_open_siblings_before_terminal_entry ... ok +test machine::parallel_tests::overlapping_agent_surfaces_are_serialized_in_authored_order ... ok +test machine::parallel_tests::parallel_lanes_do_not_cross_the_dependency_barrier_early ... ok +test machine::tests::all_backing_off_steps_return_timers ... ok +test machine::tests::cancel_request_closes_the_active_lease_before_the_terminal_fact ... ok +test machine::parallel_tests::workspace_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::tests::deterministic_lease_rejects_invalid_and_foreign_fields ... ok +test machine::tests::crashed_attempt_does_not_consume_an_iteration ... ok +test machine::tests::deterministic_lease_override_and_default_are_journaled ... ok +test machine::tests::every_reason_label_matches_its_serialized_form ... ok +test machine::tests::durable_cancel_request_outranks_crash_recovery ... ok +test machine::tests::failed_deterministic_completion_preserves_exit_code_and_stderr ... ok +test machine::tests::machine_starts_runnable_step_with_stable_effect_key ... ok +test machine::tests::inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail ... ok +test machine::tests::every_failed_run_terminates_with_declared_completion_reasons ... ok +test machine::tests::repeated_cancel_request_is_idempotent ... ok +test machine::tests::manual_recovery_parks_needs_human_and_never_redispatches ... ok +test machine::tests::successful_memo_is_never_scheduled_again ... ok +test machine::tests::verification_failure_schedules_a_durable_retry ... ok +test machine::tests::worker_reported_failure_without_detail_still_records_a_verification ... ok +test memory::tests::caps_compare_exact_decimals_and_each_token_dimension ... ok +test machine::tests::reset_recovery_dispatches_the_original_pinned_revision ... ok +test retry::tests::jitter_is_repeatable_and_bounded ... ok +test schema::tests::a_property_named_ref_is_not_a_reference ... ok +test schema::tests::in_document_uri_references_resolve_to_the_node_they_name ... ok +test schema::tests::references_the_bound_leaves_opaque_are_refused_by_the_engine ... ok +test schema::tests::refusal_names_the_cycle_it_found ... ok +test schema::tests::shared_declarations_and_boolean_schemas_are_validated ... ok +test spec::tests::a_misspelled_step_level_key_is_a_parse_error ... ok +test spec::tests::a_misspelled_verification_gate_key_is_a_parse_error_not_a_dropped_gate ... ok +test spec::tests::agent_cwd_is_carried_and_must_be_run_root_relative ... ok +test spec::tests::agent_without_cwd_hashes_as_before ... ok +test spec::tests::cycles_are_rejected ... ok +test spec::tests::external_surface_paths_must_have_one_canonical_spelling ... ok +test spec::tests::on_non_zero_parses_the_two_declared_policies_and_refuses_anything_else ... ok +test spec::tests::preflight_data_is_fail_closed ... ok +test schema::tests::every_accepted_corpus_schema_is_accepted ... ok +test schema::tests::every_refused_corpus_schema_compiles_but_is_refused_by_the_bound ... ok +test spec::tests::spec_version_is_semver_and_gated ... ok +test spec::tests::the_default_policy_is_absent_from_the_serialized_boundary_spec ... ok +test spec::tests::the_full_ladder_parses_in_the_one_dialect ... ok +test spec::tests::unknown_root_and_nested_fields_are_rejected ... ok +test spec::tests::workspace_mounts_and_worktrees_must_have_one_canonical_spelling ... ok +test spec::tests::zero_agent_flow_is_valid ... ok +test state::budget::tests::adds_costs_exactly_beyond_machine_decimal_precision ... ok +test state::budget::tests::overflow_and_malformed_cost_leave_total_unchanged ... ok +test state::tests::a_completion_that_omits_a_surface_does_not_drop_it_from_the_pin_chain ... ok +test state::tests::budget_decimal_strings_add_without_floats ... ok +test state::tests::end_pin_chain_is_enforced_and_a_broken_chain_is_a_hard_error ... ok +test state::tests::journal_replays_data_gate_verdict_without_rerunning_completed_code ... ok +test verify::tests::a_recorded_nonzero_exit_passes_its_gate_and_names_the_code ... ok +test verify::tests::an_unbounded_schema_in_a_journal_fails_its_gate_instead_of_aborting ... ok +test verify::tests::deterministic_output_requires_successful_exit_and_content ... ok +test verify::tests::json_schema_is_a_control_gate ... ok +test verify::tests::recording_an_exit_code_does_not_relax_a_declared_content_gate ... ok +test verify::tests::recording_does_not_absorb_a_missing_or_sentinel_exit_status ... ok +test verify::tests::the_default_policy_still_fails_a_nonzero_exit ... ok +test spec::tests::sdk_boundary_accepts_a_valid_10_000_step_reverse_chain ... ok +test spec::tests::sdk_boundary_rejects_a_10_000_step_cycle_with_a_typed_error ... ok +test schema::tests::deeply_nested_schemas_do_not_overflow_the_checker ... ok + +test result: ok. 73 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.75s + + Running tests/memoization.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/memoization-79f552e4cb727501) + +running 4 tests +test distinct_large_kernel_integers_do_not_alias_through_float_rounding ... ok +test match_reuses_output_with_provenance_and_zero_cost_without_dispatch ... ok +test changed_spec_or_input_dispatches_and_legacy_or_failed_records_miss ... ok +test canonical_corpus_agrees_with_typescript_and_key_permutations ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/spec_parity.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/spec_parity-a9966affd5aca87f) + +running 16 tests +test agent_cwd_is_not_accepted_on_other_verbs_or_under_another_name ... ok +test a_non_string_agent_cwd_fails_closed_rather_than_defaulting ... ok +test agent_working_directories_have_identical_canonical_bytes_and_hash ... ok +test agent_cwd_declaration_acceptance_matches_the_sdk_corpus ... ok +test an_undeclared_agent_cwd_is_not_serialized ... ok +test placement_requirements_have_identical_canonical_bytes_and_hash ... ok +test step_memory_has_identical_canonical_bytes_and_hash ... ok +test the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash ... ok +test memory_declaration_acceptance_matches_the_sdk_corpus ... ok +test the_kernel_parses_the_event_triggered_spec_and_stamps_the_same_hash ... ok +test placement_declaration_acceptance_matches_the_sdk_corpus ... ok +test the_kernel_parses_the_rung_c_agent_spec_and_stamps_the_same_hash ... ok +test the_kernel_round_trips_declared_agent_transports_and_rejects_unknown_values ... ok +test the_kernel_parses_the_sdk_compiled_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_b_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_advisory_repair_spec_and_stamps_the_same_hash ... ok + +test result: ok. 16 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running unittests src/lib.rs (/home/daytona/.relayflows-toolchain/target/2962130851/debug/deps/relayflowd_journal-0286edd157fee7d6) + +running 32 tests +test channel::tests::stale_attempts_and_raw_forged_acknowledgements_cannot_change_offsets ... ok +test channel::tests::channels_cross_segment_boundaries_and_terminal_runs_reject_mutations ... ok +test registry::tests::a_pre_migration_registry_gains_boot_id_and_its_claims_are_repairable ... ok +test channel::tests::failed_channel_writes_never_expose_delivery_or_advance_acknowledged_offset ... ok +test registry::tests::a_previous_boots_claim_with_no_run_is_repaired ... ok +test registry::tests::a_registered_run_dedupes_across_boots ... ok +test registry::tests::a_same_boot_claim_with_no_run_yet_is_a_duplicate_not_wreckage ... ok +test registry::tests::concurrent_new_boot_retries_have_one_recovery_owner ... ok +test registry::tests::prior_boot_unregistered_run_admission_is_repaired ... ok +test registry::tests::concurrent_same_boot_run_admissions_have_one_owner ... ok +test registry::tests::registry_is_a_rebuildable_run_locator ... ok +test registry::tests::releasing_is_scoped_to_the_claiming_run ... ok +test registry::tests::releasing_a_claim_lets_the_same_boot_retry ... ok +test registry::tests::run_admission_reuses_registered_run_and_rejects_spec_drift ... ok +test subscriptions::tests::last_run_for_subscription_returns_none_before_first_arrival ... ok +test subscriptions::tests::detect_without_latch_stays_available_for_the_next_sweep ... ok +test subscriptions::tests::last_run_for_subscription_returns_the_lex_greatest_ulid_regardless_of_insertion ... ok +test subscriptions::tests::prune_sweep_claims_deletes_only_rows_older_than_cutoff ... ok +test subscriptions::tests::latch_is_a_no_op_if_a_fresh_event_arrived_between_detect_and_latch ... ok +test subscriptions::tests::sweep_does_not_re_emit_the_same_stale_row_on_a_later_tick ... ok +test subscriptions::tests::sweep_ignores_subscriptions_whose_silence_is_still_within_budget ... ok +test subscriptions::tests::sweep_election_gives_the_first_caller_the_result_and_second_gets_empty ... ok +test subscriptions::tests::sweep_marks_row_stale_when_silence_exceeds_budget ... ok +test subscriptions::tests::upsert_after_stale_re_arms_and_next_silence_can_re_emit ... ok +test subscriptions::tests::upsert_is_idempotent_across_bumps_and_preserves_event_type_updates ... ok +test tests::append_is_durable_and_monotonic_after_reopen ... ok +test tests::an_unconfirmed_election_is_reclaimed_by_the_next_attempt_not_treated_as_done ... ok +test tests::effects_are_deduplicated_at_the_journal_boundary ... ok +test tests::failed_commit_is_returned_not_swallowed ... ok +test tests::rollover_is_atomic_scaffolding_for_epoch_resume ... ok +test tests::terminal_run_refuses_every_later_entry_atomically ... ok +test channel::tests::independent_connections_serialize_send_receive_and_acknowledgement ... ok + +test result: ok. 32 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.26s + + Doc-tests relayflowd + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_core + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_journal + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + diff --git a/evidence/561/live-default.txt b/evidence/561/live-default.txt new file mode 100644 index 00000000..816006af --- /dev/null +++ b/evidence/561/live-default.txt @@ -0,0 +1,41 @@ +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.02s completionReason: success +○ llm-2 (llm) 0.00s +○ llm-3 (llm) 0.00s +○ llm-4 (llm) 0.00s +○ llm-5 (llm) 0.00s +○ llm-6 (llm) 0.00s +○ llm-7 (llm) 0.00s +○ llm-8 (llm) 0.00s +○ llm-9 (llm) 0.00s +○ llm-10 (llm) 0.00s +WAITING [worker_lease] Run "01M390T5TRKMCHJN0MZBG754TK" step "llm-2" (llm) is running under a worker lease until 1790230498816. +↻ llm-2 (llm) 4.53s +WAITING [worker_lease] Run "01M390T5VVXW1VFV7G94XJZTVR" step "llm-3" (llm) is running under a worker lease until 1790230498820. +↻ llm-3 (llm) 0.45s +WAITING [worker_lease] Run "01M390T5XAS9XMRPG2WMCRHGC7" step "llm-4" (llm) is running under a worker lease until 1790230498823. +↻ llm-4 (llm) 0.38s +WAITING [worker_lease] Run "01M390T5ZBXH88ZB6AWXX8EWVR" step "llm-5" (llm) is running under a worker lease until 1790230498825. +↻ llm-5 (llm) 0.34s +✓ llm-3 (llm) 3.20s completionReason: success +WAITING [worker_lease] Run "01M390T8XN5HMWM4FJAAY0KG8F" step "llm-6" (llm) is running under a worker lease until 1790230501615. +↻ llm-6 (llm) 3.08s +✓ llm-2 (llm) 7.49s completionReason: success +WAITING [worker_lease] Run "01M390T94D0MRH9XBJX81XEA8V" step "llm-7" (llm) is running under a worker lease until 1790230501831. +↻ llm-7 (llm) 3.22s +✓ llm-5 (llm) 3.55s completionReason: success +✓ llm-4 (llm) 3.60s completionReason: success +WAITING [worker_lease] Run "01M390T9CE328ZGT9R42D4M8KE" step "llm-8" (llm) is running under a worker lease until 1790230502085. +↻ llm-8 (llm) 3.42s +WAITING [worker_lease] Run "01M390T9CQ4S3G5E9A57T1MKCG" step "llm-9" (llm) is running under a worker lease until 1790230502094. +↻ llm-9 (llm) 3.39s +✓ llm-6 (llm) 5.78s completionReason: success +WAITING [worker_lease] Run "01M390TBKZ311HSG7S8G914VB9" step "llm-10" (llm) is running under a worker lease until 1790230504374. +↻ llm-10 (llm) 5.60s +✓ llm-8 (llm) 5.98s completionReason: success +✓ llm-9 (llm) 6.06s completionReason: success +✓ llm-7 (llm) 6.17s completionReason: success +✓ llm-10 (llm) 8.44s completionReason: success +○ run-11 (deterministic) 0.00s +✓ run-11 (deterministic) 0.04s completionReason: success +RUN 01M390T1RQGKTMT336054C6WK0 completed (12 steps) completionReason: success diff --git a/evidence/561/live-journals.txt b/evidence/561/live-journals.txt new file mode 100644 index 00000000..2fb01876 --- /dev/null +++ b/evidence/561/live-journals.txt @@ -0,0 +1,20 @@ +llm-2 {'x': 4} +llm-3 {'x': 5} +llm-4 {'x': 6} +llm-5 {'x': 7} +llm-6 {'x': 8} +llm-7 {'x': 9} +llm-8 {'x': 10} +llm-9 {'x': 11} +llm-10 {'x': 12} +/tmp/flows-561-live-default 13 journals, 9 LLM children, one attempt each, no lease_expired +llm-2 {'x': 4} +llm-3 {'x': 5} +llm-4 {'x': 6} +llm-5 {'x': 7} +llm-6 {'x': 8} +llm-7 {'x': 9} +llm-8 {'x': 10} +llm-9 {'x': 11} +llm-10 {'x': 12} +/tmp/flows-561-live-one 13 journals, 9 LLM children, one attempt each, no lease_expired diff --git a/evidence/561/live-one.txt b/evidence/561/live-one.txt new file mode 100644 index 00000000..6158d3f3 --- /dev/null +++ b/evidence/561/live-one.txt @@ -0,0 +1,41 @@ +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.02s completionReason: success +○ llm-2 (llm) 0.00s +○ llm-3 (llm) 0.00s +○ llm-4 (llm) 0.00s +○ llm-5 (llm) 0.00s +○ llm-6 (llm) 0.00s +○ llm-7 (llm) 0.00s +○ llm-8 (llm) 0.00s +○ llm-9 (llm) 0.00s +○ llm-10 (llm) 0.00s +WAITING [worker_lease] Run "01M390VD1WQ6037RPAC4X0HR4M" step "llm-2" (llm) is running under a worker lease until 1790230538824. +↻ llm-2 (llm) 2.39s +✓ llm-2 (llm) 5.05s completionReason: success +WAITING [worker_lease] Run "01M390VFW4Y1K27XZF8SK0GPA6" step "llm-3" (llm) is running under a worker lease until 1790230541500. +↻ llm-3 (llm) 2.96s +✓ llm-3 (llm) 4.90s completionReason: success +WAITING [worker_lease] Run "01M390VHSDC8E63DSF4G3KWGX3" step "llm-4" (llm) is running under a worker lease until 1790230543462. +↻ llm-4 (llm) 4.88s +✓ llm-4 (llm) 7.85s completionReason: success +WAITING [worker_lease] Run "01M390VMPVM5W37PV1QM4F3PDJ" step "llm-5" (llm) is running under a worker lease until 1790230546451. +↻ llm-5 (llm) 7.82s +✓ llm-5 (llm) 10.33s completionReason: success +WAITING [worker_lease] Run "01M390VQ5ZZ1CTGF7B7BVPW9Y4" step "llm-6" (llm) is running under a worker lease until 1790230548987. +↻ llm-6 (llm) 10.32s +✓ llm-6 (llm) 12.93s completionReason: success +WAITING [worker_lease] Run "01M390VSR1QZ2KK9CBRWCBMW56" step "llm-7" (llm) is running under a worker lease until 1790230551608. +↻ llm-7 (llm) 12.91s +✓ llm-7 (llm) 15.34s completionReason: success +WAITING [worker_lease] Run "01M390VW4BSDDECPRQY9VYFS3T" step "llm-8" (llm) is running under a worker lease until 1790230554057. +↻ llm-8 (llm) 15.34s +✓ llm-8 (llm) 18.01s completionReason: success +WAITING [worker_lease] Run "01M390VYRTXDVSBXDW263BQCBX" step "llm-9" (llm) is running under a worker lease until 1790230556752. +↻ llm-9 (llm) 18.00s +✓ llm-9 (llm) 20.59s completionReason: success +WAITING [worker_lease] Run "01M390W1AFWDJVGY1QBG11QERA" step "llm-10" (llm) is running under a worker lease until 1790230559366. +↻ llm-10 (llm) 20.58s +✓ llm-10 (llm) 22.67s completionReason: success +○ run-11 (deterministic) 0.00s +✓ run-11 (deterministic) 0.03s completionReason: success +RUN 01M390VAWN5D0R3NWNPE2BJF68 completed (12 steps) completionReason: success diff --git a/evidence/561/mutation-green.txt b/evidence/561/mutation-green.txt new file mode 100644 index 00000000..7ace727f --- /dev/null +++ b/evidence/561/mutation-green.txt @@ -0,0 +1,15 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-probe-cache.test.ts (5 tests) 20833ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 1 7158ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 4 5367ms + ✓ authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 6617ms + ✓ authored run CLI probe cache > caches probe failures while refusing all nine calls 398ms + ✓ authored run CLI probe cache > shares probe results across agent calls too 1292ms + + Test Files 1 passed (1) + Tests 5 passed (5) + Start at 06:20:05 + Duration 22.42s (transform 858ms, setup 0ms, collect 1.40s, tests 20.83s, environment 0ms, prepare 43ms) + diff --git a/evidence/561/mutation-red.txt b/evidence/561/mutation-red.txt new file mode 100644 index 00000000..e3c8d9fe --- /dev/null +++ b/evidence/561/mutation-red.txt @@ -0,0 +1,109 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +(node:40764) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M391141161E357A40MBYPN3A step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M391141161E357A40MBYPN3A/llm-1. +(Use `node --trace-warnings ...` to show where the warning was created) +(node:40764) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M39128M2NQ6ATSY779PXPJVP step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M39128M2NQ6ATSY779PXPJVP/llm-1. + ❯ tests/authored-probe-cache.test.ts (5 tests | 5 failed) 87312ms + × authored run CLI probe cache > nine calls have no expired attempts at capacity 1 37509ms + → 01M391141161E357A40MBYPN3A.sqlite3: expected '[{"at_ms":1790230695973,"attempt":nul…' not to contain 'lease_expired' + × authored run CLI probe cache > nine calls have no expired attempts at capacity 4 37935ms + → 01M39128M2NQ6ATSY779PXPJVP.sqlite3: expected '[{"at_ms":1790230733446,"attempt":nul…' not to contain 'lease_expired' + × authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 6548ms + → expected 3340.2447240000038 to be less than 1000 + × authored run CLI probe cache > caches probe failures while refusing all nine calls 3300ms + → expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9 + × authored run CLI probe cache > shares probe results across agent calls too 2018ms + → expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 5 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 1 +AssertionError: 01M391141161E357A40MBYPN3A.sqlite3: expected '[{"at_ms":1790230695973,"attempt":nul…' not to contain 'lease_expired' + +Expected: "lease_expired" +Received: "[{"at_ms":1790230695973,"attempt":null,"entry_type":"run.spawned","payload":{"created_by":"protocol-v0","journal_version":1,"parent_run_id":null,"spec":{"name":"nine-llms/llm-1","steps":[{"cli":"/tmp/flows-chain-niZO3d/adapter.mjs","depends_on":[],"id":"llm-1","max_iterations":1,"model":"test-model","prompt":"Return JSON for 0","retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"llm","verification":{"json_schema":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}}}],"version":"0.1.0"},"spec_hash":"c305c1eca299e57e82538e0d90c3cf298cfd7e23d0dce51b6ed3f18b3b4044be"},"run_id":"01M391141161E357A40MBYPN3A","segment_id":1,"seq":1,"step_id":null},{"at_ms":1790230695977,"attempt":null,"entry_type":"step.routed","payload":{"fallbacks_attempted":[],"profile":"attached-worker","provider":"worker"},"run_id":"01M391141161E357A40MBYPN3A","segment_id":1,"seq":2,"step_id":"llm-1"},{"at_ms":1790230695977,"attempt":1,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-5aa55ed4-158a-48e8-b824-cefda6262a22-llm","idempotency_key":"c7efa3a49306bed647bf9b905c6dacc3ef3d5572011ed962679f0f4588307983","lease_deadline_ms":1790230725977,"lease_id":"01M3911419QSC2TBW8EY121M8Q","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M391141161E357A40MBYPN3A","segment_id":1,"seq":3,"step_id":"llm-1"},{"at_ms":1790230726015,"attempt":1,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","tokens_in":0,"tokens_out":0},"completed_by":"kernel","completionReason":"lease_expired","disposition":"retry","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":1790230726015,"output":null,"spend":{"dollars":0,"tokens_input":0,"tokens_output":0,"wallclock_ms":30038},"step_spec_hash":"51478a00df69af7604bc32ac9e6e6e50a1c627bea9728ebf337c58ce6c6fd487","verification":null},"run_id":"01M391141161E357A40MBYPN3A","segment_id":1,"seq":4,"step_id":"llm-1"},{"at_ms":1790230726015,"attempt":1,"entry_type":"sleep.until","payload":{"reason":"retry_backoff","wait_id":"01M39121BZRBF3ND2YS8TJX1FM","wake_at_ms":1790230726015},"run_id":"01M391141161E357A40MBYPN3A","segment_id":1,"seq":5,"step_id":"llm-1"},{"at_ms":1790230726023,"attempt":1,"entry_type":"wait.completed","payload":{"completionReason":"timer_fired","result":null,"wait_id":"01M39121BZRBF3ND2YS8TJX1FM"},"run_id":"01M391141161E357A40MBYPN3A","segment_id":1,"seq":6,"step_id":"llm-1"},{"at_ms":1790230726024,"attempt":2,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-5aa55ed4-158a-48e8-b824-cefda6262a22-llm","idempotency_key":"c7efa3a49306bed647bf9b905c6dacc3ef3d5572011ed962679f0f4588307983","lease_deadline_ms":1790230756024,"lease_id":"01M39121C8SDT36NE3KZVWBZFD","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M391141161E357A40MBYPN3A","segment_id":1,"seq":7,"step_id":"llm-1"},{"at_ms":1790230729145,"attempt":2,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completed_by":"local-agent-5aa55ed4-158a-48e8-b824-cefda6262a22-llm","completionReason":"success","disposition":"step_done","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":null,"output":{"x":4},"spend":{"dollars":0,"dollars_unmetered":true,"tokens_input":0,"tokens_output":0,"wallclock_ms":3121},"step_spec_hash":"51478a00df69af7604bc32ac9e6e6e50a1c627bea9728ebf337c58ce6c6fd487","verification":{"detail":"all gates passed","gate":"json_schema","verdict":"pass"}},"run_id":"01M391141161E357A40MBYPN3A","segment_id":1,"seq":8,"step_id":"llm-1"},{"at_ms":1790230729148,"attempt":null,"entry_type":"run.completed","payload":{"budget_total":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completionReason":"success","failed_step_id":null},"run_id":"01M391141161E357A40MBYPN3A","segment_id":1,"seq":9,"step_id":null}]" + + ❯ tests/authored-probe-cache.test.ts:82:49 + 80| for (const file of runs) { + 81| const { entries } = await client.journalRead(file.slice(0, -8), … + 82| expect(JSON.stringify(entries), file).not.toContain('lease_expir… + | ^ + 83| expect(entries.filter(e => typeof e === 'object' && e !== null + 84| && 'entry_type' in e && e.entry_type === 'step.attempt.started… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 4 +AssertionError: 01M39128M2NQ6ATSY779PXPJVP.sqlite3: expected '[{"at_ms":1790230733446,"attempt":nul…' not to contain 'lease_expired' + +Expected: "lease_expired" +Received: "[{"at_ms":1790230733446,"attempt":null,"entry_type":"run.spawned","payload":{"created_by":"protocol-v0","journal_version":1,"parent_run_id":null,"spec":{"name":"nine-llms/llm-1","steps":[{"cli":"/tmp/flows-chain-NRjmsX/adapter.mjs","depends_on":[],"id":"llm-1","max_iterations":1,"model":"test-model","prompt":"Return JSON for 0","retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"llm","verification":{"json_schema":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}}}],"version":"0.1.0"},"spec_hash":"07c29d761706e56ddaec92ab6a4f9d14dc2908bf1ed33a23f0974fc3b9e0b042"},"run_id":"01M39128M2NQ6ATSY779PXPJVP","segment_id":1,"seq":1,"step_id":null},{"at_ms":1790230733449,"attempt":null,"entry_type":"step.routed","payload":{"fallbacks_attempted":[],"profile":"attached-worker","provider":"worker"},"run_id":"01M39128M2NQ6ATSY779PXPJVP","segment_id":1,"seq":2,"step_id":"llm-1"},{"at_ms":1790230733449,"attempt":1,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-7c2039e8-ba49-4acd-8ce8-b2f8630149a3-llm","idempotency_key":"adab9f30063e09a1f304a04de8171072f287ce64b49293c83ffdd9deffd2d795","lease_deadline_ms":1790230763449,"lease_id":"01M39128M933BEVH37JZN7Y9P7","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M39128M2NQ6ATSY779PXPJVP","segment_id":1,"seq":3,"step_id":"llm-1"},{"at_ms":1790230763516,"attempt":1,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","tokens_in":0,"tokens_out":0},"completed_by":"kernel","completionReason":"lease_expired","disposition":"retry","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":1790230763516,"output":null,"spend":{"dollars":0,"tokens_input":0,"tokens_output":0,"wallclock_ms":30067},"step_spec_hash":"5941f015b8f3c5e1e634001e6db17b6da03ff4369604ecd7a5e3ec65e983c441","verification":null},"run_id":"01M39128M2NQ6ATSY779PXPJVP","segment_id":1,"seq":4,"step_id":"llm-1"},{"at_ms":1790230763516,"attempt":1,"entry_type":"sleep.until","payload":{"reason":"retry_backoff","wait_id":"01M39135ZWXK4XHGM47SQS7FHX","wake_at_ms":1790230763516},"run_id":"01M39128M2NQ6ATSY779PXPJVP","segment_id":1,"seq":5,"step_id":"llm-1"},{"at_ms":1790230763532,"attempt":1,"entry_type":"wait.completed","payload":{"completionReason":"timer_fired","result":null,"wait_id":"01M39135ZWXK4XHGM47SQS7FHX"},"run_id":"01M39128M2NQ6ATSY779PXPJVP","segment_id":1,"seq":6,"step_id":"llm-1"},{"at_ms":1790230763533,"attempt":2,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-7c2039e8-ba49-4acd-8ce8-b2f8630149a3-llm","idempotency_key":"adab9f30063e09a1f304a04de8171072f287ce64b49293c83ffdd9deffd2d795","lease_deadline_ms":1790230793533,"lease_id":"01M391360D3KAP5XF0CNFFHFB1","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M39128M2NQ6ATSY779PXPJVP","segment_id":1,"seq":7,"step_id":"llm-1"},{"at_ms":1790230766692,"attempt":2,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completed_by":"local-agent-7c2039e8-ba49-4acd-8ce8-b2f8630149a3-llm","completionReason":"success","disposition":"step_done","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":null,"output":{"x":4},"spend":{"dollars":0,"dollars_unmetered":true,"tokens_input":0,"tokens_output":0,"wallclock_ms":3159},"step_spec_hash":"5941f015b8f3c5e1e634001e6db17b6da03ff4369604ecd7a5e3ec65e983c441","verification":{"detail":"all gates passed","gate":"json_schema","verdict":"pass"}},"run_id":"01M39128M2NQ6ATSY779PXPJVP","segment_id":1,"seq":8,"step_id":"llm-1"},{"at_ms":1790230766698,"attempt":null,"entry_type":"run.completed","payload":{"budget_total":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completionReason":"success","failed_step_id":null},"run_id":"01M39128M2NQ6ATSY779PXPJVP","segment_id":1,"seq":9,"step_id":null}]" + + ❯ tests/authored-probe-cache.test.ts:82:49 + 80| for (const file of runs) { + 81| const { entries } = await client.journalRead(file.slice(0, -8), … + 82| expect(JSON.stringify(entries), file).not.toContain('lease_expir… + | ^ + 83| expect(entries.filter(e => typeof e === 'object' && e !== null + 84| && 'entry_type' in e && e.entry_type === 'step.attempt.started… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run +AssertionError: expected 3340.2447240000038 to be less than 1000 + ❯ tests/authored-probe-cache.test.ts:104:41 + 102| // F1 permits one synchronous probe (~300ms), not nine (~3s). + 103| // Leave process-startup headroom for loaded CI; exact counts belo… + 104| expect(starts.at(-1)! - starts[0]!).toBeLessThan(1_000); + | ^ + 105| expectOneProbe(logs()); + 106| await executeAuthoredFlow(nine, client, undefined, options); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > caches probe failures while refusing all nine calls +AssertionError: expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9 + +- Expected ++ Received + +- 1 ++ 9 + + ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47 + 68| + 69| function expectOneProbe(logs: ReturnType l.kind === 'auth')).toHaveLength(1); + | ^ + 71| expect(logs.filter(l => l.kind === 'identify').length - logs.filter(… + 72| } + ❯ tests/authored-probe-cache.test.ts:123:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > shares probe results across agent calls too +AssertionError: expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3 + +- Expected ++ Received + +- 1 ++ 3 + + ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47 + 68| + 69| function expectOneProbe(logs: ReturnType l.kind === 'auth')).toHaveLength(1); + | ^ + 71| expect(logs.filter(l => l.kind === 'identify').length - logs.filter(… + 72| } + ❯ tests/authored-probe-cache.test.ts:136:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/5]⎯ + + Test Files 1 failed (1) + Tests 5 failed (5) + Start at 06:18:09 + Duration 89.27s (transform 1.07s, setup 0ms, collect 1.78s, tests 87.31s, environment 0ms, prepare 44ms) + diff --git a/evidence/561/mutation.patch b/evidence/561/mutation.patch new file mode 100644 index 00000000..1c075699 --- /dev/null +++ b/evidence/561/mutation.patch @@ -0,0 +1,13 @@ +diff --git a/packages/sdk/src/authored-worker-step.ts b/packages/sdk/src/authored-worker-step.ts +index 8f14a6f..b3dcf65 100644 +--- a/packages/sdk/src/authored-worker-step.ts ++++ b/packages/sdk/src/authored-worker-step.ts +@@ -49,7 +49,7 @@ export function authoredWorkerRunner( + // and real-probing auth/model readiness. An authored agent step gets + // nothing for free just because it was declared in TS instead of YAML. + const { report, flow: resolved } = checkAuthoredFlow(authoring, flowPath, +- projectConfig ??= readProjectConfig(dirname(resolve(flowPath))), {}, cliProbeCache); ++ projectConfig ??= readProjectConfig(dirname(resolve(flowPath))), {}); + if (!report.ok || resolved === undefined) { + const refusal = report.diagnostics.find( + (diagnostic): diagnostic is PreflightDiagnostic & { severity: 'refusal' } => diff --git a/evidence/561/package.json b/evidence/561/package.json new file mode 100644 index 00000000..089153bc --- /dev/null +++ b/evidence/561/package.json @@ -0,0 +1 @@ +{"type":"module"} diff --git a/evidence/561/prompt-lab.txt b/evidence/561/prompt-lab.txt new file mode 100644 index 00000000..59e1b038 --- /dev/null +++ b/evidence/561/prompt-lab.txt @@ -0,0 +1,2 @@ +examples/prompt-lab/jobs/shared.ts: absent +examples/prompt-lab/jobs/new-agency.ts: absent diff --git a/evidence/561/regression-typecheck.txt b/evidence/561/regression-typecheck.txt new file mode 100644 index 00000000..01fe490f --- /dev/null +++ b/evidence/561/regression-typecheck.txt @@ -0,0 +1 @@ +exit code: 0 diff --git a/evidence/561/restore.txt b/evidence/561/restore.txt new file mode 100644 index 00000000..022f75ba --- /dev/null +++ b/evidence/561/restore.txt @@ -0,0 +1,5 @@ +$ git diff --exit-code -- packages/sdk/src/authored-worker-step.ts +exit code: 0 +HEAD SHA256: b9cd463d9f1ee274175a0b659119262c0ba5b18f1afc11b866c3e0f7f34023b7 +Restored SHA256: b9cd463d9f1ee274175a0b659119262c0ba5b18f1afc11b866c3e0f7f34023b7 +byte-for-byte restore: identical diff --git a/evidence/561/runtime-parallel-llm-repro.flow.ts b/evidence/561/runtime-parallel-llm-repro.flow.ts new file mode 100644 index 00000000..7076e0c0 --- /dev/null +++ b/evidence/561/runtime-parallel-llm-repro.flow.ts @@ -0,0 +1,8 @@ +import { flow } from "@relayflows/surface"; +const s = { type: "object", required: ["x"], properties: { x: { type: "number" } } }; +export default flow<{ n: number }>("repro-par", async (f, input) => { + await f.run("echo start"); + const xs = await Promise.all([1, 2, 3, 4, 5, 6, 7, 8, 9].map((i) => f.llm(`Return {"x": ${i}+${input.n}} as JSON only.`, { output: s, model: "claude-haiku-4-5-20251001" }))); + await f.run(`echo ${JSON.stringify(JSON.stringify(xs))}`); + f.done("success"); +}); diff --git a/evidence/561/sdk-suite.txt b/evidence/561/sdk-suite.txt new file mode 100644 index 00000000..6a11a9d9 --- /dev/null +++ b/evidence/561/sdk-suite.txt @@ -0,0 +1,1373 @@ + +> @relayflows/sdk@2.0.30 test +> sh scripts/test.sh + + +> @relayflows/sdk@2.0.30 test:prep +> ( cd ../../kernel && sh ../ops/cargo.sh build ) && ( [ ! -d ../../testdata/preflight ] || find ../../testdata/preflight -name '*-cli' -type f -exec chmod +x {} + ) + + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.06s + +> @relayflows/sdk@2.0.30 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + + +> @relayflows/sdk@2.0.30 build +> tsc && node scripts/make-cli-executable.mjs + + +> @relayflows/sdk@2.0.30 typecheck:tests +> tsc -p tsconfig.tests.json + + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd +LIVE_KERNEL flows=/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js + + ✓ tests/preflight.test.ts (67 tests) 131ms + ✓ tests/cloud-read.test.ts (44 tests) 72ms + ✓ tests/cli.test.ts (70 tests) 1878ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 432ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 461ms + ❯ tests/hosted-extension-isolation.test.ts (22 tests | 14 failed) 302ms + × hosted extension capability isolation > executes the exact capability-only handler for a queued receipt 11ms + → bubblewrap is unavailable + × hosted extension capability isolation > executes the exact capability-only handler for a duplicate receipt 3ms + → bubblewrap is unavailable + × hosted extension capability isolation > launches through the captured process primitive after builtin export synchronization 7ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > ignores inherited launcher overrides and decodes manifests with the captured Buffer intrinsic 3ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > streams verified bytes when the live store is replaced and no writable staging path exists 214ms + → promise rejected "Error: Hosted extension sandbox exited wi… { code: '…' }" instead of resolving + × hosted extension capability isolation > mounts pinned private Surface bytes when the live package changes before launch 4ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > refuses oversized Surface files through the bounded descriptor reader 6ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + × hosted extension capability isolation > shields verified Surface files before async settlement 4ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > preserves a typed host refusal while disclosing only a fixed marker to the child 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to be Error: private Cloud policy detail { code: '…' } // Object.is equality + × hosted extension capability isolation > denies ambient credentials, host files, writes, network, subprocesses, and undeclared context verbs 7ms + → bubblewrap is unavailable + × hosted extension capability isolation > enforces OS address-space and data bounds on native Buffer allocation 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + × hosted extension capability isolation > blocks extra handler fields and authority-bearing receipt fields at the parent port 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension capability isolation > constructs adapter authority with the captured freeze intrinsic 2ms + → bubblewrap is unavailable + × hosted extension capability isolation > writes the Surface manifest and protocol without inherited toJSON behavior 2ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ✓ tests/cloud-live.test.ts (55 tests) 2073ms + ✓ argv and wiring > routes both live invocations through runCli, and refuses a missing run id 2005ms + ✓ tests/cloud-sync.test.ts (40 tests) 997ms + ✓ tests/plugin-extension.test.ts (90 tests) 444ms + ✓ tests/cloud-transcript-codex.test.ts (39 tests) 15ms + ✓ tests/observer-link.test.ts (39 tests) 143ms +(node:24746) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-flow.test.ts (34 tests) 796ms + ✓ tests/agent-transcript.test.ts (29 tests) 289ms + ✓ tests/cli-status.test.ts (27 tests) 1037ms + ✓ flows status > resolves the run with no arguments from inside a worker-spawned agent 789ms + ✓ tests/cloud-run.test.ts (58 tests) 726ms + ✓ tests/relay-cli-surface.test.ts (77 tests) 32ms + ✓ tests/babysitter-native-extension.test.ts (41 tests | 1 skipped) 1460ms + ✓ tests/authored-completion-detail.test.ts (59 tests) 148ms + ✓ tests/worker-cli.test.ts (18 tests) 26564ms + ✓ registered CLI model defaults > passes the same priced Claude default to the real provider invocation 444ms + ✓ step discovery environment > names the run, step, attempt and an absolute data dir for a direct agent spawn 462ms + ✓ step discovery environment > exports none of the four without a data dir, even when the worker inherited them 440ms + ✓ wrapper discovery environment > sets the four names from the dispatch and still refuses ambient values and other secrets 457ms + ✓ wrapper discovery environment > exports none of the four to a wrapper without a data dir, even when the worker inherited them 433ms + ✓ custom wrapper execution identity > passes an explicit safe environment at identification and execution 483ms + ✓ custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 379ms + ✓ custom wrapper execution identity > bounds wrapper execution after acknowledgement 480ms + ✓ custom wrapper execution identity > bounds captured wrapper output 425ms + ✓ custom wrapper execution identity > refuses a duplicate execute protocol frame 415ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 2047ms + ✓ custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 2013ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3674ms + ✓ custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11709ms + ✓ custom wrapper execution bounds are reader-owned > accepts an execute token and an over-8KiB payload flushed in one write 451ms + ✓ custom wrapper execution bounds are reader-owned > accepts the same over-8KiB payload whether or not it coalesces with the execute token 1362ms + ✓ custom wrapper execution bounds are reader-owned > still bounds an un-terminated handshake buffer and names the bound 458ms + ✓ delivers the journaled memory pack to the real wrapper and excludes its charge from completion usage 428ms + ✓ tests/authored-root.test.ts (20 tests) 167ms + ✓ tests/step-failure-diagnostic.test.ts (25 tests) 59ms + ✓ tests/daemon-lifecycle.test.ts (42 tests) 40ms + ❯ tests/hosted-extension-protocol.test.ts (24 tests | 8 failed) 40097ms + × hosted extension hostile protocol > uses captured JSON intrinsics for the complete parent boundary 6ms + → bubblewrap is unavailable + × hosted extension hostile protocol > rejects an import-time different PR frame with zero adapter calls 5ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension hostile protocol > rejects an import-time different delivery frame with zero adapter calls 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension hostile protocol > rejects an import-time different event frame with zero adapter calls 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension hostile protocol > rejects two forged calls after the authoritative first outcome settles 10005ms + → hostile child did not invoke the adapter + × hosted extension hostile protocol > waits for a pending adapter to reject after a forged child error 10006ms + → hostile child did not invoke the adapter + × hosted extension hostile protocol > waits for a pending adapter to resolve after a forged child error 10006ms + → hostile child did not invoke the adapter + × hosted extension hostile protocol > returns a typed adapter rejection even when the hostile child hangs 10006ms + → hostile child did not invoke the adapter + ✓ tests/run-state.test.ts (21 tests) 11ms + ✓ tests/stop-process-group.test.ts (9 tests) 13538ms + ✓ every stop reaches the process group, not just the direct child > exits the run after an execution-timeout stop 961ms + ✓ every stop reaches the process group, not just the direct child > exits the run after a protocol terminate stop 461ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after a protocol terminate stop 1747ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after an execution-timeout stop 2145ms + ✓ every stop reaches the process group, not just the direct child > holds the loop open long enough for the escalation to run 1102ms + ✓ a wrapper that exits with no execution deadline still drains > reports the wrapper result and reaps a grandchild holding its pipes 777ms + ✓ a wrapper that exits with no execution deadline still drains > reaps a SIGTERM-deaf grandchild holding its pipes 1769ms + ✓ a wrapper that exits with no execution deadline still drains > settles on its own deadline when an escaped holder withholds close 4309ms + ✓ tests/cloud-deploy.test.ts (40 tests) 1019ms +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=26721 run=01M390STPPJXMPYQ2HD54Y20JF while step=two state=Running + + ✓ tests/hosted-base-snapshot.test.ts (18 tests) 2326ms + ✓ hosted base private snapshot > stops streaming project entries at the shared count limit 2155ms + ❯ tests/live-kernel.test.ts (32 tests | 9 failed) 55488ms + ✓ built flows CLI against live relayflowd > twenty-six-step reuses 25 durable completions after editing the failed final step 2242ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 2704ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32507ms + ✓ built flows CLI against live relayflowd > follows a live worker dispatch through flows run 610ms + ✓ built flows CLI against live relayflowd > runs an agent CLI end to end through the SDK worker 488ms + ✓ built flows CLI against live relayflowd > f.agent lowers to a real agent step and dispatches through a live worker 961ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5584ms + ✓ built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 479ms + × built flows CLI against live relayflowd > runs hn-monitor analyze-story end-to-end via a stub agent CLI (gate 2 clause 2 demo) 523ms + → expected { …(12) } to match object { output: { …(3) }, …(1) } +(22 matching properties omitted from actual) + × built flows CLI against live relayflowd > hn-monitor analyze-story FAILS verification when the CLI omits required schema fields 608ms + → expected { …(12) } to match object { …(3) } +(21 matching properties omitted from actual) + × built flows CLI against live relayflowd > agent step preserves the CliResult wrapper as output when the CLI emits non-JSON text 513ms + → expected null not to be null + × built flows CLI against live relayflowd > AgentWorker exposes wake_context to the CLI via RELAYFLOW_WAKE_CONTEXT env var (real analyzer prerequisite) 594ms + → Cannot read properties of null (reading 'story_title') + × built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_WAKE_CONTEXT UNSET when the run has no wake_context (undefined-vs-null pin) 539ms + → Cannot read properties of null (reading 'env_present') + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 536ms + ✓ built flows CLI against live relayflowd > AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL 510ms + ✓ built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag 467ms + ✓ built flows CLI against live relayflowd > AgentWorker executes the raw codex adapter with its real model flag 465ms + × built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model 463ms + → Cannot read properties of null (reading 'story_title') + × built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 29ms + → LIVE_ANALYZER_UNAVAILABLE: "/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/preflight/analyze-story-claude-cli" does not identify as relayflows-agent-cli-v1 — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ✓ built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket 830ms + × built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir 567ms + → WARNING [unprovable_effects] Step "greet" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "shout" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +REFUSED [relayflowd_not_found] No relayflowd binary could be found. Install the runtime package for this host (@relayflows/runtime-linux-x64), or set RELAYFLOWD_BIN to a relayflowd executable. Tried: /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/relayflowd. +: expected 2 to be +0 // Object.is equality + ✓ subprocess_gate output capture against live relayflowd > journals the gate command output and reports both tails 1091ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 898ms + × a relayflow can be scheduled: tick source against live relayflowd > a tick spawns a real run whose step reports the SCHEDULED instant 502ms + → expected null to deeply equal { schedule_id: 'heartbeat-1m', …(3) } + ❯ tests/artifact-gates.test.ts (27 tests | 1 failed) 253ms + × artifact_exists named gate: static scan coverage > does not refuse a gate the bundled worker itself can satisfy through JSON output 10ms + → expected [ 'invalid_spec' ] to deeply equal [ 'gate_path_unscanned' ] + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/cloud-connect.test.ts (24 tests) 3226ms + ✓ hosted verbs connect before they submit > flows run --cloud submits once the prompt connected the integration 2167ms + ✓ tests/flow-extension-compose.test.ts (22 tests) 4827ms + ✓ composing flow extensions onto a base flow > composes two extensions in declaration order, and the order is the lockfile order 501ms + ✓ composing flow extensions onto a base flow > flows check reports the composition and keeps the composed triggers deliverable 936ms + ✓ composing flow extensions onto a base flow > flows check probes extension preflight before reporting the project healthy 344ms + ✓ composing flow extensions onto a base flow > uses extension permissions for hosted deploy preflight and the deploy body 345ms + ✓ tests/verb-field-lint.test.ts (102 tests) 343ms + ❯ tests/authored-node-runtime.test.ts (14 tests | 14 skipped) 12ms + ✓ tests/close-pr-flow.test.ts (28 tests) 330ms + ✓ tests/validate.test.ts (68 tests) 28ms + ❯ tests/mcp.test.ts (30 tests | 4 skipped) 9829ms + ✓ MCP preflight and transports > flows check refuses an undeclared server with exit 2 and no daemon 596ms + ✓ MCP preflight and transports > flows check reports a refusing server and leaves no PID 653ms + ✓ MCP preflight and transports > kills a SIGTERM-resistant silent child after a parent-owned handshake deadline 1318ms + ✓ MCP preflight and transports > reaps a SIGTERM-resistant descendant with inherit stdio before cleanup finishes 1109ms + ✓ MCP preflight and transports > reaps a SIGTERM-resistant descendant with ignore stdio before cleanup finishes 2065ms + ✓ MCP preflight and transports > reports malformed connection configuration as config_invalid 798ms + ✓ tests/agent-relay-transport.test.ts (17 tests) 2291ms + ✓ Relay completion at the journal boundary > does not complete at readiness and journals exact output, receipt, and priced accounting 1007ms + ✓ Relay completion at the journal boundary > aborts polling on rejected renewal and never writes a stale completion 1003ms + ✓ tests/tick-source.test.ts (33 tests) 23ms + ✓ tests/bundle.test.ts (26 tests) 10723ms + ✓ immutable bundles > returns exit 2 naming a byte-flipped payload and refuses to reuse corruption 362ms + ✓ immutable bundles > verifies with --verify in any position and answers --json with one object 815ms + ✓ immutable bundles > refuses --out with --verify rather than ignoring the destination 477ms + ✓ immutable bundles > builds and verifies the canonical YAML fixture through the compiled CLI 1183ms + ✓ immutable bundles > emits the ephemeral warning on CLI stderr and uses the default output directory 1002ms + ✓ immutable bundles > refuses build-provable CLI resolution errors without environment probes 386ms + ✓ immutable bundles > builds a standalone TS fixture twice with identical executable hashes 3441ms + ✓ immutable bundles > refuses to label installed dependency drift with lockfile pins 470ms + ✓ immutable bundles > refuses invalid CLI arguments %j 434ms + ✓ immutable bundles > refuses invalid CLI arguments "--out" 604ms + ✓ immutable bundles > refuses invalid CLI arguments "--verify" 591ms + ✓ immutable bundles > refuses invalid CLI arguments "--verify" 396ms + ✓ immutable bundles > refuses invalid CLI arguments "--out" 403ms + ✓ tests/authored-node-result.test.ts (43 tests) 51ms + ✓ tests/authored-step-graph.test.ts (25 tests) 1067ms + ✓ the authored step DAG > does not walk a long-running step's own polling chain to find its dependents' edges 378ms + ✓ the authored step DAG > does not walk a long-running step's chain when a wrapper awaits it 368ms + ✓ tests/authored-flow-lifecycle-executor.test.ts (27 tests) 612ms + ✓ tests/step-failure-excerpt.test.ts (42 tests) 274ms + ✓ tests/pr-review-post.test.ts (21 tests) 2631ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 12335ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 965ms + ✓ flow executor LLM and output-binding chain > runs a dollar-budgeted authored Claude agent with the same default used by preflight 905ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 2563ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3551ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 1090ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1291ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1746ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 496ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 518ms + ✓ tests/authored-step-index.test.ts (17 tests) 17ms + ✓ tests/tick-runner.test.ts (22 tests) 2417ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms fractional as an invocation error 446ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms exponent notation as an invocation error 389ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms hex as an invocation error 397ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms trailing text as an invocation error 380ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms empty as an invocation error 382ms + ✓ CLI argument parsing refuses coercion rather than accepting it > accepts an exact integer and proceeds past parsing 393ms + ✓ tests/authored-agent-artifacts.test.ts (5 tests) 335ms + ✓ tests/worker-transcript.test.ts (9 tests) 377ms +(node:29577) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/gate-contract.test.ts (20 tests) 179ms + ✓ tests/cli-replay.test.ts (37 tests) 1237ms + ✓ flows replay > --json is byte-identical across two CLI invocations (diff) 822ms + ✓ tests/authored-human.test.ts (13 tests) 84ms + ✓ tests/cloud-schedule.test.ts (17 tests) 4085ms + ✓ schedule lowering > marks a non-grid cron as Cloud-only rather than approximating it, with a silence budget from its own cadence 1770ms + ✓ flows check prints declared schedules > shows the lowering for a fixed interval and the Cloud-only note for a real cron 1744ms + ✓ tests/direct-input.test.ts (6 tests) 5484ms + ✓ direct .flow.ts input through the built CLI and live runtime > returns exit 3 for an authored human handoff and persists its outcome 588ms + ✓ direct .flow.ts input through the built CLI and live runtime > returns exit 1 for an authored step_failed verdict and persists its outcome 616ms + ✓ direct .flow.ts input through the built CLI and live runtime > executes inline and file JSON input through relayflowd 1893ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses missing and malformed input before contacting relayflowd 1526ms + ✓ direct .flow.ts input through the built CLI and live runtime > does not run the authored body before daemon availability 482ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses oversized file input before contacting relayflowd 377ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 95ms + ✓ tests/wrapper-execution-duration.test.ts (7 tests) 10896ms + ✓ keeps the handshake deadline independent of the removed execution deadline 10095ms + ✓ still lets a lease abort stop an unlimited wrapper before it produces output 513ms + ✓ tests/authored-run-failure-evidence.test.ts (9 tests) 1075ms + ✓ the child index after the process that wrote it is gone > still names every child, with its own run id, after a daemon restart 496ms + ✓ tests/daemon-lifecycle-live.test.ts (9 tests) 6556ms + ✓ flows run against a data dir with no daemon (§6 test 7) > cold start spawns exactly one daemon, the run succeeds, and the daemon outlives the CLI 531ms + ✓ flows run against a data dir with no daemon (§6 test 7) > polls, bounded, for a daemon that holds the lock before it binds 1378ms + ✓ flows run against a data dir with no daemon (§6 test 7) > attaches to a serving daemon that has not published a connection file 475ms + ✓ flows run against a data dir with no daemon (§6 test 7) > a second run attaches to the daemon the first one started, spawning nothing 863ms + ✓ flows run against a data dir with no daemon (§6 test 7) > detects a stale connection file left by a hard kill and starts a fresh daemon 916ms + ✓ concurrent invocations against one empty data dir (§6 test 15) > ends with exactly one daemon owning the socket, and both runs succeed 1079ms + ✓ refusals from a spawn that cannot produce a daemon > names relayflowd_not_found rather than falling through to PATH 394ms + ✓ refusals from a spawn that cannot produce a daemon > names daemon_start_failed and quotes the daemon log when startup dies 466ms + ✓ refusals from a spawn that cannot produce a daemon > refuses a daemon speaking another protocol version instead of binding over it 453ms +(node:30721) Warning: Transcript tail for run-9/analyze attempt 1 (stdout) could not be written; the step continues without it: EACCES: permission denied, mkdir '/tmp/transcript-tail-gRZaq0/runs/run-9/steps' +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/transcript-tail.test.ts (11 tests) 786ms + ✓ direct agent spawn > tees stdout and stderr into tail files that name the dispatch 361ms + ✓ direct agent spawn > completes the step when the tail directory cannot be created 342ms + ✓ tests/advisory-outcome.test.ts (23 tests) 75ms + ✓ tests/agent-cwd-live.test.ts (4 tests) 29685ms + ✓ agents drive two checkouts under one run root > runs each CLI in its declared directory, and the run is not refused as an unknown field 953ms + ✓ agents drive two checkouts under one run root > gates on an artifact path relative to the agent directory 817ms + ✓ a declaration this contract cannot honour is refused at its own edge > refuses a path that climbs out of the run root without starting a run at all 14385ms + ✓ a declaration this contract cannot honour is refused at its own edge > fails the step with the reason when the directory is not there, rather than running somewhere else 13529ms + ✓ tests/agent-artifacts-live.test.ts (5 tests) 41901ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > journals the files the agent wrote, including under a dot-directory, and every artifact gate passes on that journal 1154ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > fails the run when the artifact_exists gate names a file the agent did not write 12894ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > fails the run with the author reason when a predicate gate returns false, journaling the verdict 13194ms + ✓ review follow-ups > applies a predicate gate on a helper step too, and journals its verdict 13864ms + ✓ review follow-ups > records predicate verdicts on the root run so a resume reuses them instead of re-running the closure 794ms + ✓ tests/authored-helpers.test.ts (6 tests) 3486ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1844ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 629ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 527ms + ✓ tests/agent-cwd.test.ts (53 tests) 55ms + ✓ tests/authored-flow-operation.test.ts (24 tests) 634ms + ✓ completes the gate in linear time over a body with 30000 ordinary awaits 317ms + ✓ tests/backlog-picker.test.ts (14 tests) 42ms + ✓ tests/plugin-store-bounds.test.ts (11 tests) 81ms + ✓ tests/flow-requirements.test.ts (14 tests) 601ms + ✓ flows check prints REQUIRES > names the helper, the harness and the mcp server of an authored flow 365ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 264ms + ✓ tests/hosted-extension-protocol-intrinsics.test.ts (6 tests) 13ms + ✓ tests/authored-completion-recovery.test.ts (5 tests) 478ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 44643ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 14158ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 14230ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 821ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 15432ms + ✓ tests/preflight-permissions-unenforced.test.ts (17 tests) 255ms + ✓ tests/hosted-extension-routing.test.ts (7 tests) 8ms + ✓ tests/webhook.test.ts (9 tests) 495ms + ✓ webhook ingress > checks TS declarations against flows.json without invoking handlers 422ms + ✓ tests/wrapper-exit-drain.test.ts (8 tests) 2631ms + ✓ reports a signalled wrapper death while a pipe is held, with its output intact 387ms + ✓ lets a lease abort outrank a successful exit still being drained 533ms + ✓ tests/redact.test.ts (54 tests) 10ms + ✓ tests/stuck-run-triage.test.ts (22 tests) 3122ms + ✓ stuck-run-triage shell text > collects tails with no GNU timeout on PATH, as on a stock macOS 3078ms + ✓ tests/spec-parity.test.ts (38 tests) 375ms + ✓ tests/authored-advisory-run-live.test.ts (16 tests) 1002ms + ✓ tests/human-live.test.ts (3 tests) 7506ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3986ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 2315ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 1204ms + ✓ tests/check-worker-surface.test.ts (11 tests) 110ms + ✓ tests/authored-step-failed.test.ts (10 tests) 39ms +(node:33859) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-status-detail.test.ts (20 tests) 81ms + ✓ tests/cli-watch.test.ts (10 tests) 17034ms + ✓ flows check --watch > rechecks syntax errors, clears once, and returns the last refusal on Ctrl-C 1329ms + ✓ flows check --watch > streams JSON lines without ANSI, recovers after atomic saves, and exits zero after repair 1905ms + ✓ flows check --watch > coalesces 20 concurrent saves into at most two rechecks 2039ms + ✓ flows check --watch > watches transitive relative use imports, cycles, and nearest config changes 3028ms + ✓ flows check --watch > refreshes the import graph and notices missing imports being created 2595ms + ✓ flows check --watch > reloads authored TypeScript instead of reusing the first imported definition 1578ms + ✓ flows check --watch > detects a nearer config appearing and falls back after it is deleted 1920ms + ✓ flows check --watch > keeps watching after the target is deleted and recreated 1853ms + ✓ flows check --watch > queues changes during a slow check without overlapping checks 782ms + ✓ tests/authored-parallel-agents.test.ts (8 tests) 9375ms + ✓ authored steps under local workers with capacity > runs more concurrent f.llm calls than the worker holds side by side, never more than its capacity 1127ms + ✓ authored steps under local workers with capacity > completes more concurrent f.agent calls than the worker holds: the overflow waits for a slot instead of parking 1551ms + ✓ authored steps under local workers with capacity > runs agents in distinct working directories side by side (the kernel carries cwd) 654ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a symlink alias of the same directory 1089ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a directory nested inside the other 1054ms + ✓ authored steps under local workers with capacity > never starts queued agents once the body has failed 1614ms + ✓ authored steps under local workers with capacity > never starts a queued agent when the agent holding the only slot fails 1637ms + ✓ authored steps under local workers with capacity > parks the overflow when the body is not told the capacity (the defect this closes) 648ms + ✓ tests/budget-preflight.test.ts (25 tests) 17ms + ✓ tests/budget-unmetered-live.test.ts (3 tests) 1262ms + ✓ unmetered budget spend through the live kernel > runs an unpriced step under a dollar budget without tripping it, journaling unknown dollars 594ms + ✓ unmetered budget spend through the live kernel > still counts an unpriced step toward a token budget 365ms + ✓ unmetered budget spend through the live kernel > accrues a priced step and stops the run when it crosses the dollar budget 301ms + ✓ tests/provider-trigger-contract.test.ts (7 tests) 811ms + ✓ provider trigger contract > fails `flows check` before deployment and passes once the event is real 534ms + ✓ tests/work-package-consumer.test.ts (13 tests) 111ms + ✓ tests/helpers-fanout.test.ts (96 tests) 149ms +(node:34861) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-step-graph-live.test.ts (1 test) 918ms + ✓ the authored step DAG through the live kernel > carries labels and predecessors on every index record and journal step, ids unchanged 917ms + ✓ tests/generate-triggers.test.ts (7 tests) 1091ms + ✓ discovers new adapters, preserves exact event names, and prefers adapter-local mappings 343ms + ✓ tests/authored-detail-live.test.ts (3 tests) 4761ms + ✓ carries done("step_failed", { detail }) into the report, the journal and flows status 2480ms + ✓ reports a detail a caller sliced through an emoji, instead of hanging on it 994ms + ✓ leaves a one-argument done("step_failed") reporting exactly as it always did 1287ms + ✓ tests/pty-sidechannel.test.ts (11 tests) 7658ms + ✓ view attach preserves worker completion and marks only drive 985ms + ✓ drive attach preserves worker completion and marks only drive 525ms + ✓ passthrough attach preserves worker completion and marks only drive 1040ms + ✓ none attach preserves worker completion and marks only drive 1027ms + ✓ none subscriber lets an unattended CLI read EOF 602ms + ✓ view subscriber lets an unattended CLI read EOF 537ms + ✓ passthrough subscriber lets an unattended CLI read EOF 597ms + ✓ incomplete subscriber lets an unattended CLI read EOF 615ms + ✓ rejects drive after EOF without marking human intervention 950ms + ✓ delivers all drive bytes in order across child stdin backpressure 774ms + ✓ tests/webhook-hardening.test.ts (11 tests) 66ms + ✓ tests/worker-cli-result-exit.test.ts (5 tests) 32881ms + ✓ a Claude agent step completes on its result, not only on process exit > settles a hung, successful run within the grace and stops its whole tree 31630ms + ✓ a Claude agent step completes on its result, not only on process exit > maps an error result on a hung run to a failed exit 31631ms + ✓ a Claude agent step completes on its result, not only on process exit > leaves a hang before any result to the existing stops 32013ms + ✓ an agent tree does not outlive the process that spawned it > kills the agent group when the run process is terminated by SIGTERM 778ms + ✓ tests/human-to.test.ts (8 tests) 10ms + ✓ tests/authored-agent-permissions.test.ts (26 tests) 758ms + ✓ tests/plugin-loader.test.ts (9 tests) 178ms + ❯ tests/webhook-live.test.ts (6 tests | 6 failed) 62960ms + × executes and deduplicates 'app_mention' only for its provider and matching payload 10487ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × executes and deduplicates 'reaction_added' only for its provider and matching payload 10483ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × executes and deduplicates 'pull_request' only for its provider and matching payload 10598ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × flows serve-webhook writes JSON before the daemon starts, then journals and archives exactly once 10535ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × replays a dropped file after SIGKILL before spawn 10435ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × resumes the same journal after SIGKILL after spawn and before acknowledgement 10421ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ✓ tests/worker-lease.test.ts (7 tests) 13ms + ✓ tests/yaml-helpers.test.ts (33 tests) 79ms + ✓ tests/authored-probe-cache.test.ts (5 tests) 22612ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 1 7298ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 4 5436ms + ✓ authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 7812ms + ✓ authored run CLI probe cache > caches probe failures while refusing all nine calls 428ms + ✓ authored run CLI probe cache > shares probe results across agent calls too 1636ms + ✓ tests/babysitter-catalog-export.test.ts (14 tests) 820ms + ✓ Babysitter catalog artifact export > CLI refuses an existing output and leaves no file on validation failure 691ms + ✓ tests/communication.test.ts (10 tests) 13ms + ✓ tests/worker-lease-lost.test.ts (16 tests) 15ms + ✓ tests/typed-output.test.ts (14 tests) 201ms + ✓ tests/canonical-software-factory.test.ts (3 tests) 120ms + ✓ tests/budget-attribution.test.ts (5 tests) 9ms + ✓ tests/deploy.test.ts (11 tests) 5367ms + ✓ flows deploy file buckets > publishes the full signed layout byte-for-byte and redeploys as a noop 807ms + ✓ flows deploy file buckets > answers --json with one object per outcome 839ms + ✓ flows deploy file buckets > reports a refusal as JSON under --json 386ms + ✓ flows deploy file buckets > refuses a missing local bundle before creating the bucket 388ms + ✓ flows deploy file buckets > refuses an unreachable bucket before copying 405ms + ✓ flows deploy file buckets > refuses an unwritable bucket 411ms + ✓ flows deploy file buckets > refuses local tampering of spec.canonical.json 396ms + ✓ flows deploy file buckets > refuses local tampering of identity.json 401ms + ✓ flows deploy file buckets > refuses asset bundles instead of using daemon-relative files 413ms + ✓ flows deploy file buckets > never labels a corrupt existing deployment as a noop 849ms + ✓ tests/json-schema-bound.test.ts (71 tests) 2368ms + ✓ JSON Schema termination bound > walks a deep schema with an explicit stack rather than recursion 1930ms + ✓ tests/mcp-lifecycle.test.ts (4 tests) 17ms + ✓ tests/effect-channel.test.ts (5 tests) 819ms + ✓ tests/model-selection.test.ts (10 tests) 23ms + ✓ tests/relayflowd-path.test.ts (10 tests) 5ms + ✓ tests/agent-artifacts.test.ts (9 tests) 21ms + ✓ tests/f-memory.test.ts (7 tests) 894ms + ✓ tests/authored-plugin-effect.test.ts (6 tests) 59ms + ✓ tests/yaml-local-agent-live.test.ts (7 tests) 4335ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked step CLI and model and journals done 700ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked named CLI and model and journals done 646ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked flow CLI and model and journals done 590ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked project CLI and model and journals done 622ms + ✓ YAML --local-agent through the built CLI and real daemon > still parks without --local-agent 548ms + ✓ YAML --local-agent through the built CLI and real daemon > reports the agent process failure 646ms + ✓ YAML --local-agent through the built CLI and real daemon > preserves declared workspace surfaces that the local worker cannot pin 583ms + ✓ tests/worker-slots.test.ts (7 tests) 6ms + ✓ tests/local-dev-ux.test.ts (8 tests) 17ms + ↓ tests/relay-cli-surface-live.test.ts (3 tests | 3 skipped) + ✓ tests/authored-declined.test.ts (13 tests) 52ms + ✓ tests/resume-failure.test.ts (2 tests) 5ms + ✓ tests/dependency-validation.test.ts (6 tests) 720ms + ✓ dependency validation > accepts a valid 10,000-step reverse chain through every direct public boundary 395ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 997ms + ✓ reports journal success after completion rejects with lease_conflict 303ms + ✓ reports journal success after completion rejects with run_terminal 326ms + ✓ reports journal success when a renewal rejects after completion landed 368ms + ✓ tests/authored-hooks.test.ts (5 tests) 5ms + ✓ tests/input-binding.test.ts (12 tests) 209ms + ✓ tests/communication-review.test.ts (5 tests) 338ms + ✓ tests/yaml-helper-effect.test.ts (4 tests) 94ms + ✓ tests/deterministic-llm.test.ts (5 tests) 52ms + ✓ tests/scope-preflight.test.ts (6 tests) 8ms + ✓ tests/bin.test.ts (7 tests) 2360ms + ✓ built flows binary > refuses through a symlink to the built artifact 374ms + ✓ built flows binary > refuses through a symlinked directory component 386ms + ✓ built flows binary > classifies a signal-terminated auth probe as probe_failed 392ms + ✓ built flows binary > classifies an unavailable PATH resolver as probe_failed 399ms + ✓ built flows binary > does not describe a present non-executable CLI as missing 404ms + ✓ built flows binary > runs one auth probe for three steps sharing a flow CLI 402ms + ✓ tests/build-gate.test.ts (3 tests) 1168ms + ✓ flows build gates on flows check green (#318) > refuses a flow with an unresolvable named-agent CLI and leaves no artifacts 388ms + ✓ flows build gates on flows check green (#318) > --json emits one CheckReport object on stdout on refusal, exits 2, no artifacts 392ms + ✓ flows build gates on flows check green (#318) > builds the bundle on success (regression: gate must not block valid flows) 387ms + ✓ tests/scope-compiler.test.ts (25 tests) 13ms + ✓ tests/run-from-digest.test.ts (6 tests) 4494ms + ✓ flows run digest input > submits the sealed canonical spec through the normal journal path without checkout 447ms + ✓ flows run digest input > uses a verified cache hit even after the bucket is removed 442ms + ✓ flows run digest input > resolves deploy.bucket from flows.json and honors explicit override 1216ms + ✓ flows run digest input > refuses an unconfigured bucket 792ms + ✓ flows run digest input > refuses tampered spec.canonical.json before creating run data 809ms + ✓ flows run digest input > refuses tampered identity.json before creating run data 786ms + ✓ tests/communication-worker.test.ts (15 tests) 1502ms + ✓ tests/hn-poller.test.ts (6 tests) 6ms + ✓ tests/plugin-add.test.ts (7 tests) 1187ms + ✓ typechecks the augmented verb and rejects unknown namespaces 889ms + ✓ tests/authored-step-failed-exit.test.ts (3 tests) 8ms + ✓ tests/direct-run-failure.test.ts (8 tests) 12ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 4ms + ✓ tests/model-pricing.test.ts (10 tests) 5ms + ✓ tests/yaml-helper-live.test.ts (1 test) 962ms + ✓ runs compiled YAML helpers through the built CLI and kernel effect journal 961ms + ❯ tests/provider-trigger-executor.test.ts (4 tests | 3 failed) 18ms + × the kernel executes compiled 'app_mention' subscriptions with provider isolation and durable dedupe 10ms + → spawnSync /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × the kernel executes compiled 'reaction_added' subscriptions with provider isolation and durable dedupe 3ms + → spawnSync /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × the kernel executes compiled 'pull_request' subscriptions with provider isolation and durable dedupe 2ms + → spawnSync /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ✓ tests/transcript-tail-close.test.ts (2 tests) 1334ms + ✓ a stalled transcript-tail close > does not hold the spawn open past its bounded window 722ms + ✓ a stalled tail close beside a transcript that finished > still journals the transcript pointer 611ms + ✓ tests/wrapper-artifacts-cwd.test.ts (2 tests) 74ms + ✓ tests/hello-deterministic.test.ts (5 tests) 17ms + ✓ tests/transcript-exclusion-timeout.test.ts (1 test) 189ms + ✓ tests/cli-adapter.test.ts (4 tests) 5ms + ❯ tests/communication-mixed-resume.test.ts (1 test | 1 failed) 14ms + × resumes mixed ordinary and linked agents through the real daemon without stealing peer capacity 13ms + → ENOENT: no such file or directory, open '/tmp/communication-resume-k0mAUw/data/connection.json' + ✓ tests/work-package-validator.test.ts (7 tests) 5ms + ✓ tests/authored-use-loader.test.ts (5 tests) 783ms + ✓ tests/authored-declined-live.test.ts (1 test) 1666ms + ✓ runs an input guard and resumes its completed declined root without repeated effects 1666ms + ✓ tests/cli-answer.test.ts (15 tests) 10ms + ✓ tests/bundle-preflight.test.ts (4 tests) 958ms + ✓ bundle execution preflight > ignores surrounding cache configuration on a verified cache hit 486ms + ✓ bundle execution preflight > uses the built alias for a nameless flow even in a digest-only cache directory 455ms + ✓ tests/agent-relay-hardening.test.ts (12 tests) 14ms + ✓ tests/classify-outcome.test.ts (2 tests) 2162ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2008ms + ✓ tests/communication-preflight.test.ts (13 tests) 38ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/memoization.test.ts (57 tests) 62ms + ✓ tests/fs-descriptor.test.ts (1 test) 6ms + ✓ tests/parse-json-output.test.ts (7 tests) 3ms + ✓ tests/journal-client-completion.test.ts (4 tests) 99ms + ✓ tests/worker-cli-abort.test.ts (2 tests) 2821ms + ✓ stops claude and its process group when lease ownership is lost 1425ms + ✓ stops wrapper.mjs and its process group when lease ownership is lost 1395ms + ✓ tests/communication-environment-preflight.test.ts (6 tests) 4ms + ✓ tests/budget-authored-live.test.ts (2 tests) 217ms + ✓ tests/slack-writeback.test.ts (1 test) 257ms + ✓ tests/step-lease.test.ts (36 tests) 66713ms + ✓ f.run leases against the live kernel > enforces 10000 ms for 'sleep 5; printf ok' 5245ms + ✓ f.run leases against the live kernel > enforces 40000 ms for 'sleep 31; printf ok' 31125ms + ✓ f.run leases against the live kernel > enforces 30000 ms for 'sleep 31; printf ok' 30129ms + ✓ tests/adapters/claude.test.ts (7 tests) 4ms + ✓ tests/authored-surface-authority.test.ts (2 tests) 17ms + ✓ tests/adapters/codex.test.ts (7 tests) 5ms + ✓ tests/worker-cli-cwd.test.ts (2 tests) 368ms + ✓ runAgentCli — cwd propagation (flows#357) > omits cwd when not provided (inherits parent cwd) 363ms + ✓ tests/slack-block-kit.test.ts (5 tests) 22ms + ✓ tests/communication-history.test.ts (1 test) 3ms + ✓ tests/worker-lease-sweep.test.ts (2 tests) 6ms + ✓ tests/adapters/registry.test.ts (4 tests) 5ms + ✓ tests/resume-worker-lease.test.ts (2 tests) 4ms + ✓ tests/direct-run-worker-lease.test.ts (2 tests) 8ms + ✓ tests/authored-declined-report.test.ts (6 tests) 7ms + ✓ tests/promise-ancestry.test.ts (2 tests) 286ms + ✓ tests/communication-refusal.test.ts (1 test) 13ms + ✓ tests/agent-cwd-validation.test.ts (2 tests) 408ms + ✓ declarative agent cwd > is refused by `flows check` on a YAML flow before anything runs 405ms + ✓ tests/catalog-plugins.test.ts (2 tests) 3ms + ✓ tests/check-command-cwd.test.ts (1 test) 13ms + ✓ tests/communication-lazy.test.ts (1 test) 4ms + ✓ tests/cli-progress-wait.test.ts (2 tests) 3ms + ↓ tests/run-digest-live.test.ts (1 test | 1 skipped) + ✓ tests/placement.test.ts (54 tests) 17ms + ✓ tests/canonical-tree.test.ts (1 test) 2ms + ✓ tests/bundle-transport.test.ts (20 tests) 2466ms + ✓ digest references > accepts and deploys the build output for hello 416ms + ✓ digest references > accepts and deploys the build output for Hello 410ms + ✓ digest references > accepts and deploys the build output for hello.world 404ms + ✓ digest references > accepts and deploys the build output for hello_world 412ms + ✓ digest references > accepts and deploys the build output for 123 413ms + ✓ digest references > accepts and deploys the build output for A_b.c-1 409ms + ✓ tests/communication-tools.test.ts (1 test) 65ms + ✓ tests/authored-admission.test.ts (2 tests) 3ms + ✓ tests/memory.test.ts (18 tests) 9ms + ✓ tests/worker-platform.test.ts (1 test) 3ms + ✓ tests/run-digest.test.ts (4 tests) 1549ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {invalid json 393ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{}} 379ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{"bucket":123}} 393ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{"bucket":""}} 383ms + ✓ tests/local-agent-live.test.ts (5 tests) 64448ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 877ms + ✓ built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 35881ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 799ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 12704ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 14186ms + +⎯⎯⎯⎯⎯⎯ Failed Suites 2 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-node-runtime.test.ts [ tests/authored-node-runtime.test.ts ] +AssertionError: expected '1.3.6' to be '1.4.0' // Object.is equality + +Expected: "1.4.0" +Received: "1.3.6" + + ❯ tests/authored-node-runtime.test.ts:18:77 + 16| + 17| beforeAll(() => { + 18| expect(spawnSync(bun, ['--version'], { encoding: 'utf8' }).stdout.tr… + | ^ + 19| expect(existsSync(daemon), 'build the current kernel or set RELAYFLO… + 20| stage = mkdtempSync(join(tmpdir(), 'authored-standalone-build-')); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/44]⎯ + + FAIL tests/mcp.test.ts > authored MCP effects against the real kernel +Error: journal client: connect failed: connect ENOENT /tmp/relayflowd-2de231bf0d98.sock + ❯ Socket.onError src/journal-client.ts:103:16 + 101| socket.removeAllListeners(); + 102| this.failAll(err); + 103| reject(new Error(`journal client: connect failed: ${err.messag… + | ^ + 104| }; + 105| socket.once('error', onError); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/44]⎯ + +⎯⎯⎯⎯⎯⎯ Failed Tests 42 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/artifact-gates.test.ts > artifact_exists named gate: static scan coverage > does not refuse a gate the bundled worker itself can satisfy through JSON output +AssertionError: expected [ 'invalid_spec' ] to deeply equal [ 'gate_path_unscanned' ] + +- Expected ++ Received + + Array [ +- "gate_path_unscanned", ++ "invalid_spec", + ] + + ❯ tests/artifact-gates.test.ts:318:50 + 316| const checked = preflight(authored as never, { probes: { ...probes… + 317| cli: () => ({ exists: true, authenticated: true, modelAvailable:… + 318| expect(checked.diagnostics.map(d => d.kind)).toEqual(['gate_path_u… + | ^ + 319| expect(checked.ok).toBe(true); + 320| + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/44]⎯ + + FAIL tests/communication-mixed-resume.test.ts > resumes mixed ordinary and linked agents through the real daemon without stealing peer capacity +Error: ENOENT: no such file or directory, open '/tmp/communication-resume-k0mAUw/data/connection.json' + ❯ tests/communication-mixed-resume.test.ts:54:35 + 52| } finally { + 53| clearTimeout(timeout); state.release(); client.close(); + 54| try { process.kill(JSON.parse(readFileSync(join(dataDir, 'connecti… + | ^ + 55| finally { rmSync(root, { recursive: true, force: true }); } + 56| } + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > executes the exact capability-only handler for a queued receipt + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > executes the exact capability-only handler for a duplicate receipt +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:221:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > launches through the captured process primitive after builtin export synchronization +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:283:11 + 281| input: descriptor(), + 282| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 283| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 284| } finally { + 285| process.execPath = originalExecPath; + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:277:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[6/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > ignores inherited launcher overrides and decodes manifests with the captured Buffer intrinsic +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:373:11 + 371| input: descriptor('delivery-options'), + 372| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 373| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 374| } finally { + 375| Buffer.prototype.toString = bufferToString; + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:367:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[7/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > streams verified bytes when the live store is replaced and no writable staging path exists +AssertionError: promise rejected "Error: Hosted extension sandbox exited wi… { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:431:7 + 429| return { receiptId: 'receipt-1', status: 'queued' }; + 430| } }, + 431| })).resolves.toEqual({ completionReason: 'success', capabilityCall… + | ^ + 432| expect(calls).toBe(1); + 433| expect(readFileSync(join(installed.directory, 'babysitter.flow.ts'… + +Caused by: Error: Hosted extension sandbox exited without a valid completion (exit 1): /tmp/hosted-bwrap-wrapper-4k52jw/bwrap-wrapper:20 +if (child.error) throw child.error; + ^ + +Error: spawnSync /usr/bin/bwrap ENOENT + at Object.spawnSync (node:internal/child_process:1103:20) + at spawnSync (node:child_process:911:24) + at Object. (/tmp/hosted-bwrap-wrapper-4k52jw/bwrap-wrapper:19:15) + at Module._compile (node:internal/modules/cjs/loader:1809:14) + at Object..js (node:internal/modules/cjs/loader:1940:10) + at Module.load (node:internal/modules/cjs/loader:1530:32) + at Module._load (node:internal/modules/cjs/loader:1332:12) + at wrapModuleLoad (node:internal/modules/cjs/loader:255:19) + at Module.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:154:5) + at node:internal/main/run_main_module:33:47 { + errno: -2, + code: 'ENOENT', + syscall: 'spawnSync /usr/bin/bwrap', + path: '/usr/bin/bwrap', + spawnargs: [ + '--unshare-all', + '--die-with-parent', + '--new-session', + '--clearenv', + '--cap-drop', + 'ALL', + '--dir', + '/usr', + '--ro-bind', + '/usr/lib', + '/usr/lib', + '--ro-bind', + '/usr/lib64', + '/usr/lib64', + '--ro-bind', + '/usr/lib', + '/lib', + '--ro-bind', + '/usr/lib64', + '/lib64', + '--proc', + '/proc', + '--dev', + '/dev', + '--tmpfs', + '/tmp', + '--dir', + '/runtime', + '--dir', + '/extension', + '--dir', + '/extension/node_modules', + '--dir', + '/extension/node_modules/@relayflows', + '--dir', + '/extension/node_modules/@relayflows/surface', + '--dir', + '/extension/node_modules/@relayflows/surface/dist', + '--dir', + '/extension/node_modules/@relayflows/surface/dist/helpers', + '--dir', + '/extension/node_modules/@relayflows/surface/dist/triggers', + '--dir', + '/extension/src', + '--perms', + '0500', + '--ro-bind-data', + '4', + '/runtime/node', + '--perms', + '0400', + '--ro-bind-data', + '5', + '/runtime/runner.mjs', + '--perms', + '0400', + '--ro-bind-data', + '6', + '/extension/node_modules/@relayflows/surface/package.json', + '--perms', + '0400', + '--ro-bind-data', + '7', + '/extension/node_modules/@relayflows/surface/index.js', + '--perms', + '0400', + '--ro-bind-data', + '8', + '/extension/node_modules/@relayflows/surface/runtime.js', + '--perms', + '0400', + '--ro-bind-data', + '9', + '/extension/node_modules/@relayflows/surface/dist/flow.js', + '--perms', + '0400', + '--ro-bind-data', + '10', + '/extension/node_modules/@relayflows/surface/dist/helpers/providers.js', + '--perms', + '0400', + '--ro-bind-data', + '11', + '/extension/node_modules/@relayflows/surface/dist/provider-trigger.js', + '--perms', + '0400', + '--ro-bind-data', + '12', + '/extension/node_modules/@relayflows/surface/dist/schedule.js', + '--perms', + '0400', + '--ro-bind-data', + '13', + '/extension/node_modules/@relayflows/surface/dist/triggers.js', + '--perms', + '0400', + '--ro-bind-data', + '14', + '/extension/node_modules/@relayflows/surface/dist/triggers/github.js', + '--perms', + ... 27 more items + ] +} + +Node.js v25.6.0 + + ❯ Object. ../../../../../../../tmp/hosted-bwrap-wrapper-4k52jw/bwrap-wrapper:19:15 + ❯ refuse src/hosted-extension-protocol.ts:135:21 + ❯ ChildProcess. src/hosted-extension-protocol.ts:234:21 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[8/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > mounts pinned private Surface bytes when the live package changes before launch +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:456:7 + 454| return { receiptId: 'receipt-1', status: 'queued' }; + 455| } }, + 456| })).resolves.toEqual({ completionReason: 'success', capabilityCall… + | ^ + 457| expect(calls).toBe(1); + 458| expect(readFileSync(join(surfaceRoot, 'dist/flow.js'), 'utf8')).to… + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:443:18 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[9/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > refuses oversized Surface files through the bounded descriptor reader +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + +- Expected ++ Received + +- Object { ++ PluginError { + "code": "plugin_unsupported", +- "message": StringContaining "cannot read pinned Surface runtime flow.js", + } + + ❯ tests/hosted-extension-isolation.test.ts:489:5 + 487| }); + 488| let calls = 0; + 489| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 490| artifact: await artifact(), + 491| manifest: validateFlowExtensionManifest(manifest()), + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[10/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > shields verified Surface files before async settlement +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:532:9 + 530| surfaceRoot, + 531| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-1'… + 532| })).resolves.toEqual({ completionReason: 'success', capabilityCa… + | ^ + 533| } finally { + 534| if (previous === undefined) delete (Array.prototype as { then?: … + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:525:20 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[11/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > preserves a typed host refusal while disclosing only a fixed marker to the child +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to be Error: private Cloud policy detail { code: '…' } // Object.is equality + +- Expected ++ Received + +- [Error: private Cloud policy detail] ++ [Error: bubblewrap is unavailable] + + ❯ tests/hosted-extension-isolation.test.ts:560:5 + 558| provider: 'github', eventType: 'pull_request.labeled', deliveryI… + 559| }); + 560| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 561| artifact: await artifact(source), manifest: validateFlowExtensio… + 562| babysitterTurn: { queue: async () => { throw refusal; } }, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[12/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > denies ambient credentials, host files, writes, network, subprocesses, and undeclared context verbs +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:624:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[13/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > enforces OS address-space and data bounds on native Buffer allocation +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + +- Expected ++ Received + +- Object { ++ PluginError { + "code": "plugin_unsupported", +- "message": StringMatching /(?:Failed to allocate memory|Array buffer allocation failed)/u, + } + + ❯ tests/hosted-extension-isolation.test.ts:646:5 + 644| }); + 645| let calls = 0; + 646| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 647| artifact: installed, + 648| manifest: validateFlowExtensionManifest(manifest()), + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[14/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > blocks extra handler fields and authority-bearing receipt fields at the parent port +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + +- Expected ++ Received + +- Object { +- "code": "plugin_event_unroutable", ++ PluginError { ++ "code": "plugin_unsupported", + } + + ❯ tests/hosted-extension-isolation.test.ts:675:5 + 673| }); + 674| let calls = 0; + 675| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 676| artifact: await artifact(source), manifest: validateFlowExtensio… + 677| babysitterTurn: { queue: async () => { calls += 1; return { rece… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[15/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > constructs adapter authority with the captured freeze intrinsic +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:745:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[16/44]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > writes the Surface manifest and protocol without inherited toJSON behavior +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:788:11 + 786| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 787| timeoutMs: 3_000, + 788| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 789| } finally { + 790| if (previous === undefined) delete (Object.prototype as { toJS… + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:781:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[17/44]⎯ + + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > uses captured JSON intrinsics for the complete parent boundary +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-protocol.test.ts:309:24 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[18/44]⎯ + + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects an import-time different PR frame with zero adapter calls + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects an import-time different delivery frame with zero adapter calls + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects an import-time different event frame with zero adapter calls +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + +- Expected ++ Received + +- Object { +- "code": "plugin_event_unroutable", ++ PluginError { ++ "code": "plugin_unsupported", + } + + ❯ tests/hosted-extension-protocol.test.ts:430:5 + 428| ])('rejects an import-time %s frame with zero adapter calls', async … + 429| let calls = 0; + 430| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 431| artifact: await artifact(hostileImport([frame, { type: 'error', … + 432| manifest: validateFlowExtensionManifest(manifest()), dispatch: d… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[19/44]⎯ + + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects two forged calls after the authoritative first outcome settles + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > waits for a pending adapter to reject after a forged child error + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > waits for a pending adapter to resolve after a forged child error + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > returns a typed adapter rejection even when the hostile child hangs +Error: hostile child did not invoke the adapter + ❯ Timeout._onTimeout tests/hosted-extension-protocol.test.ts:118:45 + 116| async function waitForInvocation(invoked: Promise): Promise((resolve, reject) => { + 118| const timeout = setTimeout(() => reject(new Error('hostile child d… + | ^ + 119| void invoked.then(() => { clearTimeout(timeout); resolve(); }, rej… + 120| }); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[20/44]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > runs hn-monitor analyze-story end-to-end via a stub agent CLI (gate 2 clause 2 demo) +AssertionError: expected { …(12) } to match object { output: { …(3) }, …(1) } +(22 matching properties omitted from actual) + +- Expected ++ Received + + Object { +- "output": Object { +- "reasoning": "stub agent runtime — deterministic output for gate-2 clause-2 demo", +- "relevance_score": 5, +- "story_title": "stub", +- }, ++ "output": null, + "verification": Object { +- "gate": "json_schema", +- "verdict": "pass", ++ "gate": "execution", ++ "verdict": "fail", + }, + } + + ❯ tests/live-kernel.test.ts:657:36 + 655| && (entry as { step_id?: string }).step_id === 'analyze-story', + 656| ) as { payload: { output: unknown; verification: unknown } } | und… + 657| expect(stepCompleted?.payload).toMatchObject({ + | ^ + 658| output: { + 659| story_title: 'stub', + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[21/44]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story FAILS verification when the CLI omits required schema fields +AssertionError: expected { …(12) } to match object { …(3) } +(21 matching properties omitted from actual) + +- Expected ++ Received + + Object { +- "completionReason": "retries_exhausted", ++ "completionReason": "worker_error", + "output": null, + "verification": Object { +- "gate": "json_schema", ++ "gate": "execution", + "verdict": "fail", + }, + } + + ❯ tests/live-kernel.test.ts:752:36 + 750| // its verification record names the json_schema rejection. The re… + 751| // parsed value is nulled before the completion is persisted. + 752| expect(stepCompleted?.payload).toMatchObject({ + | ^ + 753| completionReason: 'retries_exhausted', + 754| output: null, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[22/44]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > agent step preserves the CliResult wrapper as output when the CLI emits non-JSON text +AssertionError: expected null not to be null + ❯ tests/live-kernel.test.ts:823:24 + 821| // here (parseJsonOutput returned null on non-JSON stdout) and + 822| // these assertions would all fail. + 823| expect(output).not.toBeNull(); + | ^ + 824| expect(output.exit_code).toBe(0); + 825| expect(output.stdout_tail).toContain('looked at the story'); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[23/44]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker exposes wake_context to the CLI via RELAYFLOW_WAKE_CONTEXT env var (real analyzer prerequisite) +TypeError: Cannot read properties of null (reading 'story_title') + ❯ tests/live-kernel.test.ts:891:42 + 889| ) as { payload: { output: { story_title: string; reasoning: string… + 890| expect(stepCompleted).toBeDefined(); + 891| expect(stepCompleted!.payload.output.story_title).toBe(`echoed:${s… + | ^ + 892| expect(stepCompleted!.payload.output.reasoning).toContain(String(s… + 893| + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[24/44]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_WAKE_CONTEXT UNSET when the run has no wake_context (undefined-vs-null pin) +TypeError: Cannot read properties of null (reading 'env_present') + ❯ tests/live-kernel.test.ts:958:38 + 956| ) as { payload: { output: { env_present: boolean } } } | undefined; + 957| expect(completed).toBeDefined(); + 958| expect(completed!.payload.output.env_present).toBe(false); + | ^ + 959| + 960| delete process.env.RELAYFLOW_WAKE_CONTEXT; + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[25/44]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model +TypeError: Cannot read properties of null (reading 'story_title') + ❯ tests/live-kernel.test.ts:1194:38 + 1192| expect(completed).toBeDefined(); + 1193| // UNSET, not EMPTY and not the leaked parent value. + 1194| expect(completed!.payload.output.story_title).toBe('model:UNSET'); + | ^ + 1195| + 1196| delete process.env.RELAYFLOW_MODEL; + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[26/44]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +Error: LIVE_ANALYZER_UNAVAILABLE: "/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/preflight/analyze-story-claude-cli" does not identify as relayflows-agent-cli-v1 — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ❯ tests/live-kernel.test.ts:1223:15 + 1221| const notice = `LIVE_ANALYZER_UNAVAILABLE: ${readiness.detail}`; + 1222| if (process.env['RELAYFLOWS_ALLOW_ANALYZER_SKIP'] !== '1') { + 1223| throw new Error( + | ^ + 1224| `${notice} — failing because gate-2 acceptance requires the … + 1225| + 'Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is … + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[27/44]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir +AssertionError: WARNING [unprovable_effects] Step "greet" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "shout" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +REFUSED [relayflowd_not_found] No relayflowd binary could be found. Install the runtime package for this host (@relayflows/runtime-linux-x64), or set RELAYFLOWD_BIN to a relayflowd executable. Tried: /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/relayflowd. +: expected 2 to be +0 // Object.is equality + +- Expected ++ Received + +- 0 ++ 2 + + ❯ tests/live-kernel.test.ts:1388:40 + 1386| ]); + 1387| + 1388| expect(first.status, first.stderr).toBe(0); + | ^ + 1389| expect(second.status, second.stderr).toBe(0); + 1390| expect(first.stdout).toContain('completionReason: success'); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[28/44]⎯ + + FAIL tests/live-kernel.test.ts > a relayflow can be scheduled: tick source against live relayflowd > a tick spawns a real run whose step reports the SCHEDULED instant +AssertionError: expected null to deeply equal { schedule_id: 'heartbeat-1m', …(3) } + +- Expected: +Object { + "lag_ms": 43000, + "schedule_id": "heartbeat-1m", + "scheduled_for_ms": 1764000000000, + "slot": 29400000, +} + ++ Received: +null + + ❯ tests/live-kernel.test.ts:1739:39 + 1737| // The bound: the run reports the grid instant and its own lag, so… + 1738| // backfilled run can tell it is running for a slot from the past. + 1739| expect(completed!.payload.output).toEqual({ + | ^ + 1740| schedule_id: 'heartbeat-1m', + 1741| slot: 29_400_000, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[29/44]⎯ + + FAIL tests/provider-trigger-executor.test.ts > the kernel executes compiled 'app_mention' subscriptions with provider isolation and durable dedupe + FAIL tests/provider-trigger-executor.test.ts > the kernel executes compiled 'reaction_added' subscriptions with provider isolation and durable dedupe + FAIL tests/provider-trigger-executor.test.ts > the kernel executes compiled 'pull_request' subscriptions with provider isolation and durable dedupe +Error: spawnSync /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ submit tests/provider-trigger-executor.test.ts:43:89 + 41| steps: [{ id: 'effect', type: 'deterministic', command: `printf ac… + 42| })))); + 43| const submit = (envelope: unknown, key: string, executor = source.na… + | ^ + 44| '--data-dir', dir, 'run', spec, '--event', JSON.stringify({ type: … + 45| ], { encoding: 'utf8', stdio: 'pipe' })) as { matched: boolean; dedu… + ❯ tests/provider-trigger-executor.test.ts:50:12 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[30/44]⎯ + + FAIL tests/webhook-live.test.ts > executes and deduplicates 'app_mention' only for its provider and matching payload + FAIL tests/webhook-live.test.ts > executes and deduplicates 'reaction_added' only for its provider and matching payload + FAIL tests/webhook-live.test.ts > executes and deduplicates 'pull_request' only for its provider and matching payload +Error: webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ until tests/webhook-live.test.ts:39:9 + 37| const deadline = Date.now() + 10_000; + 38| while (Date.now() < deadline) { if (await predicate()) return; await… + 39| throw new Error(`webhook integration timed out: ${detail()}`); + | ^ + 40| } + 41| async function daemon(dir: string): Promise { + ❯ daemon tests/webhook-live.test.ts:43:3 + ❯ tests/webhook-live.test.ts:100:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[31/44]⎯ + + FAIL tests/webhook-live.test.ts > flows serve-webhook writes JSON before the daemon starts, then journals and archives exactly once +Error: webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ until tests/webhook-live.test.ts:39:9 + 37| const deadline = Date.now() + 10_000; + 38| while (Date.now() < deadline) { if (await predicate()) return; await… + 39| throw new Error(`webhook integration timed out: ${detail()}`); + | ^ + 40| } + 41| async function daemon(dir: string): Promise { + ❯ daemon tests/webhook-live.test.ts:43:3 + ❯ tests/webhook-live.test.ts:121:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[32/44]⎯ + + FAIL tests/webhook-live.test.ts > replays a dropped file after SIGKILL before spawn +Error: webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ until tests/webhook-live.test.ts:39:9 + 37| const deadline = Date.now() + 10_000; + 38| while (Date.now() < deadline) { if (await predicate()) return; await… + 39| throw new Error(`webhook integration timed out: ${detail()}`); + | ^ + 40| } + 41| async function daemon(dir: string): Promise { + ❯ daemon tests/webhook-live.test.ts:43:3 + ❯ tests/webhook-live.test.ts:137:17 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[33/44]⎯ + + FAIL tests/webhook-live.test.ts > resumes the same journal after SIGKILL after spawn and before acknowledgement +Error: webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ until tests/webhook-live.test.ts:39:9 + 37| const deadline = Date.now() + 10_000; + 38| while (Date.now() < deadline) { if (await predicate()) return; await… + 39| throw new Error(`webhook integration timed out: ${detail()}`); + | ^ + 40| } + 41| async function daemon(dir: string): Promise { + ❯ daemon tests/webhook-live.test.ts:43:3 + ❯ tests/webhook-live.test.ts:150:17 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[34/44]⎯ + +⎯⎯⎯⎯⎯⎯ Unhandled Errors ⎯⎯⎯⎯⎯⎯ + +Vitest caught 2 unhandled errors during the test run. +This might cause false positive tests. Resolve unhandled errors to make sure your tests are not affected. + +⎯⎯⎯⎯ Unhandled Rejection ⎯⎯⎯⎯⎯ +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-protocol.test.ts:454:17 + ❯ node_modules/@vitest/runner/dist/index.js:533:5 + ❯ runTest node_modules/@vitest/runner/dist/index.js:1056:11 + ❯ runSuite node_modules/@vitest/runner/dist/index.js:1205:15 + ❯ runSuite node_modules/@vitest/runner/dist/index.js:1205:15 + ❯ runFiles node_modules/@vitest/runner/dist/index.js:1262:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +This error originated in "tests/hosted-extension-protocol.test.ts" test file. It doesn't mean the error was thrown inside the file itself, but while it was running. +The latest test that might've caused the error is "rejects two forged calls after the authoritative first outcome settles". It might mean one of the following: +- The error was thrown, while Vitest was running this test. +- If the error occurred after the test had been completed, this was the last documented test before it was thrown. + +⎯⎯⎯⎯⎯ Uncaught Exception ⎯⎯⎯⎯⎯ +Error: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/release/relayflowd ENOENT + ❯ Process.ChildProcess._handle.onexit node:internal/child_process:285:19 + ❯ onErrorNT node:internal/child_process:483:16 + ❯ processTicksAndRejections node:internal/process/task_queues:90:21 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { errno: -2, code: 'ENOENT', syscall: 'spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/release/relayflowd', path: '/home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/release/relayflowd', spawnargs: [ '--data-dir', '/tmp/flows-mcp-daemon-4iiY74', 'serve' ] } +This error originated in "tests/mcp.test.ts" test file. It doesn't mean the error was thrown inside the file itself, but while it was running. +The latest test that might've caused the error is "authored MCP effects against the real kernel". It might mean one of the following: +- The error was thrown, while Vitest was running this test. +- If the error occurred after the test had been completed, this was the last documented test before it was thrown. +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ + + Test Files 9 failed | 186 passed | 3 skipped (198) + Tests 42 failed | 3174 passed | 26 skipped (3242) + Errors 2 errors + Start at 06:13:21 + Duration 271.86s (transform 4.22s, setup 0ms, collect 57.58s, tests 711.01s, environment 26ms, prepare 8.10s) + diff --git a/evidence/561/tsconfig.tests.json b/evidence/561/tsconfig.tests.json new file mode 100644 index 00000000..218208a8 --- /dev/null +++ b/evidence/561/tsconfig.tests.json @@ -0,0 +1,14 @@ +{ + "extends": "../../packages/sdk/tsconfig.tests.json", + "include": [ + "../../packages/sdk/src/**/*.ts", + "../../packages/sdk/tests/authored-probe-cache.test.ts", + "../../packages/sdk/tests/preflight-run-cache.test.ts" + ], + "compilerOptions": { + "typeRoots": [ + "../../packages/sdk/node_modules/@types", + "../../packages/sdk/node_modules" + ] + } +} diff --git a/evidence/561/typecheck.txt b/evidence/561/typecheck.txt new file mode 100644 index 00000000..cb3a7722 --- /dev/null +++ b/evidence/561/typecheck.txt @@ -0,0 +1,4 @@ + +> @relayflows/sdk@2.0.30 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + diff --git a/summary.md b/summary.md index e92fc88e..5dab5700 100644 --- a/summary.md +++ b/summary.md @@ -1,204 +1,50 @@ -# Spend the analysis `flows check` already does +# Cache authored CLI preflight per run (#561, F1) -`flows check` computed the facts that would have prevented five dead runs and -said nothing about them. This spends two of them, and establishes that the -third fact is not a fact. +Nine concurrent `f.llm` calls synchronously probed the same CLI nine times before worker-slot admission. Slow probes blocked the worker's event loop long enough for already-issued 30-second leases to expire. `WorkerSlots` admission is unchanged. -## What changed +Reuse preflight's existing `(cli, source, model, execution)` cache across one authored run, for both LLM and agent calls. Read project configuration lazily once per runner. Keep the synchronous public preflight API and existing fail-closed resolution checks. -### `agent_worker_unresolved` — a check-only warning (permanent) +Auth/model readiness is now checked once per key per run, matching declarative execution. Auth changes during a run are detected by execution rather than another preflight; failed probes are cached too, so later calls receive the refusal instead of repeating a timeout. New runs probe again. -A spec with `agent` steps needs a worker *attached for step type `agent`*. -Without one the run parks at the first such step. `flows check` already walks -those steps to print `REQUIRES codex (step "implement"), …` — the analysis -exists; it just stopped one sentence short of the remedy. +F2 (asynchronous probing) is deferred to [#574](https://github.com/AgentWorkforce/flows/issues/574). Distinct keys still require distinct synchronous probes. **One hung probe blocks for the model-readiness timeout: 60 s. That exceeds the 30 s lease and will still kill a concurrent step.** This focused fix does not establish the broader invariant that no synchronous probe runs while any lease is live. -``` -REQUIRES claude (step "implement") -WARNING [agent_worker_unresolved] 2 agent steps ("implement", "review") require an -attached worker. For a local run, use `flows run --local-agent `, unless you -already attach an agent worker for this daemon. `flows check` does not verify -worker attachment: with no worker attached the run parks at the first agent step. -``` +Validation commands and literal captured output are in [evidence/561/README.md](evidence/561/README.md). Regression coverage uses a real kernel and slow fake CLI at capacity 1 and default capacity 4; it inspects every child journal, bounds session overlap, counts probes, checks failed-probe caching, agent parity, and fresh-run isolation. A separate test pins CLI/source/model cache keys. -Design decisions worth naming: - -- **Warning, never a refusal, worded as a requirement rather than a - prediction.** Worker attachment is *unknown* to `flows check`, not absent: - `check` is daemon-free by construction, so it cannot see a worker attached - elsewhere. Saying "this will park" would be a guess. -- **It is a property of the invocation, not of the spec,** so it opts in - through a new `CheckInvocation` seam on `checkFlow` / `checkAuthoredFlow` - rather than entering `preflight`, which stays a pure function of the spec - plus environment probes. Only `flows check` sets it. `flows run` knows the - answer (it attaches its own worker under `--local-agent`), `flows build` and - `flows deploy` check a spec that will run elsewhere, and an SDK caller - reaching `checkAuthoredFlow` directly is generally running a worker already. -- **Scoped to `agent` steps.** `--local-agent` attaches no `llm` worker on the - YAML path, so counting `llm` steps and naming that flag for them would be - false. They are not counted and not mentioned. -- **Counted from the compiled steps, not from `requirements`.** A YAML helper - step (`slack: { post: … }`) compiles to an `agent` step carrying a helper - envelope and needs the same SDK worker, but `requirements` reports it as an - integration, dedupes by harness, and includes `llm` use. Compilation is the - only walk that sees every step that needs the worker. -- **Emitted in the `REQUIRES` position,** because it reads as a footnote to - that line. `emitCheckReport` holds it back from the leading diagnostic batch - and emits it once, after `REQUIRES` and before `CHECK PASSED`; diagnostics - stay on stderr, report lines stay on stdout. `--json` returns before any of - that and keeps the single ordered `diagnostics` array. -- **Emitted alongside a preflight refusal.** An environment refusal is fixed - and rerun; the worker question is still open on the next pass, and staying - silent about it is what made an author meet it one dead run at a time. - -Known blind spot, documented rather than papered over: an authored `.flow.ts` -is checked through `checkMcpHeader`, which preflights a synthetic one-step -header spec without compiling the body, so there are no agent steps to count. -Same blind spot as `permissions_unenforced`. - -### `gate_path_unscanned` — a preflight warning (temporary, retires with #513) - -An `artifact_exists` gate reads the journaled `output.artifacts` list and -nothing else. The bundled worker's *scan* skips any entry whose name starts -with `.` and any entry named exactly `node_modules`, so a gate on a path inside -one of those prefixes cannot rest on the scan, however faithfully the agent -writes the file: +Acceptance: +- [x] Nine concurrent fake-CLI calls at capacity 1 and default: success, one attempt per child, no `lease_expired`. +- [x] The issue's exact live-Claude repro succeeds locally at both capacities; journal outputs are 4 through 12. +- [ ] Restore `Promise.all` in prompt-lab: both requested files are absent from this checkout. No example restoration is claimed. +- [x] Mutation verification: removing the cache argument fails all five tests; restoring the committed bytes passes all five. + +The full SDK suite is **not green**. `npm test` captured: + +```text + Test Files 9 failed | 186 passed | 3 skipped (198) + Tests 42 failed | 3174 passed | 26 skipped (3242) + Errors 2 errors ``` -WARNING [gate_path_unscanned] Step "review" gates on artifact_exists path -".workflow-artifacts/rust/review.md", but the bundled agent worker's artifact scan -records nothing under ".workflow-artifacts": it skips entries whose name starts with -"." and entries named "node_modules". The gate reads the journaled output.artifacts -and never the disk, so writing the file is not enough: the step has to report the -path itself — as object-shaped JSON stdout carrying its own "artifacts" array, as a -completed Relay task output, or from a custom worker. If the gate is meant to rest -on the scan, write the artifact to a path the scan records. -``` -The issue asked for a refusal, on the premise that such a gate is statically -unsatisfiable. That premise is false, and review F1 is right to reject it: the -scan is only one writer of `output.artifacts`. The same **bundled** worker -promotes object-shaped JSON stdout — and a completed Relay task's output — -verbatim into `output` (`worker.ts`), so an agent that answers with its own -`{"artifacts": [...]}` puts a hidden path in the list and the gate passes; and -`step.complete` accepts any `output` from any worker. Which route a step takes -is a run-time fact, so this reports the scan's limitation and leaves the verdict -to the run. A regression runs the real `AgentWorker` over a fake `claude` that -writes `.workflow-artifacts/review.md` *and* reports it as JSON, then runs the -real lowered gate command over exactly what the worker journaled: the scan -records nothing, the gate exits 0. A refusal would have rejected that spec at -`check`, `run`, `build` and every SDK submission. - -- **One rule, two consumers.** The exclusion predicate moved out of - `agent-artifacts.ts` into `src/artifact-scan-policy.ts`; the real scan now - consumes it, so the warning cannot drift from the scan it describes. A - parity test writes all twelve case paths to a temp dir and asserts the real - `snapshotWorkspaceFiles` output is exactly the predicate's complement. -- **The whole excluded prefix is named,** not the offending segment alone — - that prefix is the directory an author relying on the scan has to move the - artifact out of. -- **Segments are compared exactly, with no normalization.** The gate matches - the author's literal string against the worker's literal list, so - `node_modules-copy/out.md`, `reports/node_modules.md`, `review.md` and - `reports/v1.2/review.md` say nothing, and a backslash is an ordinary - filename character rather than a separator. -- **Collected before preflight's early returns.** `probeNamedGate` runs after - CLI resolution, model governance, scope and budget can each return, so an - author with an unresolved CLI would not learn about the unscanned path until - a later pass — or at run time. A test pins the diagnostic order as - `['gate_path_unscanned', 'cli_unresolved']`. -- **It lives in `PREFLIGHT_WARNING_KINDS`,** so it travels with every public - preflight consumer — `flows check`, `run`, `build` and SDK submissions — and - refuses none of them. - -### `gate_output_not_captured` — **not added**, because it is not true - -The acceptance criterion was conditional: warn that a `subprocess_gate`'s -output is not captured *"for as long as that is true"*. It is not true today. -Verified against a live `relayflowd` before writing anything: +The full output is [sdk-suite.txt](evidence/561/sdk-suite.txt). Failures include unavailable bubblewrap, Bun 1.3.6 versus required 1.4.0, missing default kernel paths, and existing worker/gate expectations. These failures were not fixed or fully baseline-classified in this focused change. The kernel suite's complete command/output is [documented separately](evidence/561/README.md#broader-checks). + +Mutation command (from `packages/sdk`, before and after restore): +```sh +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npx vitest run tests/authored-probe-cache.test.ts ``` -"stepId": "emit.gate", "completionReason": "retries_exhausted", "exitCode": 1, -"stdoutTail": "GATE_STDOUT_MARKER\n", "stderrTail": "GATE_STDERR_MARKER\n" + +Captured failure summary (full [output](evidence/561/mutation-red.txt)): + +```text + Test Files 1 failed (1) + Tests 5 failed (5) ``` -and in the journal itself, on the lowered gate step's `step.completed`: +Captured restored summary (full [output](evidence/561/mutation-green.txt)): -```json -"output": { "exit_code": 1, "stdout_tail": "GATE_STDOUT_MARKER\n", - "stderr_tail": "GATE_STDERR_MARKER\n" } +```text + Test Files 1 passed (1) + Tests 5 passed (5) ``` -A warning claiming otherwise would have been a false diagnostic added to a -change whose whole point is that `check` should only say what it knows. What -the criterion actually asks for is that the warning exist exactly while the -bug does — so instead of the warning, this adds a **live-kernel regression -that pins the capture**. If capture ever regresses, that test fails, and the -warning becomes warranted at the moment it becomes true. - -## Tests - -- `tests/artifact-gates.test.ts` — six parameterised warnings asserting the - full excluded prefix is named and that none of them refuses; six lookalike - paths that must say nothing; the scan/predicate parity test against a real - `snapshotWorkspaceFiles` run; the early-return ordering test; an end-to-end - `checkFlow` case pinning the warning and the unrelated refusal beside it; - and the F1 regression described above, which drives the real `AgentWorker` - and the real lowered gate command over an excluded path the agent reports - itself. -- `tests/check-worker-surface.test.ts` (new) — message content, singular vs - plural and the "and N more" truncation, exit code 0, the YAML helper step - being counted where `requirements` omits it, silence for `llm`-only and - deterministic-only flows, survival alongside a preflight refusal, opt-in - discipline, `--json` emitting it once in both the payload and on stderr, and - an **ordered `CliIo` transcript** (both streams in one list) pinning the - position after `REQUIRES` and before `CHECK PASSED` — in both the - `REQUIRES`-present and `REQUIRES`-absent shapes. -- `tests/preflight.test.ts` — `gate_path_unscanned` added to the - warning-reachability list, which asserts set equality against - `PREFLIGHT_WARNING_KINDS`. -- `tests/live-kernel.test.ts` (new block) — the `subprocess_gate` capture - regression, asserting both the rendered message and the journal payload, with - a silent-gate control so the assertions cannot pass on a fixed string. - -`npm test` in `packages/sdk`: 2381 passed, 40 failed. All 40 failures are -pre-existing and environmental — they reproduce identically on a stashed tree -(same 40 failures, same 7 files): `relayflowd` looked up under -`kernel/target/{debug,release}/` while this toolchain builds outside the repo, -and flows needing real harness CLIs. `tsc --noEmit` is clean for `src`, the -type tests, and `tests`. - -## Files - -| File | Change | -| --- | --- | -| `src/artifact-scan-policy.ts` | new — the scan's exclusion rule as a pure predicate, shared | -| `src/named-gate-preflight.ts` | new — the `gate_path_unscanned` warning (retires with #513) | -| `src/cli/check-worker-surface.ts` | new — the `agent_worker_unresolved` warning | -| `src/agent-artifacts.ts` | consumes the extracted predicate instead of its own copy | -| `src/preflight.ts` | collects gate scan coverage before every early return | -| `src/failure-kinds.ts` | the two new kinds, each with its retirement note | -| `src/cli/check.ts` | `CheckInvocation` opt-in seam | -| `src/cli.ts` | opts in at the `check` dispatch; defers the warning to the `REQUIRES` position | -| `docs/SURFACE.md` | documents both kinds where the facts they describe already live | - -Each diagnostic is a distinct kind, so it can be suppressed and later deleted -on its own. Both temporary kinds carry their retirement condition in a comment -at the definition site. - -## Out of scope, unchanged - -The underlying bugs (#511, #513) are not fixed, and the spec is not -round-tripped past the daemon's validator (#502). - -## Merge with origin/main (conflict resolution note) - -Main deliberately broadened the artifact scan so dot-directories like -`.workflow-artifacts/` are journaled and gate-able; only exact names `.git`, -`.relayflowd` and `node_modules` are skipped, at any depth. This branch's -shared `artifact-scan-policy.ts` predicate now encodes THAT set (not the older -dot-prefix rule), so `gate_path_unscanned` warns only on paths the merged scan -can never record — `node_modules/**`, `.git/**`, `.relayflowd/**` — and the -`.workflow-artifacts/` gates main enabled produce no warning. `cli.ts` keeps -both the `agent_worker_unresolved` deferral and main's model-provenance -helper; `agent-artifacts.ts` keeps the shared-predicate import. +The mutation fails the journal assertion itself at both capacities, with `lease_expired` in the child journal. [Restore evidence](evidence/561/restore.txt) includes the literal clean diff result and identical SHA-256 hashes. No workflow files changed. From 1ccaaedcee9bac234cedf09913ab50d4ed6a61b9 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Thu, 24 Sep 2026 06:40:14 +0000 Subject: [PATCH 04/11] test(sdk): keep the artifact-gate cwd regression inside the run root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The regression added by #517 declares `cwd` on a spec it hands to `preflight`, and used an `os.tmpdir()` directory — an absolute path. #566 landed one commit earlier and made an absolute `cwd` a compile refusal (`agent-cwd.ts`: a declared `cwd` is run-root-relative, the same rule `relayflowd_core::spec::is_run_root_relative_path` applies at the kernel boundary). Each PR was green alone; together they are not, and `main` at e30226c fails this test with `invalid_spec` where it expects `gate_path_unscanned`. The fixture now makes its directory inside the run root and declares the relative name. Nothing else moves: the warning, `ok`, the real `AgentWorker` dispatch, the empty scan snapshot, the journaled JSON output and the lowered gate command are asserted exactly as before. Reverting this file to its e30226c bytes fails the case and restoring it passes; both captures are in evidence/561/artifact-gates-{red,green}.txt. Co-Authored-By: Claude Opus 5 --- packages/sdk/tests/artifact-gates.test.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/sdk/tests/artifact-gates.test.ts b/packages/sdk/tests/artifact-gates.test.ts index 55a68963..ca650309 100644 --- a/packages/sdk/tests/artifact-gates.test.ts +++ b/packages/sdk/tests/artifact-gates.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'node:events'; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { basename, dirname, join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; // A fake `claude` whose "work" is whatever the test's `onSpawn` hook writes @@ -52,6 +52,17 @@ import { execFileSync } from 'node:child_process'; const dirs: string[] = []; afterEach(() => { onSpawn = undefined; claudeResult = 'done'; for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); }); function tempDir(): string { const d = mkdtempSync(join(tmpdir(), 'artifact-gates-')); dirs.push(d); return d; } +/** + * A temp directory INSIDE the run root, returned with the run-root-relative + * name a spec may declare. A `cwd` in a spec is run-root-relative by contract + * (`agent-cwd.ts`), so a `tempDir()` under `os.tmpdir()` cannot be named by + * one; only tests that hand the worker a directory directly can use that. + */ +function runRootDir(): { directory: string; declared: string } { + const directory = mkdtempSync(join(process.cwd(), 'artifact-gates-')); + dirs.push(directory); + return { directory, declared: basename(directory) }; +} describe('worker-side artifacts', () => { it('journals the files the CLI created or changed in its cwd, content-hashed, including dot-directories, with .git and node_modules excluded', async () => { @@ -301,7 +312,7 @@ describe('artifact_exists named gate: static scan coverage', () => { * submission. */ it('does not refuse a gate the bundled worker itself can satisfy through JSON output', async () => { - const cwd = tempDir(); + const { directory: cwd, declared } = runRootDir(); const path = 'node_modules/review.md'; onSpawn = (dir) => { mkdirSync(join(dir!, 'node_modules'), { recursive: true }); @@ -309,7 +320,7 @@ describe('artifact_exists named gate: static scan coverage', () => { }; claudeResult = JSON.stringify({ artifacts: [path] }); const authored = { version: '0.1.0', name: 'x', steps: [{ id: 'review', type: 'agent', instruction: 'i', - cli: 'claude', cwd, verification: { type: 'artifact_exists', path } }] }; + cli: 'claude', cwd: declared, verification: { type: 'artifact_exists', path } }] }; // `claude` carries a default model, so its probe has to answer the // model-scoped readiness question too; nothing else about it matters here. From 100cd17caec0b8a4f12307276a145aa52da19271 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Thu, 24 Sep 2026 06:54:23 +0000 Subject: [PATCH 05/11] docs: classify every check failure against main and re-capture #561 evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repository check on this branch failed with 30 tests across five files. None came from #561. Each is classified by reverting this branch's three source files to origin/main (e30226c), re-running and restoring: * artifact-gates (1) — main is red; #566 and #517 conflict semantically. Fixed in the preceding commit. * live-kernel (7) — this sandbox's HOME declares "type": "commonjs" above the checkout, so testdata/preflight's extensionless ESM fixture CLIs load as CommonJS and emit nothing, silently. Local setup only. * authored-node-runtime — Bun 1.3.6 where every workflow pins 1.4.0. * hosted-extension (22) — unprivileged user namespaces denied to this container; bwrap cannot run even once installed. The mutation is re-run at this head. The mutated run reproduces the issue's exact signature — lease_expired on a first attempt that never heartbeated, retry, second attempt success — and the restore is byte-identical by SHA-256. The live-Claude repro could not be re-run: this environment's claude is no longer authenticated. The capture says so rather than the acceptance box claiming a pass it cannot show. Co-Authored-By: Claude Opus 5 --- evidence/561/artifact-gates-green.txt | 11 ++ evidence/561/artifact-gates-red.txt | 36 ++++++ .../561/live-one-recheck-unauthenticated.txt | 28 +++++ evidence/561/mutation-green-rerun.txt | 16 +++ evidence/561/mutation-red-rerun.txt | 110 ++++++++++++++++++ summary.md | 41 +++++-- 6 files changed, 232 insertions(+), 10 deletions(-) create mode 100644 evidence/561/artifact-gates-green.txt create mode 100644 evidence/561/artifact-gates-red.txt create mode 100644 evidence/561/live-one-recheck-unauthenticated.txt create mode 100644 evidence/561/mutation-green-rerun.txt create mode 100644 evidence/561/mutation-red-rerun.txt diff --git a/evidence/561/artifact-gates-green.txt b/evidence/561/artifact-gates-green.txt new file mode 100644 index 00000000..471e37b4 --- /dev/null +++ b/evidence/561/artifact-gates-green.txt @@ -0,0 +1,11 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/artifact-gates.test.ts (27 tests) 259ms + + Test Files 1 passed (1) + Tests 27 passed (27) + Start at 06:37:46 + Duration 1.89s (transform 840ms, setup 0ms, collect 1.46s, tests 259ms, environment 0ms, prepare 44ms) + +EXIT=0 diff --git a/evidence/561/artifact-gates-red.txt b/evidence/561/artifact-gates-red.txt new file mode 100644 index 00000000..3623239c --- /dev/null +++ b/evidence/561/artifact-gates-red.txt @@ -0,0 +1,36 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/artifact-gates.test.ts (27 tests | 1 failed) 248ms + × artifact_exists named gate: static scan coverage > does not refuse a gate the bundled worker itself can satisfy through JSON output 10ms + → expected [ 'invalid_spec' ] to deeply equal [ 'gate_path_unscanned' ] + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/artifact-gates.test.ts > artifact_exists named gate: static scan coverage > does not refuse a gate the bundled worker itself can satisfy through JSON output +AssertionError: expected [ 'invalid_spec' ] to deeply equal [ 'gate_path_unscanned' ] + +- Expected ++ Received + + Array [ +- "gate_path_unscanned", ++ "invalid_spec", + ] + + ❯ tests/artifact-gates.test.ts:318:50 + 316| const checked = preflight(authored as never, { probes: { ...probes… + 317| cli: () => ({ exists: true, authenticated: true, modelAvailable:… + 318| expect(checked.diagnostics.map(d => d.kind)).toEqual(['gate_path_u… + | ^ + 319| expect(checked.ok).toBe(true); + 320| + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 26 passed (27) + Start at 06:37:40 + Duration 1.90s (transform 820ms, setup 0ms, collect 1.48s, tests 248ms, environment 0ms, prepare 42ms) + +EXIT=1 diff --git a/evidence/561/live-one-recheck-unauthenticated.txt b/evidence/561/live-one-recheck-unauthenticated.txt new file mode 100644 index 00000000..e062844b --- /dev/null +++ b/evidence/561/live-one-recheck-unauthenticated.txt @@ -0,0 +1,28 @@ +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.03s completionReason: success +○ llm-2 (llm) 0.00s +○ llm-3 (llm) 0.00s +○ llm-4 (llm) 0.00s +○ llm-5 (llm) 0.00s +○ llm-6 (llm) 0.00s +○ llm-7 (llm) 0.00s +○ llm-8 (llm) 0.00s +○ llm-9 (llm) 0.00s +○ llm-10 (llm) 0.00s +✗ llm-2 (llm) 1.04s +✗ llm-3 (llm) 0.12s +✗ llm-4 (llm) 0.10s +✗ llm-5 (llm) 0.08s +✗ llm-6 (llm) 0.07s +✗ llm-7 (llm) 0.05s +✗ llm-8 (llm) 0.04s +✗ llm-9 (llm) 0.03s +✗ llm-10 (llm) 0.01s +REFUSED [invalid_spec] llm_cli_unresolved: Step "llm-2" declares CLI "claude", but "claude auth status" exited non-zero (exit 1); authenticate it or repair that adapter's authentication probe. It reported: { + "loggedIn": false, + "authMethod": "none", + "apiProvider": "firstParty", + "analyticsDisabled": false, + "projectsDirectory": "/home/daytona/.claude/projects", + "configDirectory": "/home/daytona/.claude" +} diff --git a/evidence/561/mutation-green-rerun.txt b/evidence/561/mutation-green-rerun.txt new file mode 100644 index 00000000..ef03e378 --- /dev/null +++ b/evidence/561/mutation-green-rerun.txt @@ -0,0 +1,16 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-probe-cache.test.ts (5 tests) 21193ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 1 7215ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 4 5382ms + ✓ authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 6922ms + ✓ authored run CLI probe cache > caches probe failures while refusing all nine calls 400ms + ✓ authored run CLI probe cache > shares probe results across agent calls too 1272ms + + Test Files 1 passed (1) + Tests 5 passed (5) + Start at 06:39:32 + Duration 22.85s (transform 923ms, setup 0ms, collect 1.49s, tests 21.19s, environment 0ms, prepare 46ms) + +EXIT=0 diff --git a/evidence/561/mutation-red-rerun.txt b/evidence/561/mutation-red-rerun.txt new file mode 100644 index 00000000..6627bd81 --- /dev/null +++ b/evidence/561/mutation-red-rerun.txt @@ -0,0 +1,110 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +(node:67055) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M39259MZHZXZ7X44NHZCS8W6 step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M39259MZHZXZ7X44NHZCS8W6/llm-1. +(Use `node --trace-warnings ...` to show where the warning was created) +(node:67055) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M3926E60P3W8EP2AMJQYZG26 step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M3926E60P3W8EP2AMJQYZG26/llm-1. + ❯ tests/authored-probe-cache.test.ts (5 tests | 5 failed) 87110ms + × authored run CLI probe cache > nine calls have no expired attempts at capacity 1 37494ms + → 01M39259MZHZXZ7X44NHZCS8W6.sqlite3: expected '[{"at_ms":1790231881379,"attempt":nul…' not to contain 'lease_expired' + × authored run CLI probe cache > nine calls have no expired attempts at capacity 4 37695ms + → 01M3926E60P3W8EP2AMJQYZG26.sqlite3: expected '[{"at_ms":1790231918788,"attempt":nul…' not to contain 'lease_expired' + × authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 6620ms + → expected 3272.878248000008 to be less than 1000 + × authored run CLI probe cache > caches probe failures while refusing all nine calls 3293ms + → expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9 + × authored run CLI probe cache > shares probe results across agent calls too 2005ms + → expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 5 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 1 +AssertionError: 01M39259MZHZXZ7X44NHZCS8W6.sqlite3: expected '[{"at_ms":1790231881379,"attempt":nul…' not to contain 'lease_expired' + +Expected: "lease_expired" +Received: "[{"at_ms":1790231881379,"attempt":null,"entry_type":"run.spawned","payload":{"created_by":"protocol-v0","journal_version":1,"parent_run_id":null,"spec":{"name":"nine-llms/llm-1","steps":[{"cli":"/tmp/flows-chain-rdhewz/adapter.mjs","depends_on":[],"id":"llm-1","max_iterations":1,"model":"test-model","prompt":"Return JSON for 0","retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"llm","verification":{"json_schema":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}}}],"version":"0.1.0"},"spec_hash":"349f0346dd65d309f62411ba0662a8dd0b447c1885688723f1466b637c8a56eb"},"run_id":"01M39259MZHZXZ7X44NHZCS8W6","segment_id":1,"seq":1,"step_id":null},{"at_ms":1790231881382,"attempt":null,"entry_type":"step.routed","payload":{"fallbacks_attempted":[],"profile":"attached-worker","provider":"worker"},"run_id":"01M39259MZHZXZ7X44NHZCS8W6","segment_id":1,"seq":2,"step_id":"llm-1"},{"at_ms":1790231881382,"attempt":1,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-84839f6d-0485-47f3-a269-63da47bc137c-llm","idempotency_key":"d715af9bd7ea6db52bd6270e7e21509df2ab943b869dbf6fc86a5c7f9cf83f57","lease_deadline_ms":1790231911382,"lease_id":"01M39259N6YPBF3R9WK4K9VY9W","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M39259MZHZXZ7X44NHZCS8W6","segment_id":1,"seq":3,"step_id":"llm-1"},{"at_ms":1790231911633,"attempt":1,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","tokens_in":0,"tokens_out":0},"completed_by":"kernel","completionReason":"lease_expired","disposition":"retry","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":1790231911633,"output":null,"spend":{"dollars":0,"tokens_input":0,"tokens_output":0,"wallclock_ms":30251},"step_spec_hash":"05227dd556b8cd9c9569a9f21475c1151e4a0c06676a21f4d7b7da12e850083f","verification":null},"run_id":"01M39259MZHZXZ7X44NHZCS8W6","segment_id":1,"seq":4,"step_id":"llm-1"},{"at_ms":1790231911633,"attempt":1,"entry_type":"sleep.until","payload":{"reason":"retry_backoff","wait_id":"01M392676H321D3HJ7VD35ZRZH","wake_at_ms":1790231911633},"run_id":"01M39259MZHZXZ7X44NHZCS8W6","segment_id":1,"seq":5,"step_id":"llm-1"},{"at_ms":1790231911675,"attempt":1,"entry_type":"wait.completed","payload":{"completionReason":"timer_fired","result":null,"wait_id":"01M392676H321D3HJ7VD35ZRZH"},"run_id":"01M39259MZHZXZ7X44NHZCS8W6","segment_id":1,"seq":6,"step_id":"llm-1"},{"at_ms":1790231911678,"attempt":2,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-84839f6d-0485-47f3-a269-63da47bc137c-llm","idempotency_key":"d715af9bd7ea6db52bd6270e7e21509df2ab943b869dbf6fc86a5c7f9cf83f57","lease_deadline_ms":1790231941678,"lease_id":"01M392677Y1TKFEJX8GRQ6SDSC","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M39259MZHZXZ7X44NHZCS8W6","segment_id":1,"seq":7,"step_id":"llm-1"},{"at_ms":1790231914510,"attempt":2,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completed_by":"local-agent-84839f6d-0485-47f3-a269-63da47bc137c-llm","completionReason":"success","disposition":"step_done","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":null,"output":{"x":4},"spend":{"dollars":0,"dollars_unmetered":true,"tokens_input":0,"tokens_output":0,"wallclock_ms":2832},"step_spec_hash":"05227dd556b8cd9c9569a9f21475c1151e4a0c06676a21f4d7b7da12e850083f","verification":{"detail":"all gates passed","gate":"json_schema","verdict":"pass"}},"run_id":"01M39259MZHZXZ7X44NHZCS8W6","segment_id":1,"seq":8,"step_id":"llm-1"},{"at_ms":1790231914513,"attempt":null,"entry_type":"run.completed","payload":{"budget_total":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completionReason":"success","failed_step_id":null},"run_id":"01M39259MZHZXZ7X44NHZCS8W6","segment_id":1,"seq":9,"step_id":null}]" + + ❯ tests/authored-probe-cache.test.ts:82:49 + 80| for (const file of runs) { + 81| const { entries } = await client.journalRead(file.slice(0, -8), … + 82| expect(JSON.stringify(entries), file).not.toContain('lease_expir… + | ^ + 83| expect(entries.filter(e => typeof e === 'object' && e !== null + 84| && 'entry_type' in e && e.entry_type === 'step.attempt.started… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 4 +AssertionError: 01M3926E60P3W8EP2AMJQYZG26.sqlite3: expected '[{"at_ms":1790231918788,"attempt":nul…' not to contain 'lease_expired' + +Expected: "lease_expired" +Received: "[{"at_ms":1790231918788,"attempt":null,"entry_type":"run.spawned","payload":{"created_by":"protocol-v0","journal_version":1,"parent_run_id":null,"spec":{"name":"nine-llms/llm-1","steps":[{"cli":"/tmp/flows-chain-9JP7FZ/adapter.mjs","depends_on":[],"id":"llm-1","max_iterations":1,"model":"test-model","prompt":"Return JSON for 0","retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"llm","verification":{"json_schema":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}}}],"version":"0.1.0"},"spec_hash":"c64bf415ad510199f9776febf4aa0e653b46631c6e47ad089002bab9c081ddb0"},"run_id":"01M3926E60P3W8EP2AMJQYZG26","segment_id":1,"seq":1,"step_id":null},{"at_ms":1790231918792,"attempt":null,"entry_type":"step.routed","payload":{"fallbacks_attempted":[],"profile":"attached-worker","provider":"worker"},"run_id":"01M3926E60P3W8EP2AMJQYZG26","segment_id":1,"seq":2,"step_id":"llm-1"},{"at_ms":1790231918792,"attempt":1,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-dfac5f2e-8efd-432d-90df-d7df28623916-llm","idempotency_key":"edd9d4e816c2f7282bb372d7c981502fc78da8add5d93a7d37251cd55539e2b4","lease_deadline_ms":1790231948792,"lease_id":"01M3926E689AVKYPRKJF16M3RH","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M3926E60P3W8EP2AMJQYZG26","segment_id":1,"seq":3,"step_id":"llm-1"},{"at_ms":1790231948870,"attempt":1,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","tokens_in":0,"tokens_out":0},"completed_by":"kernel","completionReason":"lease_expired","disposition":"retry","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":1790231948870,"output":null,"spend":{"dollars":0,"tokens_input":0,"tokens_output":0,"wallclock_ms":30078},"step_spec_hash":"e470ee9c95d21004139317431d351a47395ef702b6fce01b6ef5c46e258e3c6b","verification":null},"run_id":"01M3926E60P3W8EP2AMJQYZG26","segment_id":1,"seq":4,"step_id":"llm-1"},{"at_ms":1790231948870,"attempt":1,"entry_type":"sleep.until","payload":{"reason":"retry_backoff","wait_id":"01M3927BJ6E5WN1CZKCR7P952B","wake_at_ms":1790231948870},"run_id":"01M3926E60P3W8EP2AMJQYZG26","segment_id":1,"seq":5,"step_id":"llm-1"},{"at_ms":1790231948885,"attempt":1,"entry_type":"wait.completed","payload":{"completionReason":"timer_fired","result":null,"wait_id":"01M3927BJ6E5WN1CZKCR7P952B"},"run_id":"01M3926E60P3W8EP2AMJQYZG26","segment_id":1,"seq":6,"step_id":"llm-1"},{"at_ms":1790231948886,"attempt":2,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-dfac5f2e-8efd-432d-90df-d7df28623916-llm","idempotency_key":"edd9d4e816c2f7282bb372d7c981502fc78da8add5d93a7d37251cd55539e2b4","lease_deadline_ms":1790231978886,"lease_id":"01M3927BJP7FDN6KHV59T4WM7K","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M3926E60P3W8EP2AMJQYZG26","segment_id":1,"seq":7,"step_id":"llm-1"},{"at_ms":1790231951905,"attempt":2,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completed_by":"local-agent-dfac5f2e-8efd-432d-90df-d7df28623916-llm","completionReason":"success","disposition":"step_done","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":null,"output":{"x":4},"spend":{"dollars":0,"dollars_unmetered":true,"tokens_input":0,"tokens_output":0,"wallclock_ms":3019},"step_spec_hash":"e470ee9c95d21004139317431d351a47395ef702b6fce01b6ef5c46e258e3c6b","verification":{"detail":"all gates passed","gate":"json_schema","verdict":"pass"}},"run_id":"01M3926E60P3W8EP2AMJQYZG26","segment_id":1,"seq":8,"step_id":"llm-1"},{"at_ms":1790231951907,"attempt":null,"entry_type":"run.completed","payload":{"budget_total":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completionReason":"success","failed_step_id":null},"run_id":"01M3926E60P3W8EP2AMJQYZG26","segment_id":1,"seq":9,"step_id":null}]" + + ❯ tests/authored-probe-cache.test.ts:82:49 + 80| for (const file of runs) { + 81| const { entries } = await client.journalRead(file.slice(0, -8), … + 82| expect(JSON.stringify(entries), file).not.toContain('lease_expir… + | ^ + 83| expect(entries.filter(e => typeof e === 'object' && e !== null + 84| && 'entry_type' in e && e.entry_type === 'step.attempt.started… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run +AssertionError: expected 3272.878248000008 to be less than 1000 + ❯ tests/authored-probe-cache.test.ts:104:41 + 102| // F1 permits one synchronous probe (~300ms), not nine (~3s). + 103| // Leave process-startup headroom for loaded CI; exact counts belo… + 104| expect(starts.at(-1)! - starts[0]!).toBeLessThan(1_000); + | ^ + 105| expectOneProbe(logs()); + 106| await executeAuthoredFlow(nine, client, undefined, options); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > caches probe failures while refusing all nine calls +AssertionError: expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9 + +- Expected ++ Received + +- 1 ++ 9 + + ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47 + 68| + 69| function expectOneProbe(logs: ReturnType l.kind === 'auth')).toHaveLength(1); + | ^ + 71| expect(logs.filter(l => l.kind === 'identify').length - logs.filter(… + 72| } + ❯ tests/authored-probe-cache.test.ts:123:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > shares probe results across agent calls too +AssertionError: expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3 + +- Expected ++ Received + +- 1 ++ 3 + + ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47 + 68| + 69| function expectOneProbe(logs: ReturnType l.kind === 'auth')).toHaveLength(1); + | ^ + 71| expect(logs.filter(l => l.kind === 'identify').length - logs.filter(… + 72| } + ❯ tests/authored-probe-cache.test.ts:136:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/5]⎯ + + Test Files 1 failed (1) + Tests 5 failed (5) + Start at 06:37:55 + Duration 89.09s (transform 1.09s, setup 0ms, collect 1.79s, tests 87.11s, environment 0ms, prepare 54ms) + +EXIT=1 diff --git a/summary.md b/summary.md index 5dab5700..a5413f14 100644 --- a/summary.md +++ b/summary.md @@ -13,19 +13,38 @@ Validation commands and literal captured output are in [evidence/561/README.md]( Acceptance: - [x] Nine concurrent fake-CLI calls at capacity 1 and default: success, one attempt per child, no `lease_expired`. -- [x] The issue's exact live-Claude repro succeeds locally at both capacities; journal outputs are 4 through 12. +- [x] The issue's exact live-Claude repro succeeded locally at both capacities; journal outputs are 4 through 12. It could **not** be re-run at this head: `claude` is no longer authenticated in this environment (`"loggedIn": false`), captured in [live-one-recheck-unauthenticated.txt](evidence/561/live-one-recheck-unauthenticated.txt). That capture does show one 1.04 s probe followed by eight 0.01–0.12 s cached refusals. - [ ] Restore `Promise.all` in prompt-lab: both requested files are absent from this checkout. No example restoration is claimed. - [x] Mutation verification: removing the cache argument fails all five tests; restoring the committed bytes passes all five. -The full SDK suite is **not green**. `npm test` captured: - -```text - Test Files 9 failed | 186 passed | 3 skipped (198) - Tests 42 failed | 3174 passed | 26 skipped (3242) - Errors 2 errors -``` - -The full output is [sdk-suite.txt](evidence/561/sdk-suite.txt). Failures include unavailable bubblewrap, Bun 1.3.6 versus required 1.4.0, missing default kernel paths, and existing worker/gate expectations. These failures were not fixed or fully baseline-classified in this focused change. The kernel suite's complete command/output is [documented separately](evidence/561/README.md#broader-checks). +## The repository check, and one thing this branch fixes that it did not break + +`.relayflow/check.sh` failed on this branch with `Tests 30 failed | 3194 passed`. +Each failure is now classified against `origin/main` (`e30226c`) by reverting +this branch's three source files, re-running, and restoring; +[.relayflow/repair-notes.md](.relayflow/repair-notes.md) carries the commands and +captured output. None came from #561. The classification: + +* **`tests/artifact-gates.test.ts` (1) — `main` is red.** #566 made a declared + `cwd` run-root-relative; #517 landed one commit later with a regression + declaring an absolute `os.tmpdir()` `cwd`. Green apart, red together. Fixed + here in a separate commit that moves the fixture inside the run root and + changes no assertion — CI would fail it on this branch otherwise. +* **`tests/live-kernel.test.ts` (7) — environment.** `testdata/preflight`'s + extensionless ESM agent-CLI fixtures load as CommonJS, silently producing no + output, under this sandbox's `/home/daytona/package.json` (`"type": + "commonjs"`). Handled in `check.sh`, uncommitted: CI never sees the ambiguity. +* **`tests/authored-node-runtime.test.ts` (whole suite) — missing setup.** Bun + 1.3.6 where every workflow pins 1.4.0. `check.sh` now installs it. +* **`tests/hosted-extension-{isolation,protocol}.test.ts` (22) — outside this + container.** Unprivileged user namespaces are denied and the AppArmor + restriction cannot be relaxed from inside, so `bwrap` cannot run even once + installed. This is the provisioning step `check.sh` already documents as + deliberately omitted. + +With the first three addressed, the only outstanding failures in the whole check +are those 22. The steps `check.sh` never reached are captured in the repair notes +and pass. Mutation command (from `packages/sdk`, before and after restore): @@ -33,6 +52,8 @@ Mutation command (from `packages/sdk`, before and after restore): RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npx vitest run tests/authored-probe-cache.test.ts ``` +Re-run at this head as [mutation-red-rerun.txt](evidence/561/mutation-red-rerun.txt) and [mutation-green-rerun.txt](evidence/561/mutation-green-rerun.txt); the mutated run reproduces the issue's exact signature — `"completionReason":"lease_expired"` on attempt 1 with no heartbeat, `"disposition":"retry"`, attempt 2 `success`, and `WorkerLeaseLostError: Agent lease is already expired`. `packages/sdk/src/authored-worker-step.ts` hashes `b9cd463d9f1ee274175a0b659119262c0ba5b18f1afc11b866c3e0f7f34023b7` before the mutation and after the restore. + Captured failure summary (full [output](evidence/561/mutation-red.txt)): ```text From 9f918badbb0303ca9bab4500113b78ed44baf184 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Thu, 24 Sep 2026 07:06:46 +0000 Subject: [PATCH 06/11] docs: re-run the #561 mutation at the branch head The committed mutation transcripts were captured before the artifact-gate fix (1ccaaed) landed. Re-run `evidence/561/mutation.patch` against the current head so the transcript matches the bytes a reviewer checks out, and record the restore with `git diff --exit-code` plus a sha256sum. The failing capture carries the issue's exact journal shape at both capacity 1 and the default: attempt 1 completes `lease_expired` with `wallclock_ms: 30011`, a `retry_backoff` sleep follows, and attempt 2 succeeds. Restored, all five cases pass. Co-Authored-By: Claude Opus 5 --- evidence/561/README.md | 23 +++++ evidence/561/mutation-green-at-head.txt | 15 ++++ evidence/561/mutation-red-at-head.txt | 109 ++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 evidence/561/mutation-green-at-head.txt create mode 100644 evidence/561/mutation-red-at-head.txt diff --git a/evidence/561/README.md b/evidence/561/README.md index 084d328d..cf56a206 100644 --- a/evidence/561/README.md +++ b/evidence/561/README.md @@ -25,6 +25,29 @@ git diff --exit-code -- packages/sdk/src/authored-worker-step.ts The mutation removes only the fifth `checkAuthoredFlow` argument, as captured in [mutation.patch](mutation.patch). After the failure, `git restore -- packages/sdk/src/authored-worker-step.ts` restores the committed bytes. [restore.txt](restore.txt) captures `git diff --exit-code` and the SHA-256 comparison with HEAD. +### Re-verified at `100cd17` (branch head) + +The mutation was run once more at the branch head, after the artifact-gate fix +(`1ccaaed`) landed, so the transcript matches the bytes a reviewer checks out: + +```sh +git apply evidence/561/mutation.patch +cd packages/sdk && RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd \ + ./node_modules/.bin/vitest run tests/authored-probe-cache.test.ts # exit 1 +cd .. && git restore -- packages/sdk/src/authored-worker-step.ts +git diff --exit-code -- packages/sdk/src/authored-worker-step.ts # clean +sha256sum packages/sdk/src/authored-worker-step.ts +# b9cd463d9f1ee274175a0b659119262c0ba5b18f1afc11b866c3e0f7f34023b7 +cd packages/sdk && RELAYFLOWD_BIN=... ./node_modules/.bin/vitest run \ + tests/authored-probe-cache.test.ts # exit 0 +``` + +[Mutation failure at head](mutation-red-at-head.txt) — all 5 cases fail, and the +capacity-1 and capacity-4 journals carry the issue's exact shape: attempt 1 +`"completionReason":"lease_expired"` with `"wallclock_ms":30011`, a +`retry_backoff` sleep, then attempt 2 `"completionReason":"success"`. +[Restored pass at head](mutation-green-at-head.txt) — 5 passed, exit 0. + The spread bound allows one 300 ms synchronous probe plus process startup under load (1,000 ms total); the original nine probes take over 3 seconds. Exact auth and identify-only counts additionally pin the cache independently of timing. Worker identification sessions are counted separately from preflight probes. Tests also cover failures, agents, run isolation, capacity, and every child journal. ## Broader checks diff --git a/evidence/561/mutation-green-at-head.txt b/evidence/561/mutation-green-at-head.txt new file mode 100644 index 00000000..899a7b5e --- /dev/null +++ b/evidence/561/mutation-green-at-head.txt @@ -0,0 +1,15 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-probe-cache.test.ts (5 tests) 21261ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 1 7297ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 4 5416ms + ✓ authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 6854ms + ✓ authored run CLI probe cache > caches probe failures while refusing all nine calls 385ms + ✓ authored run CLI probe cache > shares probe results across agent calls too 1306ms + + Test Files 1 passed (1) + Tests 5 passed (5) + Start at 07:04:59 + Duration 23.51s (transform 1.26s, setup 0ms, collect 2.06s, tests 21.26s, environment 0ms, prepare 44ms) + diff --git a/evidence/561/mutation-red-at-head.txt b/evidence/561/mutation-red-at-head.txt new file mode 100644 index 00000000..c8455dc1 --- /dev/null +++ b/evidence/561/mutation-red-at-head.txt @@ -0,0 +1,109 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +(node:108917) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M393KT82NQP31ZWBD3QYHA6A step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M393KT82NQP31ZWBD3QYHA6A/llm-1. +(Use `node --trace-warnings ...` to show where the warning was created) +(node:108917) [FLOWS_WORKER_LEASE_LOST] Warning: test-llm: run_id=01M393MYTHA4SAQGDXAAPE6GB3 step_id=llm-1 attempt=1: WorkerLeaseLostError: Agent lease is already expired for 01M393MYTHA4SAQGDXAAPE6GB3/llm-1. + ❯ tests/authored-probe-cache.test.ts (5 tests | 5 failed) 87390ms + × authored run CLI probe cache > nine calls have no expired attempts at capacity 1 37510ms + → 01M393KT82NQP31ZWBD3QYHA6A.sqlite3: expected '[{"at_ms":1790233405701,"attempt":nul…' not to contain 'lease_expired' + × authored run CLI probe cache > nine calls have no expired attempts at capacity 4 37883ms + → 01M393MYTHA4SAQGDXAAPE6GB3.sqlite3: expected '[{"at_ms":1790233443157,"attempt":nul…' not to contain 'lease_expired' + × authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 6779ms + → expected 3439.2573919999995 to be less than 1000 + × authored run CLI probe cache > caches probe failures while refusing all nine calls 3260ms + → expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9 + × authored run CLI probe cache > shares probe results across agent calls too 1956ms + → expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 5 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 1 +AssertionError: 01M393KT82NQP31ZWBD3QYHA6A.sqlite3: expected '[{"at_ms":1790233405701,"attempt":nul…' not to contain 'lease_expired' + +Expected: "lease_expired" +Received: "[{"at_ms":1790233405701,"attempt":null,"entry_type":"run.spawned","payload":{"created_by":"protocol-v0","journal_version":1,"parent_run_id":null,"spec":{"name":"nine-llms/llm-1","steps":[{"cli":"/tmp/flows-chain-Z6S1j7/adapter.mjs","depends_on":[],"id":"llm-1","max_iterations":1,"model":"test-model","prompt":"Return JSON for 0","retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"llm","verification":{"json_schema":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}}}],"version":"0.1.0"},"spec_hash":"e6b476909fbe6ea959fb615e3667fe84c99cd7aa8d2840700827b83568adbc31"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":1,"step_id":null},{"at_ms":1790233405704,"attempt":null,"entry_type":"step.routed","payload":{"fallbacks_attempted":[],"profile":"attached-worker","provider":"worker"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":2,"step_id":"llm-1"},{"at_ms":1790233405704,"attempt":1,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-ec9336a7-372e-4035-bd93-574b346a5a12-llm","idempotency_key":"9dae83274af952459462b094bfe4bca1cd69287da66506eed1a45f7d9b42a85c","lease_deadline_ms":1790233435704,"lease_id":"01M393KT88MQF0V6DD1DSNCW8W","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":3,"step_id":"llm-1"},{"at_ms":1790233435715,"attempt":1,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","tokens_in":0,"tokens_out":0},"completed_by":"kernel","completionReason":"lease_expired","disposition":"retry","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":1790233435715,"output":null,"spend":{"dollars":0,"tokens_input":0,"tokens_output":0,"wallclock_ms":30011},"step_spec_hash":"2ddf3aa02deec6eb05ab0ff95bcec6aa5f0ec5658b1554a47e119abaecbaf4d2","verification":null},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":4,"step_id":"llm-1"},{"at_ms":1790233435715,"attempt":1,"entry_type":"sleep.until","payload":{"reason":"retry_backoff","wait_id":"01M393MQJ3SS1WEBDR0W44D1DS","wake_at_ms":1790233435715},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":5,"step_id":"llm-1"},{"at_ms":1790233435724,"attempt":1,"entry_type":"wait.completed","payload":{"completionReason":"timer_fired","result":null,"wait_id":"01M393MQJ3SS1WEBDR0W44D1DS"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":6,"step_id":"llm-1"},{"at_ms":1790233435726,"attempt":2,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-ec9336a7-372e-4035-bd93-574b346a5a12-llm","idempotency_key":"9dae83274af952459462b094bfe4bca1cd69287da66506eed1a45f7d9b42a85c","lease_deadline_ms":1790233465726,"lease_id":"01M393MQJEHDMZF830TVHDR9Z4","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":7,"step_id":"llm-1"},{"at_ms":1790233438844,"attempt":2,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completed_by":"local-agent-ec9336a7-372e-4035-bd93-574b346a5a12-llm","completionReason":"success","disposition":"step_done","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":null,"output":{"x":4},"spend":{"dollars":0,"dollars_unmetered":true,"tokens_input":0,"tokens_output":0,"wallclock_ms":3118},"step_spec_hash":"2ddf3aa02deec6eb05ab0ff95bcec6aa5f0ec5658b1554a47e119abaecbaf4d2","verification":{"detail":"all gates passed","gate":"json_schema","verdict":"pass"}},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":8,"step_id":"llm-1"},{"at_ms":1790233438851,"attempt":null,"entry_type":"run.completed","payload":{"budget_total":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completionReason":"success","failed_step_id":null},"run_id":"01M393KT82NQP31ZWBD3QYHA6A","segment_id":1,"seq":9,"step_id":null}]" + + ❯ tests/authored-probe-cache.test.ts:82:49 + 80| for (const file of runs) { + 81| const { entries } = await client.journalRead(file.slice(0, -8), … + 82| expect(JSON.stringify(entries), file).not.toContain('lease_expir… + | ^ + 83| expect(entries.filter(e => typeof e === 'object' && e !== null + 84| && 'entry_type' in e && e.entry_type === 'step.attempt.started… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > nine calls have no expired attempts at capacity 4 +AssertionError: 01M393MYTHA4SAQGDXAAPE6GB3.sqlite3: expected '[{"at_ms":1790233443157,"attempt":nul…' not to contain 'lease_expired' + +Expected: "lease_expired" +Received: "[{"at_ms":1790233443157,"attempt":null,"entry_type":"run.spawned","payload":{"created_by":"protocol-v0","journal_version":1,"parent_run_id":null,"spec":{"name":"nine-llms/llm-1","steps":[{"cli":"/tmp/flows-chain-L5o19j/adapter.mjs","depends_on":[],"id":"llm-1","max_iterations":1,"model":"test-model","prompt":"Return JSON for 0","retry":{"initial_backoff_ms":100,"jitter_percent":20,"max_backoff_ms":60000,"multiplier":2},"type":"llm","verification":{"json_schema":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}}}],"version":"0.1.0"},"spec_hash":"825456894ebae30b76964e8f2be4e44800229df3a5fe9d97330e711667cd9942"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":1,"step_id":null},{"at_ms":1790233443160,"attempt":null,"entry_type":"step.routed","payload":{"fallbacks_attempted":[],"profile":"attached-worker","provider":"worker"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":2,"step_id":"llm-1"},{"at_ms":1790233443160,"attempt":1,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-64172c0f-f0b6-4e18-9c71-be2d14f2a732-llm","idempotency_key":"30c5fbbfba3a60e60d4b2c2f1a4575e682bcc5338855708042d5e211522c65df","lease_deadline_ms":1790233473160,"lease_id":"01M393MYTRA9HDE33PGP3FVPR8","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":3,"step_id":"llm-1"},{"at_ms":1790233473223,"attempt":1,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","tokens_in":0,"tokens_out":0},"completed_by":"kernel","completionReason":"lease_expired","disposition":"retry","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":1790233473223,"output":null,"spend":{"dollars":0,"tokens_input":0,"tokens_output":0,"wallclock_ms":30063},"step_spec_hash":"99f64b0a179f1cbbe2c9235c1f0e7130481a1ac4536fd9f607d735535a6d8772","verification":null},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":4,"step_id":"llm-1"},{"at_ms":1790233473223,"attempt":1,"entry_type":"sleep.until","payload":{"reason":"retry_backoff","wait_id":"01M393NW67TMY3ETXH3V05BJ2X","wake_at_ms":1790233473223},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":5,"step_id":"llm-1"},{"at_ms":1790233473234,"attempt":1,"entry_type":"wait.completed","payload":{"completionReason":"timer_fired","result":null,"wait_id":"01M393NW67TMY3ETXH3V05BJ2X"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":6,"step_id":"llm-1"},{"at_ms":1790233473235,"attempt":2,"entry_type":"step.attempt.started","payload":{"executor":"local-agent-64172c0f-f0b6-4e18-9c71-be2d14f2a732-llm","idempotency_key":"30c5fbbfba3a60e60d4b2c2f1a4575e682bcc5338855708042d5e211522c65df","lease_deadline_ms":1790233503235,"lease_id":"01M393NW6KT69VKBMWWEA8CBJ2","max_iterations":1,"pins":{"streams":[],"workspace":[]},"recovery_mode":null,"step_type":"llm"},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":7,"step_id":"llm-1"},{"at_ms":1790233476384,"attempt":2,"entry_type":"step.completed","payload":{"budget":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completed_by":"local-agent-64172c0f-f0b6-4e18-9c71-be2d14f2a732-llm","completionReason":"success","disposition":"step_done","effects":[],"end_pins":null,"input_hash":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","next_attempt_at_ms":null,"output":{"x":4},"spend":{"dollars":0,"dollars_unmetered":true,"tokens_input":0,"tokens_output":0,"wallclock_ms":3149},"step_spec_hash":"99f64b0a179f1cbbe2c9235c1f0e7130481a1ac4536fd9f607d735535a6d8772","verification":{"detail":"all gates passed","gate":"json_schema","verdict":"pass"}},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":8,"step_id":"llm-1"},{"at_ms":1790233476387,"attempt":null,"entry_type":"run.completed","payload":{"budget_total":{"dollars":"0","dollars_unmetered":true,"tokens_in":0,"tokens_out":0},"completionReason":"success","failed_step_id":null},"run_id":"01M393MYTHA4SAQGDXAAPE6GB3","segment_id":1,"seq":9,"step_id":null}]" + + ❯ tests/authored-probe-cache.test.ts:82:49 + 80| for (const file of runs) { + 81| const { entries } = await client.journalRead(file.slice(0, -8), … + 82| expect(JSON.stringify(entries), file).not.toContain('lease_expir… + | ^ + 83| expect(entries.filter(e => typeof e === 'object' && e !== null + 84| && 'entry_type' in e && e.entry_type === 'step.attempt.started… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run +AssertionError: expected 3439.2573919999995 to be less than 1000 + ❯ tests/authored-probe-cache.test.ts:104:41 + 102| // F1 permits one synchronous probe (~300ms), not nine (~3s). + 103| // Leave process-startup headroom for loaded CI; exact counts belo… + 104| expect(starts.at(-1)! - starts[0]!).toBeLessThan(1_000); + | ^ + 105| expectOneProbe(logs()); + 106| await executeAuthoredFlow(nine, client, undefined, options); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > caches probe failures while refusing all nine calls +AssertionError: expected [ { kind: 'auth' }, …(8) ] to have a length of 1 but got 9 + +- Expected ++ Received + +- 1 ++ 9 + + ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47 + 68| + 69| function expectOneProbe(logs: ReturnType l.kind === 'auth')).toHaveLength(1); + | ^ + 71| expect(logs.filter(l => l.kind === 'identify').length - logs.filter(… + 72| } + ❯ tests/authored-probe-cache.test.ts:123:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/5]⎯ + + FAIL tests/authored-probe-cache.test.ts > authored run CLI probe cache > shares probe results across agent calls too +AssertionError: expected [ { kind: 'auth' }, …(2) ] to have a length of 1 but got 3 + +- Expected ++ Received + +- 1 ++ 3 + + ❯ expectOneProbe tests/authored-probe-cache.test.ts:70:47 + 68| + 69| function expectOneProbe(logs: ReturnType l.kind === 'auth')).toHaveLength(1); + | ^ + 71| expect(logs.filter(l => l.kind === 'identify').length - logs.filter(… + 72| } + ❯ tests/authored-probe-cache.test.ts:136:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/5]⎯ + + Test Files 1 failed (1) + Tests 5 failed (5) + Start at 07:03:19 + Duration 89.41s (transform 1.12s, setup 0ms, collect 1.84s, tests 87.39s, environment 0ms, prepare 42ms) + From 909a1d2b062d6b4f5d347b66d9801e1b055bbd92 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Thu, 24 Sep 2026 07:13:51 +0000 Subject: [PATCH 07/11] Keep relayflow working files out of the change --- summary.md | 71 ------------------------------------------------------ 1 file changed, 71 deletions(-) delete mode 100644 summary.md diff --git a/summary.md b/summary.md deleted file mode 100644 index a5413f14..00000000 --- a/summary.md +++ /dev/null @@ -1,71 +0,0 @@ -# Cache authored CLI preflight per run (#561, F1) - -Nine concurrent `f.llm` calls synchronously probed the same CLI nine times before worker-slot admission. Slow probes blocked the worker's event loop long enough for already-issued 30-second leases to expire. `WorkerSlots` admission is unchanged. - -Reuse preflight's existing `(cli, source, model, execution)` cache across one authored run, for both LLM and agent calls. Read project configuration lazily once per runner. Keep the synchronous public preflight API and existing fail-closed resolution checks. - -Auth/model readiness is now checked once per key per run, matching declarative execution. Auth changes during a run are detected by execution rather than another preflight; failed probes are cached too, so later calls receive the refusal instead of repeating a timeout. New runs probe again. - -F2 (asynchronous probing) is deferred to [#574](https://github.com/AgentWorkforce/flows/issues/574). Distinct keys still require distinct synchronous probes. **One hung probe blocks for the model-readiness timeout: 60 s. That exceeds the 30 s lease and will still kill a concurrent step.** This focused fix does not establish the broader invariant that no synchronous probe runs while any lease is live. - -Validation commands and literal captured output are in [evidence/561/README.md](evidence/561/README.md). Regression coverage uses a real kernel and slow fake CLI at capacity 1 and default capacity 4; it inspects every child journal, bounds session overlap, counts probes, checks failed-probe caching, agent parity, and fresh-run isolation. A separate test pins CLI/source/model cache keys. - -Acceptance: - -- [x] Nine concurrent fake-CLI calls at capacity 1 and default: success, one attempt per child, no `lease_expired`. -- [x] The issue's exact live-Claude repro succeeded locally at both capacities; journal outputs are 4 through 12. It could **not** be re-run at this head: `claude` is no longer authenticated in this environment (`"loggedIn": false`), captured in [live-one-recheck-unauthenticated.txt](evidence/561/live-one-recheck-unauthenticated.txt). That capture does show one 1.04 s probe followed by eight 0.01–0.12 s cached refusals. -- [ ] Restore `Promise.all` in prompt-lab: both requested files are absent from this checkout. No example restoration is claimed. -- [x] Mutation verification: removing the cache argument fails all five tests; restoring the committed bytes passes all five. - -## The repository check, and one thing this branch fixes that it did not break - -`.relayflow/check.sh` failed on this branch with `Tests 30 failed | 3194 passed`. -Each failure is now classified against `origin/main` (`e30226c`) by reverting -this branch's three source files, re-running, and restoring; -[.relayflow/repair-notes.md](.relayflow/repair-notes.md) carries the commands and -captured output. None came from #561. The classification: - -* **`tests/artifact-gates.test.ts` (1) — `main` is red.** #566 made a declared - `cwd` run-root-relative; #517 landed one commit later with a regression - declaring an absolute `os.tmpdir()` `cwd`. Green apart, red together. Fixed - here in a separate commit that moves the fixture inside the run root and - changes no assertion — CI would fail it on this branch otherwise. -* **`tests/live-kernel.test.ts` (7) — environment.** `testdata/preflight`'s - extensionless ESM agent-CLI fixtures load as CommonJS, silently producing no - output, under this sandbox's `/home/daytona/package.json` (`"type": - "commonjs"`). Handled in `check.sh`, uncommitted: CI never sees the ambiguity. -* **`tests/authored-node-runtime.test.ts` (whole suite) — missing setup.** Bun - 1.3.6 where every workflow pins 1.4.0. `check.sh` now installs it. -* **`tests/hosted-extension-{isolation,protocol}.test.ts` (22) — outside this - container.** Unprivileged user namespaces are denied and the AppArmor - restriction cannot be relaxed from inside, so `bwrap` cannot run even once - installed. This is the provisioning step `check.sh` already documents as - deliberately omitted. - -With the first three addressed, the only outstanding failures in the whole check -are those 22. The steps `check.sh` never reached are captured in the repair notes -and pass. - -Mutation command (from `packages/sdk`, before and after restore): - -```sh -RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npx vitest run tests/authored-probe-cache.test.ts -``` - -Re-run at this head as [mutation-red-rerun.txt](evidence/561/mutation-red-rerun.txt) and [mutation-green-rerun.txt](evidence/561/mutation-green-rerun.txt); the mutated run reproduces the issue's exact signature — `"completionReason":"lease_expired"` on attempt 1 with no heartbeat, `"disposition":"retry"`, attempt 2 `success`, and `WorkerLeaseLostError: Agent lease is already expired`. `packages/sdk/src/authored-worker-step.ts` hashes `b9cd463d9f1ee274175a0b659119262c0ba5b18f1afc11b866c3e0f7f34023b7` before the mutation and after the restore. - -Captured failure summary (full [output](evidence/561/mutation-red.txt)): - -```text - Test Files 1 failed (1) - Tests 5 failed (5) -``` - -Captured restored summary (full [output](evidence/561/mutation-green.txt)): - -```text - Test Files 1 passed (1) - Tests 5 passed (5) -``` - -The mutation fails the journal assertion itself at both capacities, with `lease_expired` in the child journal. [Restore evidence](evidence/561/restore.txt) includes the literal clean diff result and identical SHA-256 hashes. No workflow files changed. From d69cd27bfb88aa627c013111526c4de623d9b60e Mon Sep 17 00:00:00 2001 From: Relayflow Date: Wed, 23 Sep 2026 05:57:34 +0000 Subject: [PATCH 08/11] fix(sdk): keep authored CLI probes from starving worker leases Session-Id: 01a0d409-e854-7540-a76e-a2f9cd136946 --- packages/sdk/src/authored-preflight.ts | 46 +++++ packages/sdk/src/authored-worker-step.ts | 12 +- packages/sdk/src/cli/check.ts | 162 +-------------- packages/sdk/src/cli/cli-probe.ts | 189 ++++++++++++++++++ packages/sdk/src/preflight.ts | 32 ++- .../sdk/tests/authored-parallel-llm.test.ts | 124 ++++++++++++ packages/sdk/tests/authored-preflight.test.ts | 54 +++++ packages/sdk/tests/cli-probe.test.ts | 62 ++++++ .../sdk/tests/communication-preflight.test.ts | 2 +- 9 files changed, 508 insertions(+), 175 deletions(-) create mode 100644 packages/sdk/src/authored-preflight.ts create mode 100644 packages/sdk/src/cli/cli-probe.ts create mode 100644 packages/sdk/tests/authored-parallel-llm.test.ts create mode 100644 packages/sdk/tests/authored-preflight.test.ts create mode 100644 packages/sdk/tests/cli-probe.test.ts diff --git a/packages/sdk/src/authored-preflight.ts b/packages/sdk/src/authored-preflight.ts new file mode 100644 index 00000000..aa5bc09b --- /dev/null +++ b/packages/sdk/src/authored-preflight.ts @@ -0,0 +1,46 @@ +import { dirname, resolve } from 'node:path'; +import { checkAuthoredFlow, readProjectConfig, type ProjectConfig } from './cli/check.js'; +import { probeCliAsync } from './cli/cli-probe.js'; +import { communicationInstruction } from './communication/spec.js'; +import { cliProbeKey, CliProbeError, resolvePreflight, type CliProbeOutcome } from './preflight.js'; +import type { FlowSpec } from './spec.js'; + +/** Auth/model facts belong to one execution, including concurrent cold callers. */ +export function authoredPreflight(path: string) { + const probeCache = new Map(); + const pending = new Map>(); + const directory = dirname(resolve(path)); + return async (flow: FlowSpec) => { + let config: ProjectConfig; + try { config = readProjectConfig(directory); } + catch { return checkAuthoredFlow(flow, path); } + const resolved = resolvePreflight(flow, { + projectCli: config.cli, projectConfigPath: config.path, projectSearchStart: directory, + models: config.models, modelRegistryPath: config.modelRegistryPath, + }); + if (!resolved.ok) return checkAuthoredFlow(flow, path, config); + await Promise.all(resolved.resolutions.map(async resolution => { + const step = resolved.compiled!.steps.find(step => step.id === resolution.stepId)!; + const managed = step.type === 'agent' && communicationInstruction(step.instruction) !== undefined; + const key = cliProbeKey(resolution, managed); + if (probeCache.has(key)) return; + let probing = pending.get(key); + if (probing === undefined) { + probing = (async () => { + try { + probeCache.set(key, { result: await probeCliAsync(resolution.cli, + resolution.source === 'project' ? config.directory : directory, + resolution.model, managed ? 'managed' : undefined) }); + } catch (error) { + probeCache.set(key, { failure: error instanceof CliProbeError ? error.detail : null }); + } + })(); + pending.set(key, probing); + } + await probing; + pending.delete(key); + })); + const result = checkAuthoredFlow(flow, path, config, {}, probeCache); + return result; + }; +} diff --git a/packages/sdk/src/authored-worker-step.ts b/packages/sdk/src/authored-worker-step.ts index 8f14a6fe..4801a520 100644 --- a/packages/sdk/src/authored-worker-step.ts +++ b/packages/sdk/src/authored-worker-step.ts @@ -3,9 +3,9 @@ import type { AuthoredBudget } from './authored-budget.js'; import { parseBudget } from './budget.js'; import type { AgentOptions, AgentResult, LlmOptions, NamedGate } from '@relayflows/surface'; import { compileSpec, toKernelSpec } from './compile.js'; -import { checkAuthoredFlow, readProjectConfig, type ProjectConfig } from './cli/check.js'; +import { authoredPreflight } from './authored-preflight.js'; import { classifyOutcome, type RunLifecycleOptions, type RunReport } from './cli/run.js'; -import type { CliProbeOutcome, PreflightDiagnostic } from './preflight.js'; +import type { PreflightDiagnostic } from './preflight.js'; import { AuthoredFlowExecutionError } from './authored-flow-error.js'; import { agentCwdDeclarationError, agentCwdTransportError } from './agent-cwd.js'; import type { JournalClient } from './journal-client.js'; @@ -31,10 +31,7 @@ export function authoredWorkerRunner( // slot instead of being admitted and parked for want of a free worker. const slots = workerCapacity === undefined ? undefined : { agent: new WorkerSlots(workerCapacity), llm: new WorkerSlots(workerCapacity) }; - // Repeated synchronous probes starve dispatch and heartbeats before admission. - // Match declarative preflight: cache both pass and refusal for this run only. - const cliProbeCache = new Map(); - let projectConfig: ProjectConfig | undefined; + const check = authoredPreflight(flowPath); const context: AuthoredStepContext = { ...(rootRunId === undefined ? {} : { rootRunId }), ...(waitOptions.dataDir === undefined ? {} : { dataDir: waitOptions.dataDir }), @@ -48,8 +45,7 @@ export function authoredWorkerRunner( // (cli/check.ts), searching for the nearest flows.json from `flowPath` // and real-probing auth/model readiness. An authored agent step gets // nothing for free just because it was declared in TS instead of YAML. - const { report, flow: resolved } = checkAuthoredFlow(authoring, flowPath, - projectConfig ??= readProjectConfig(dirname(resolve(flowPath))), {}, cliProbeCache); + const { report, flow: resolved } = await check(authoring); if (!report.ok || resolved === undefined) { const refusal = report.diagnostics.find( (diagnostic): diagnostic is PreflightDiagnostic & { severity: 'refusal' } => diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index c0352f1b..c3e296ea 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -1,23 +1,13 @@ import { communicationInstruction } from '../communication/spec.js'; import { checkCommunicationEnvironment } from '../communication/preflight.js'; -import { agentEnvironment, brokerEnvironment } from '../communication/environment.js'; import { accessSync, constants, readFileSync } from 'node:fs'; import { dirname, isAbsolute, join, parse as parsePath, resolve } from 'node:path'; -import { spawnSync } from 'node:child_process'; import { parse as parseYaml } from 'yaml'; import { CompileError, compileSpec, kernelToAuthoring } from '../compile.js'; import { agentWorkerDiagnostics } from './check-worker-surface.js'; import { helperReady } from '../yaml-helper-effect.js'; import { flowRequirements, type FlowRequirements } from '../flow-requirements.js'; -import { - adapterIdentification, - authenticationProbe, - cliAdapterKind, - displayInvocation, - modelReadinessProbe, - type CliInvocation, -} from '../cli-adapter.js'; -import { MODEL_ENV } from '../worker-cli.js'; +import { probeCli, resolveExecutable } from './cli-probe.js'; import { modelNameError } from '../model-name.js'; import type { FlowSpec } from '../spec.js'; import type { McpServerConfig } from '../spec.js'; @@ -26,7 +16,6 @@ import type { StepGateInspection } from '../gate-contract.js'; import type { CheckFailureKind, CheckWarningKind } from '../failure-kinds.js'; import { preflight, - CliProbeError, type CliResolution, type CliProbeResult, type CliProbeOutcome, @@ -482,155 +471,6 @@ function canonicalCli(cli: string, directory: string): string { return resolve(directory, cli); } -function probeCli( - cli: string, - directory: string, - model?: string, - execution?: 'managed', -): CliProbeResult { - const executable = resolveExecutable(cli, directory); - if (executable === undefined) return { exists: false, authenticated: false }; - const kind = cliAdapterKind(executable); - // Relay owns interactive CLI launch/injection. Its generic PTY path is not - // the headless wrapper protocol; do not demand that protocol from Gemini, - // Cursor, OpenCode, or other interactive tools. Never invent an auth pass. - if (execution === 'managed' && kind === 'relayflows-wrapper-v1') { - return { exists: true, supported: true, authenticated: 'unverified' }; - } - const environment = execution === 'managed' - ? { ...brokerEnvironment(process.env), ...agentEnvironment(executable) } : process.env; - const probe = (invocation: CliInvocation) => runProbe(executable, directory, invocation, environment); - const identification = adapterIdentification(kind); - const identified = probe(identification.invocation); - if ( - identified.status !== 0 - || (identification.expectedStdout !== undefined - && identified.stdout.trim() !== identification.expectedStdout) - ) { - return { exists: true, supported: false, authenticated: false }; - } - const auth = authenticationProbe(kind); - const authCommand = displayInvocation(cli, auth); - if (model === undefined) { - return { - exists: true, - supported: true, - authenticated: probe(auth).status === 0, - authCommand, - }; - } - - const scoped = modelReadinessProbe(kind, model); - const modelCommand = displayInvocation(cli, scoped); - // A successful real provider round trip (or identified wrapper probe) - // proves both auth and exact-model access. On failure, run the adapter's - // actual auth command solely to classify auth vs model access truthfully. - if (probe(scoped).status === 0) { - return { - exists: true, - supported: true, - authenticated: true, - modelAvailable: true, - authCommand, - modelCommand, - }; - } - const authProbe = probe(auth); - const authenticated = authProbe.status === 0; - return { - exists: true, - supported: true, - authenticated, - modelAvailable: false, - authCommand, - modelCommand, - // Only on failure: on success there is nothing to explain, and the output - // is the most identity-bearing thing this function touches. - ...(authenticated - ? {} - : { - authExitCode: authProbe.status, - authFailureDetail: redactProbeOutput( - `${authProbe.stderr}${authProbe.stdout}`, - ).trim().slice(0, 500), - }), - }; -} - -/** - * Redact anything that looks like a credential or an account identifier. - * - * `auth status` output is diagnostic, but it is also the one place an account - * email, org id or token fragment can appear. The point of surfacing it is to - * say WHY a probe failed, which survives redaction; leaking an identity into a - * refusal message that gets pasted into issues does not. - */ -function redactProbeOutput(text: string): string { - return text - .replace(/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, '') - .replace(/\b(sk|pk|oat|rt)[-_][A-Za-z0-9._-]{8,}/gi, '') - .replace(/\b[A-Fa-f0-9]{32,}\b/g, ''); -} - -function runProbe( - executable: string, - directory: string, - invocation: CliInvocation, - environment: NodeJS.ProcessEnv = process.env, -): { status: number | null; stdout: string; stderr: string } { - const env = { ...environment }; - delete env[MODEL_ENV]; - if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; - const result = spawnSync(executable, invocation.args, { - cwd: directory, - encoding: 'utf8', - // stderr was 'ignore'. A failing `auth status` writes its reason there, so - // discarding it made every authentication refusal structurally - // undiagnosable: the refusal could say a probe exited non-zero and never - // what it said. Captured, then redacted at the point of use. - stdio: ['ignore', 'pipe', 'pipe'], - timeout: invocation.timeoutMs, - env, - }); - const failure = classifySpawnFailure(result.error, result.signal, invocation.timeoutMs); - if (failure !== undefined) throw failure; - return { - status: result.status, - stdout: result.stdout, - stderr: result.stderr ?? '', - }; -} - -function resolveExecutable(command: string, directory: string): string | undefined { - if (command.includes('/') || isAbsolute(command)) { - const path = isAbsolute(command) ? command : resolve(directory, command); - try { - accessSync(path, constants.X_OK); - return path; - } catch { - return undefined; - } - } - const result = spawnSync('which', [command], { encoding: 'utf8', timeout: 5_000 }); - const failure = classifySpawnFailure(result.error, result.signal, 5_000); - if (failure !== undefined) throw failure; - return result.status === 0 ? result.stdout.trim() : undefined; -} - -function classifySpawnFailure( - error: Error | undefined, - signal: NodeJS.Signals | null, - timeoutMs: number, -): CliProbeError | undefined { - if (error !== undefined) { - const detail = (error as NodeJS.ErrnoException).code === 'ETIMEDOUT' - ? `timeout:${timeoutMs}ms` as const - : 'spawn_failed' as const; - return new CliProbeError(detail); - } - return signal === null ? undefined : new CliProbeError(`signal:${signal}`); -} - function executableExists(command: string, directory: string): boolean { return resolveExecutable(command, directory) !== undefined; } diff --git a/packages/sdk/src/cli/cli-probe.ts b/packages/sdk/src/cli/cli-probe.ts new file mode 100644 index 00000000..2268f0ad --- /dev/null +++ b/packages/sdk/src/cli/cli-probe.ts @@ -0,0 +1,189 @@ +import { accessSync, constants } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; +import { execFile, spawnSync } from 'node:child_process'; +import { agentEnvironment, brokerEnvironment } from '../communication/environment.js'; +import { adapterIdentification, authenticationProbe, cliAdapterKind, displayInvocation, + modelReadinessProbe, type CliInvocation } from '../cli-adapter.js'; +import { MODEL_ENV } from '../worker-cli.js'; +import { CliProbeError, type CliProbeResult } from '../preflight.js'; + +interface ProbeRequest { + executable: string; + directory: string; + invocation: CliInvocation; + environment: NodeJS.ProcessEnv; +} +interface ProbeOutput { status: number | null; stdout: string; stderr: string } + +/** One decision sequence; checking uses synchronous I/O, live flows yield the loop. */ +export function probeCli(...args: Parameters): CliProbeResult { + return driveSync(probeSequence(...args)); +} + +export async function probeCliAsync(...args: Parameters): Promise { + const sequence = probeSequence(...args); + let next = sequence.next(); + while (!next.done) next = sequence.next(await runProbeAsync(next.value)); + return next.value; +} + +export function resolveExecutable(command: string, directory: string): string | undefined { + return driveSync(executableSequence(command, directory)); +} + +function driveSync(sequence: Generator): T { + let next = sequence.next(); + while (!next.done) { + const request = next.value; + const result = spawnSync(request.executable, request.invocation.args, probeOptions(request)); + const failure = classifySpawnFailure(result.error, result.signal, request.invocation.timeoutMs); + if (failure !== undefined) throw failure; + next = sequence.next({ status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }); + } + return next.value; +} + +function probeOptions({ directory, invocation, environment }: ProbeRequest) { + const env = { ...environment }; + delete env[MODEL_ENV]; + if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; + return { cwd: directory, encoding: 'utf8' as const, timeout: invocation.timeoutMs, + maxBuffer: 1024 * 1024, env }; +} + +function runProbeAsync(request: ProbeRequest): Promise { + return new Promise((resolve, reject) => { + const child = execFile(request.executable, request.invocation.args, probeOptions(request), + (error, stdout, stderr) => { + // Numeric exit codes are probe results; launch, timeout, signal and buffer + // errors are failures to collect a fact, just as in the synchronous driver. + if (error?.killed && typeof error.code !== 'string') return reject(new CliProbeError(`timeout:${request.invocation.timeoutMs}ms`)); + const failure = classifySpawnFailure( + error !== null && typeof error.code !== 'number' && error.signal == null ? error : undefined, + error?.signal ?? null, request.invocation.timeoutMs); + if (failure !== undefined) return reject(failure); + resolve({ status: error === null ? 0 : typeof error.code === 'number' ? error.code : null, + stdout, stderr }); + }); + child.stdin?.end(); + }); +} + +function* probeSequence( + cli: string, + directory: string, + model?: string, + execution?: 'managed', +): Generator { + const executable = yield* executableSequence(cli, directory); + if (executable === undefined) return { exists: false, authenticated: false }; + const kind = cliAdapterKind(executable); + // Relay owns interactive CLI launch/injection. Its generic PTY path is not + // the headless wrapper protocol; do not demand that protocol from Gemini, + // Cursor, OpenCode, or other interactive tools. Never invent an auth pass. + if (execution === 'managed' && kind === 'relayflows-wrapper-v1') { + return { exists: true, supported: true, authenticated: 'unverified' }; + } + const environment = execution === 'managed' + ? { ...brokerEnvironment(process.env), ...agentEnvironment(executable) } : process.env; + const probe = (invocation: CliInvocation): ProbeRequest => ({ executable, directory, invocation, environment }); + const identification = adapterIdentification(kind); + const identified = yield probe(identification.invocation); + if ( + identified.status !== 0 + || (identification.expectedStdout !== undefined + && identified.stdout.trim() !== identification.expectedStdout) + ) { + return { exists: true, supported: false, authenticated: false }; + } + const auth = authenticationProbe(kind); + const authCommand = displayInvocation(cli, auth); + if (model === undefined) { + return { + exists: true, + supported: true, + authenticated: (yield probe(auth)).status === 0, + authCommand, + }; + } + + const scoped = modelReadinessProbe(kind, model); + const modelCommand = displayInvocation(cli, scoped); + // A successful real provider round trip (or identified wrapper probe) + // proves both auth and exact-model access. On failure, run the adapter's + // actual auth command solely to classify auth vs model access truthfully. + if ((yield probe(scoped)).status === 0) { + return { + exists: true, + supported: true, + authenticated: true, + modelAvailable: true, + authCommand, + modelCommand, + }; + } + const authProbe = yield probe(auth); + const authenticated = authProbe.status === 0; + return { + exists: true, + supported: true, + authenticated, + modelAvailable: false, + authCommand, + modelCommand, + // Only on failure: on success there is nothing to explain, and the output + // is the most identity-bearing thing this function touches. + ...(authenticated + ? {} + : { + authExitCode: authProbe.status, + authFailureDetail: redactProbeOutput( + `${authProbe.stderr}${authProbe.stdout}`, + ).trim().slice(0, 500), + }), + }; +} + +/** + * Redact anything that looks like a credential or an account identifier. + * + * `auth status` output is diagnostic, but it is also the one place an account + * email, org id or token fragment can appear. The point of surfacing it is to + * say WHY a probe failed, which survives redaction; leaking an identity into a + * refusal message that gets pasted into issues does not. + */ +function redactProbeOutput(text: string): string { + return text + .replace(/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, '') + .replace(/\b(sk|pk|oat|rt)[-_][A-Za-z0-9._-]{8,}/gi, '') + .replace(/\b[A-Fa-f0-9]{32,}\b/g, ''); +} + +function* executableSequence(command: string, directory: string): Generator { + if (command.includes('/') || isAbsolute(command)) { + const path = isAbsolute(command) ? command : resolve(directory, command); + try { + accessSync(path, constants.X_OK); + return path; + } catch { + return undefined; + } + } + const result = yield { executable: 'which', directory: process.cwd(), + invocation: { args: [command], timeoutMs: 5_000 }, environment: process.env }; + return result.status === 0 ? result.stdout.trim() : undefined; +} + +function classifySpawnFailure( + error: Error | undefined, + signal: NodeJS.Signals | null, + timeoutMs: number, +): CliProbeError | undefined { + if (error !== undefined) { + const detail = (error as NodeJS.ErrnoException).code === 'ETIMEDOUT' + ? `timeout:${timeoutMs}ms` as const + : 'spawn_failed' as const; + return new CliProbeError(detail); + } + return signal === null ? undefined : new CliProbeError(`signal:${signal}`); +} diff --git a/packages/sdk/src/preflight.ts b/packages/sdk/src/preflight.ts index 739a2eba..19c2090d 100644 --- a/packages/sdk/src/preflight.ts +++ b/packages/sdk/src/preflight.ts @@ -101,6 +101,7 @@ export interface PreflightOptions { /** Exact, project-owned model allowlist from the nearest flows.json. */ models?: readonly string[]; modelRegistryPath?: string; + probeCache?: Map; probes: PreflightProbes; } @@ -196,7 +197,12 @@ async function probeMcp(result: PreflightResult, options: PreflightOptions): Pro return { ...result, ok: !result.diagnostics.some(d => d.severity === 'refusal'), mcpTools: Object.freeze(inventory) }; } -function preflightSync(flow: unknown, options: PreflightOptions): PreflightResult { +type ResolutionOptions = Omit; + +/** Resolve static declarations before any CLI, model, command or trigger probe. */ +export function resolvePreflight(flow: unknown, options: ResolutionOptions): PreflightResult & { + compiled?: import('./compile.js').CompiledFlowSpec; +} { // Compile before touching any environment fact. `compileSpec` snapshots raw // input into inert data, validates it against the closed authoring schema, // and lowers `output` sugar into its json_schema gate — so the gate plan @@ -291,6 +297,18 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul return { ok: false, gates: compiled.steps.map(inspectStepGate), resolutions, diagnostics }; } + return { ok: true, gates: compiled.steps.map(inspectStepGate), resolutions, diagnostics, compiled }; +} + +function preflightSync(flow: unknown, options: PreflightOptions): PreflightResult { + const { compiled, ...result } = resolvePreflight(flow, options); + if (!result.ok || compiled === undefined) return result; + const { diagnostics, resolutions } = result; + const resolutionByStep = new Map(resolutions.map(resolution => [resolution.stepId, resolution])); + // `cliProbeCache` is the public run-scoped name; `probeCache` is retained as + // a compatibility alias for the async authored-preflight path. + const cliProbeResults = options.cliProbeCache ?? options.probeCache ?? new Map(); + for (const step of compiled.steps) { probeNamedGate(step, options.probes, diagnostics); warnOnVacuousGate(step, diagnostics); @@ -335,7 +353,7 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul * fact about the filesystem. Missing manifest means no known mounts — a grant * still parses but refuses as `mount_unknown` in that case. */ -function scopeDiagnostics(flow: FlowSpec, options: PreflightOptions): PreflightRefusal[] { +function scopeDiagnostics(flow: FlowSpec, options: ResolutionOptions): PreflightRefusal[] { const workspace = flow.workspace; const toolsFs = flow.tools?.fs; if (workspace === undefined && toolsFs === undefined) return []; @@ -359,7 +377,7 @@ function scopeDiagnostics(flow: FlowSpec, options: PreflightOptions): PreflightR /** Pure authoring validation: no executable, command, trigger, or daemon probe. */ function unknownModelDiagnostics( flow: FlowSpec, - options: PreflightOptions, + options: ResolutionOptions, resolutionByStep: ReadonlyMap = new Map(), ): PreflightRefusal[] { const diagnostics: PreflightRefusal[] = []; @@ -473,7 +491,7 @@ function unknownModelMessage( return `Step "${stepId}" declares model "${model}"${cliContext}, but it is not listed in ${source}; add the exact model only after verifying that project is allowed to use it.`; } -function unresolvedCliMessage(stepId: string, options: PreflightOptions): string { +function unresolvedCliMessage(stepId: string, options: ResolutionOptions): string { const context = options.projectConfigPath !== undefined ? ` Nearest project config "${options.projectConfigPath}" declares no cli; outer configs are shadowed.` : options.projectSearchStart !== undefined @@ -503,6 +521,10 @@ function resolveCli( return undefined; } +export function cliProbeKey(resolution: CliResolution, managed = false): string { + return JSON.stringify([resolution.cli, resolution.source, resolution.model ?? null, managed]); +} + function probeResolvedCli( resolution: CliResolution, probes: PreflightProbes, @@ -516,7 +538,7 @@ function probeResolvedCli( // Model is part of the key: the same CLI probed with two different models // is two different questions, and caching on the CLI alone would let a // model that the CLI cannot resolve inherit an earlier model's pass. - const cacheKey = JSON.stringify([resolution.cli, resolution.source, resolution.model ?? null, managed]); + const cacheKey = cliProbeKey(resolution, managed); let outcome = cache.get(cacheKey); if (outcome === undefined) { try { diff --git a/packages/sdk/tests/authored-parallel-llm.test.ts b/packages/sdk/tests/authored-parallel-llm.test.ts new file mode 100644 index 00000000..2e60623c --- /dev/null +++ b/packages/sdk/tests/authored-parallel-llm.test.ts @@ -0,0 +1,124 @@ +import { readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { flow } from '@relayflows/surface'; +import { executeAuthoredFlow } from '../src/authored-flow-executor.js'; +import { executeDurableAuthoredFlow } from '../src/authored-root.js'; +import { loadAuthoredFlow } from '../src/authored-flow-loader.js'; +import { LlmWorker } from '../src/llm-worker.js'; +import { DEFAULT_LOCAL_AGENT_CAPACITY } from '../src/worker-slots.js'; +import { chainFixture } from './flow-chain-fixture.js'; + +const closes: Array<() => Promise> = []; +afterEach(async () => { for (const close of closes.splice(0).reverse()) await close(); }); + +async function setup(capacity: number, probeDelayMs: number) { + const fixture = chainFixture(); + closes.push(() => fixture.close()); + const probes = join(fixture.root, 'probes.jsonl'); + const spans = join(fixture.root, 'spans.jsonl'); + writeFileSync(probes, ''); + writeFileSync(spans, ''); + writeFileSync(join(fixture.root, 'flows.json'), + JSON.stringify({ cli: fixture.wrapper, models: ['test-model', 'second-model'] })); + writeFileSync(fixture.wrapper, `#!/usr/bin/env node +import { receiveWrapperRequest } from ${JSON.stringify(resolve('../../testdata/preflight/wrapper-session.mjs'))}; +import { appendFileSync } from 'node:fs'; +if (process.argv[2] === 'auth') { + appendFileSync(${JSON.stringify(probes)}, JSON.stringify(process.env.RELAYFLOW_MODEL) + '\\n'); + await new Promise(done => setTimeout(done, ${probeDelayMs})); + process.exit(0); +} +const request = await receiveWrapperRequest(); +if (request) { + const start = Date.now(); + await new Promise(done => setTimeout(done, 150)); + appendFileSync(${JSON.stringify(spans)}, JSON.stringify({ start, end: Date.now() }) + '\\n'); + process.stdout.write('{"x":1}'); +} +`); + const client = await fixture.connect(); + const peer = client.createPeer(); + await peer.connect(); + await peer.hello('parallel-llm-regression'); + closes.push(async () => { peer.close(); }); + const worker = new LlmWorker(peer, 'parallel-llm', capacity); + await worker.attach(); + closes.push(() => worker.close()); + const lines = (path: string) => readFileSync(path, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); + async function assertJournals(expected: number) { + const ids = readdirSync(join(fixture.data, 'runs')).filter(name => name.endsWith('.sqlite3')); + expect(ids).toHaveLength(expected); + const expired: unknown[] = []; + for (const name of ids) { + const { entries } = await client.journalRead(name.slice(0, -8), 1, 500); + expired.push(...entries.filter(entry => entry.entry_type === 'step.completed' + && (entry.payload as { completionReason?: string })?.completionReason === 'lease_expired')); + } + expect(expired).toEqual([]); + } + return { fixture, client, assertJournals, assertProbes(count: number) { + expect(lines(probes)).toHaveLength(count); + }, assertCapacity() { + const intervals = lines(spans) as Array<{ start: number; end: number }>; + expect(intervals).toHaveLength(9); + const peak = Math.max(...intervals.map(({ start }) => + intervals.filter(other => other.start <= start && start < other.end).length)); + expect(peak).toBeLessThanOrEqual(capacity); + expect(peak).toBeGreaterThan(0); + } }; +} + +const output = { type: 'object', properties: { x: { type: 'number' } }, required: ['x'], additionalProperties: false }; +const many = flow('parallel-llm', async f => { + await Promise.all(Array.from({ length: 9 }, (_, i) => + f.llm(`Return {"x": ${i}+1} as JSON only.`, { output, model: 'test-model' }))); + f.done('success'); +}); + +describe.each([1, DEFAULT_LOCAL_AGENT_CAPACITY])('parallel llm capacity %i', capacity => { + it('deduplicates concurrent preflight probes', async () => { + const test = await setup(capacity, 0); + const result = await executeAuthoredFlow(many, test.client, undefined, { + flowPath: test.fixture.flowPath, workerCapacity: capacity, + }); + expect(result.completionReason).toBe('success'); + test.assertProbes(1); + test.assertCapacity(); + await test.assertJournals(10); + }, 120_000); + + it('completes nine calls without expired child leases during slow preflight', async () => { + // Before the fix, eight remaining probes block dispatch for 48 s (>30 s). + const test = await setup(capacity, 6_000); + const result = await executeAuthoredFlow(many, test.client, undefined, { + flowPath: test.fixture.flowPath, workerCapacity: capacity, + }); + await test.assertJournals(10); + expect(result.completionReason).toBe('success'); + test.assertProbes(1); + test.assertCapacity(); + }, 120_000); + + it('keeps the durable root lease alive across two cold models', async () => { + // A single cold probe exceeds the root's 30 s lease. Cache alone cannot + // rescue this; the process must keep handling heartbeats while probing. + const test = await setup(capacity, 45_000); + writeFileSync(test.fixture.flowPath, ` +import { flow } from '@relayflows/surface'; +export default flow('two-models', async f => { + await Promise.all(Array.from({ length: 9 }, (_, i) => + f.llm('Return {"x":1}', { output: ${JSON.stringify(output)}, model: i % 2 ? 'second-model' : 'test-model' }))); + f.done('success'); +}); +`); + const loaded = await loadAuthoredFlow(test.fixture.flowPath); + const result = await executeDurableAuthoredFlow(loaded, test.client, undefined, { + dataDir: test.fixture.data, workerCapacity: capacity, + }); + await test.assertJournals(11); + expect(result.completionReason).toBe('success'); + test.assertProbes(2); + test.assertCapacity(); + }, 120_000); +}); diff --git a/packages/sdk/tests/authored-preflight.test.ts b/packages/sdk/tests/authored-preflight.test.ts new file mode 100644 index 00000000..a577999f --- /dev/null +++ b/packages/sdk/tests/authored-preflight.test.ts @@ -0,0 +1,54 @@ +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, expect, it } from 'vitest'; +import { authoredPreflight } from '../src/authored-preflight.js'; +import { SPEC_SCHEMA_VERSION, type FlowSpec } from '../src/spec.js'; + +const directories: string[] = []; +afterEach(() => { for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); }); +function setup() { + const directory = mkdtempSync(join(tmpdir(), 'authored-preflight-')); + directories.push(directory); + const calls = join(directory, 'calls'); + const cli = join(directory, 'wrapper'); + writeFileSync(cli, `#!/usr/bin/env node +import { appendFileSync } from 'node:fs'; +appendFileSync(${JSON.stringify(calls)}, 'probe\\n'); +if (process.argv[2] === '--relayflows-adapter-v1') console.log('relayflows-agent-cli-v1'); +else process.exit(1); +`); + chmodSync(cli, 0o755); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ cli, models: ['allowed'] })); + return { calls, check: authoredPreflight(join(directory, 'test.flow.ts')) }; +} +function spec(id: string, model = 'allowed'): FlowSpec { + return { version: SPEC_SCHEMA_VERSION, name: 'test', steps: [{ id, type: 'llm', prompt: 'hello', model }] }; +} + +it('refuses unknown models before launching any provider probe', async () => { + const { check, calls } = setup(); + const result = await check(spec('one', 'forbidden')); + expect(result.report.diagnostics).toContainEqual(expect.objectContaining({ kind: 'model_unknown' })); + expect(existsSync(calls)).toBe(false); +}); + +it('refuses malformed specs before launching any provider probe', async () => { + const { check, calls } = setup(); + const invalid = spec('one'); + (invalid.steps[0] as { prompt: unknown }).prompt = 42; + expect((await check(invalid)).report.ok).toBe(false); + expect(existsSync(calls)).toBe(false); +}); + +it('shares failed facts across callers but retains each step identity', async () => { + const { check, calls } = setup(); + const results = await Promise.all(['one', 'two'].map(id => check(spec(id)))); + for (const [index, result] of results.entries()) { + expect(result.report.diagnostics).toContainEqual(expect.objectContaining({ + kind: 'cli_unauthenticated', stepId: index === 0 ? 'one' : 'two', + })); + } + // One identification, one exact-model probe, one auth classification. + expect(readFileSync(calls, 'utf8').trim().split('\n')).toHaveLength(3); +}); diff --git a/packages/sdk/tests/cli-probe.test.ts b/packages/sdk/tests/cli-probe.test.ts new file mode 100644 index 00000000..3d893424 --- /dev/null +++ b/packages/sdk/tests/cli-probe.test.ts @@ -0,0 +1,62 @@ +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { probeCli, probeCliAsync } from '../src/cli/cli-probe.js'; +import * as adapters from '../src/cli-adapter.js'; + +const directories: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); +function wrapper(body: string) { + const directory = mkdtempSync(join(tmpdir(), 'probe-drivers-')); + directories.push(directory); + const path = join(directory, 'wrapper'); + writeFileSync(path, '#!/usr/bin/env node\n' + body); + chmodSync(path, 0o755); + return { path, directory }; +} +const identify = `if (process.argv[2] === '--relayflows-adapter-v1') { + console.log('relayflows-agent-cli-v1'); process.exit(0); +}`; + +it.each([ + ['success', 'process.exit(0)', { authenticated: true, modelAvailable: true }], + ['model denied', "process.exit(process.env.RELAYFLOW_MODEL ? 1 : 0)", { authenticated: true, modelAvailable: false }], + ['auth denied', "console.error('person@example.com sk-123456789abcdef'); process.exit(1)", + { authenticated: false, modelAvailable: false, authExitCode: 1, + authFailureDetail: ' ' }], +] as const)('keeps synchronous and asynchronous classification equal: %s', async (_name, body, expected) => { + const { path, directory } = wrapper(identify + body); + const sync = probeCli(path, directory, 'test-model'); + expect(sync).toMatchObject(expected); + expect(await probeCliAsync(path, directory, 'test-model')).toEqual(sync); +}); + +it('keeps missing executables and unsupported identification fail closed', async () => { + const { path, directory } = wrapper("console.log('not a wrapper')"); + expect(await probeCliAsync(path, directory)).toEqual(probeCli(path, directory)); + expect(await probeCliAsync('./missing', directory)).toEqual({ exists: false, authenticated: false }); + expect(await probeCliAsync('relayflows-no-such-executable', directory)).toEqual({ exists: false, authenticated: false }); +}); + +it('reports timeout in both drivers while the async driver leaves the loop free', async () => { + const { path, directory } = wrapper(identify + 'setTimeout(() => {}, 10_000);'); + const original = adapters.modelReadinessProbe; + vi.spyOn(adapters, 'modelReadinessProbe').mockImplementation((kind, model) => + ({ ...original(kind, model), timeoutMs: 100 })); + expect(() => probeCli(path, directory, 'test-model')).toThrow(expect.objectContaining({ detail: 'timeout:100ms' })); + let ticked = false; + const timer = setTimeout(() => { ticked = true; }, 20); + await expect(probeCliAsync(path, directory, 'test-model')).rejects.toMatchObject({ detail: 'timeout:100ms' }); + clearTimeout(timer); + expect(ticked).toBe(true); +}); + +it('reports signal termination in both drivers', async () => { + const { path, directory } = wrapper(identify + "process.kill(process.pid, 'SIGTERM');"); + expect(() => probeCli(path, directory, 'test-model')).toThrow(expect.objectContaining({ detail: 'signal:SIGTERM' })); + await expect(probeCliAsync(path, directory, 'test-model')).rejects.toMatchObject({ detail: 'signal:SIGTERM' }); +}); diff --git a/packages/sdk/tests/communication-preflight.test.ts b/packages/sdk/tests/communication-preflight.test.ts index ddb9cdb8..7d8507b6 100644 --- a/packages/sdk/tests/communication-preflight.test.ts +++ b/packages/sdk/tests/communication-preflight.test.ts @@ -16,7 +16,7 @@ function fixture(cli: string) { const spec: FlowSpec = { version: '0.1.0', cli: executable, steps: [ { id: 'a', type: 'agent', instruction: 'send' }, { id: 'b', type: 'agent', instruction: 'receive' }, ], communication: { links: [{ from: 'a', to: 'b' }] } }; - return { spec, check: (flow = spec) => checkAuthoredFlow(flow, join(root, 'flow.yaml'), config) }; + return { spec, check: (flow = spec) => checkAuthoredFlow(flow, join(root, 'flow.yaml'), { projectConfig: config }) }; } describe('managed CLI preflight', () => { it('probes known CLIs with the same filtered credentials as execution', () => { From 8ae646aa511169b9fa70f659bbf55a37f8f9707e Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 13:52:08 -0700 Subject: [PATCH 09/11] fix(sdk): preserve authored probe compatibility and stdin isolation Session-Id: 01a0d409-e854-7540-a76e-a2f9cd136946 --- packages/sdk/src/cli/check.ts | 20 +++++++++++++++++--- packages/sdk/src/cli/cli-probe.ts | 7 ++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index c3e296ea..a4ccb034 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -108,6 +108,15 @@ export interface CheckExecution { flow?: FlowSpec; } +/** Compatibility options accepted by older authored callers. */ +export interface AuthoredCheckOptions { + projectConfig?: ProjectConfig; + probeCache?: CliProbeOutcomeMap; + communicationChecked?: boolean; +} + +type CliProbeOutcomeMap = Map; + /** * Facts about the *caller*, not about the spec, that change which diagnostics * apply. Preflight stays a pure function of the spec plus environment probes; @@ -176,16 +185,21 @@ function safeRequirements(authoring: FlowSpec, projectCli: string | undefined): export function checkAuthoredFlow( authoring: FlowSpec, path: string, - projectConfig?: ProjectConfig, + projectConfigOrOptions?: ProjectConfig | AuthoredCheckOptions, invocation: CheckInvocation = {}, cliProbeCache?: Map, ): CheckExecution { const absolutePath = resolve(path); try { + const options = projectConfigOrOptions !== undefined && 'projectConfig' in projectConfigOrOptions + ? projectConfigOrOptions + : undefined; + const projectConfig = options?.projectConfig ?? (projectConfigOrOptions as ProjectConfig | undefined); + const effectiveProbeCache = cliProbeCache ?? options?.probeCache; const config = projectConfig ?? readProjectConfig(dirname(absolutePath)); const probes = systemProbes(dirname(absolutePath), config); const result = preflight(authoring, { - ...(cliProbeCache === undefined ? {} : { cliProbeCache }), + ...(effectiveProbeCache === undefined ? {} : { cliProbeCache: effectiveProbeCache }), projectCli: config.cli, projectConfigPath: config.path, projectSearchStart: dirname(absolutePath), @@ -202,7 +216,7 @@ export function checkAuthoredFlow( ) : undefined; if (flow?.steps.some(step => step.type === 'agent' && communicationInstruction(step.instruction))) { - try { checkCommunicationEnvironment(flow); } + try { if (options?.communicationChecked !== true) checkCommunicationEnvironment(flow); } catch (error) { result.ok = false; result.diagnostics.push({ severity: 'refusal', kind: 'probe_failed', diff --git a/packages/sdk/src/cli/cli-probe.ts b/packages/sdk/src/cli/cli-probe.ts index 2268f0ad..ff41bf4e 100644 --- a/packages/sdk/src/cli/cli-probe.ts +++ b/packages/sdk/src/cli/cli-probe.ts @@ -35,7 +35,12 @@ function driveSync(sequence: Generator): T { let next = sequence.next(); while (!next.done) { const request = next.value; - const result = spawnSync(request.executable, request.invocation.args, probeOptions(request)); + const result = spawnSync(request.executable, request.invocation.args, { + ...probeOptions(request), + // Preserve the old synchronous probe contract: provider CLIs must not + // inherit a readable stdin that can block auth/identify probes. + stdio: ['ignore', 'pipe', 'pipe'], + }); const failure = classifySpawnFailure(result.error, result.signal, request.invocation.timeoutMs); if (failure !== undefined) throw failure; next = sequence.next({ status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }); From 3656b1310fc19deaf5552d5d6e5980d8c8a6bce4 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 13:55:05 -0700 Subject: [PATCH 10/11] fix(sdk): detect authored option bags without config key Session-Id: 01a0d409-e854-7540-a76e-a2f9cd136946 --- packages/sdk/src/cli/check.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index a4ccb034..51453707 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -191,7 +191,10 @@ export function checkAuthoredFlow( ): CheckExecution { const absolutePath = resolve(path); try { - const options = projectConfigOrOptions !== undefined && 'projectConfig' in projectConfigOrOptions + const options = projectConfigOrOptions !== undefined + && ('projectConfig' in projectConfigOrOptions + || 'probeCache' in projectConfigOrOptions + || 'communicationChecked' in projectConfigOrOptions) ? projectConfigOrOptions : undefined; const projectConfig = options?.projectConfig ?? (projectConfigOrOptions as ProjectConfig | undefined); From 2dab6ae47d2ae5ef0e6bc2c1b64cdb8b69ede4a8 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 24 Sep 2026 13:58:51 -0700 Subject: [PATCH 11/11] fix(sdk): keep authored option bags from becoming config Session-Id: 01a0d409-e854-7540-a76e-a2f9cd136946 --- packages/sdk/src/cli/check.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index 51453707..24176a3d 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -197,7 +197,9 @@ export function checkAuthoredFlow( || 'communicationChecked' in projectConfigOrOptions) ? projectConfigOrOptions : undefined; - const projectConfig = options?.projectConfig ?? (projectConfigOrOptions as ProjectConfig | undefined); + const projectConfig = options !== undefined + ? options.projectConfig + : projectConfigOrOptions as ProjectConfig | undefined; const effectiveProbeCache = cliProbeCache ?? options?.probeCache; const config = projectConfig ?? readProjectConfig(dirname(absolutePath)); const probes = systemProbes(dirname(absolutePath), config);