diff --git a/evidence/parallel-llm-fix/README.md b/evidence/parallel-llm-fix/README.md new file mode 100644 index 000000000..8da2e3d5c --- /dev/null +++ b/evidence/parallel-llm-fix/README.md @@ -0,0 +1,184 @@ +# Parallel LLM lease fix: commands and captured output + +Implementation commit: 78f7012. +The base commit is f6ece41d31e1908649c0b4540bfc53d189caaf7e. +The historical diagnosis is in ../parallel-llm-lease. Those old numbers are +not post-fix results. All links below are literal captured output, including +failures. Commands run from the repository root unless a different cwd is stated. + +## Kernel and SDK checks + +From packages/sdk: + +| Literal command | Captured output | +| --- | --- | +| `npm run test:prep` | [prep.txt](prep.txt) | +| `npm run typecheck` | [typecheck-final.txt](typecheck-final.txt) | +| `npm run typecheck:tests` | [test-types-final.txt](test-types-final.txt) | +| `npm run build` | [build.txt](build.txt) | +| `npx vitest run tests/authored-parallel-llm.test.ts` | [regression-pass.txt](regression-pass.txt) | +| `npx vitest run tests/cli-probe.test.ts tests/authored-preflight.test.ts tests/communication-preflight.test.ts tests/bundle-preflight.test.ts tests/preflight.test.ts tests/cli.test.ts tests/authored-parallel-agents.test.ts tests/authored-parallel-llm.test.ts` | [final-focused.txt](final-focused.txt) | +| `npm test` | [sdk-suite.txt](sdk-suite.txt) | + +The kernel build is a fixture prerequisite; no kernel code changed. +The complete SDK suite was attempted, not declared green. It includes environment +and fixture failures. To check one failing group on the unchanged base: + +```sh +git worktree add --detach /tmp/flows-lease-baseline f6ece41d31e1908649c0b4540bfc53d189caaf7e +ln -s /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/node_modules /tmp/flows-lease-baseline/packages/sdk/node_modules +cd /tmp/flows-lease-baseline/packages/sdk +npx vitest run tests/hosted-extension-isolation.test.ts +``` + +Output: [baseline-hosted.txt](baseline-hosted.txt). This reproduces 15 failures: +missing bubblewrap and missing local Surface build files. This comparison does +not establish the cause of every failure in the complete suite. + +## Mutation verification + +```sh +python evidence/parallel-llm-fix/mutate.py cache +python evidence/parallel-llm-fix/mutate.py async +``` + +[mutate.py](mutate.py) saves the selected source file as bytes, replaces exactly +the chosen implementation, captures the test failure, restores the saved bytes, +asserts equality, and reruns the same command. The commands and both complete +outputs are pasted in [mutations.md](mutations.md). + +- Cache mutation: construct a fresh authored preflight for each call instead of + sharing it. Both capacities run nine probes instead of one. +- Async mutation: route the async probe entry through the synchronous driver. + Two 45-second cold models starve the durable root lease; the test fails with + `lease_conflict: attempt has no active worker lease`. +- These protections overlap. With async probing retained, removing the cache + does **not** expire child leases; the slow test fails its probe-count assertion. + It would be false to claim cache-only removal causes expiry after Phase 2. + +For the original child-lease failure, the new test was also copied to the +unchanged base worktree, with the existing kernel explicitly selected: + +```sh +cp packages/sdk/tests/authored-parallel-llm.test.ts /tmp/flows-lease-baseline/packages/sdk/tests/authored-parallel-llm.test.ts +cd /tmp/flows-lease-baseline/packages/sdk +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npx vitest run tests/authored-parallel-llm.test.ts -t 'parallel llm capacity 1.*completes nine' +``` + +Output: [baseline-child-lease.txt](baseline-child-lease.txt). + +## Original diagnosis harness, rerun after the fix + +```sh +cp evidence/parallel-llm-lease/diagnosis-harness.test.ts packages/sdk/tests/zz-scratch-repro.test.ts +cd packages/sdk +SCRATCH_PROBE_MS=4000 SCRATCH_CAP=1 npx vitest run tests/zz-scratch-repro.test.ts +rm tests/zz-scratch-repro.test.ts +``` + +Output: [diagnosis-post-fix.txt](diagnosis-post-fix.txt). Its identification +count includes the nine actual wrapper sessions; only one identify/auth round +is preflight. This scratch harness is historical instrumentation, not the +regression assertion; the committed tests discover every run journal from disk. + +## Provider repro + +```sh +git fetch origin feat/examples-prompt-lab +git worktree add --detach /tmp/flows-prompt-lab origin/feat/examples-prompt-lab +git -C /tmp/flows-prompt-lab merge --no-edit relayflow/flows-software-garden-860ae350 +``` + +The fetched branch was 735f2e0. Merge refused unrelated histories: +[worktree-merge.txt](worktree-merge.txt). Instead the isolated example worktree +used this checkout's built CLI and dependencies: + +```sh +ln -s /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/node_modules /tmp/flows-prompt-lab/examples/prompt-lab/node_modules +cd /tmp/flows-prompt-lab/examples/prompt-lab +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js run evidence/runtime-findings/runtime-parallel-llm-repro.flow.ts --local-agent --agent-capacity 1 --input '{"n":1}' --data-dir /tmp/flows-verified-capacity1 +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js run evidence/runtime-findings/runtime-parallel-llm-repro.flow.ts --local-agent --agent-capacity 4 --input '{"n":1}' --data-dir /tmp/flows-verified-capacity4 +``` + +Outputs: [capacity 1](repro-final-capacity1.txt), [capacity 4](repro-final-capacity4.txt). +Earlier runs are also retained as repro-capacity1.txt and repro-capacity4.txt. + +```sh +python evidence/parallel-llm-fix/scan-journals.py /tmp/flows-verified-capacity1 /tmp/flows-verified-capacity4 +``` + +Output: [repro-final-journals.txt](repro-final-journals.txt). Every SQLite journal +is inspected, including the root and deterministic children. + +## prompt-lab Promise.all restoration + +[prompt-lab-parallel.patch](prompt-lab-parallel.patch) restores parallel calls +in runEngine and both previously serialized loops in new-agency. It is a +downstream verification patch, not a change to the example branch's history. + +The example's existing proof script was copied to prove-fixed.sh with +`npx flows` replaced by this checkout's absolute `node .../dist/cli.js` +command. That exact script is preserved here. Run it from the example directory: + +```sh +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd FLOWS_DATA_DIR=/tmp/flows-prompt-lab-data bash prove-fixed.sh /tmp/flows-prompt-lab-proof +``` + +Output: [prompt-lab.txt](prompt-lab.txt); full commands, model-step progress, +reviewer edit and resume failure: [prompt-lab-run/](prompt-lab-run/). +The parallel run produced a nine-row grid and reached both human gates, but +the final resume failed with `unawaited_step`. Therefore acceptance box 3 +is **not fully verified**. No fix to that lifecycle failure is claimed here. + +```sh +python evidence/parallel-llm-fix/scan-journals.py /tmp/flows-prompt-lab-data +``` + +Output: [prompt-lab-journals.txt](prompt-lab-journals.txt). + +## Standalone packaging + +```sh +node scripts/build-standalone-cli.mjs bun-linux-x64 /tmp/flows-parallel-fixed +``` + +Output: [standalone-build-final.txt](standalone-build-final.txt). The same build +command in the base worktree, with outfile /tmp/flows-parallel-base, is captured +in [standalone-baseline-build.txt](standalone-baseline-build.txt). + +From the example directory, smoke commands: + +```sh +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd /tmp/flows-parallel-fixed run evidence/runtime-findings/runtime-parallel-llm-repro.flow.ts --local-agent --agent-capacity 4 --input '{"n":1}' --data-dir /tmp/flows-final-repro-capacity4 +RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd /tmp/flows-parallel-base run evidence/runtime-findings/runtime-parallel-llm-repro.flow.ts --local-agent --agent-capacity 4 --input '{"n":1}' --data-dir /tmp/flows-base-standalone-data +``` + +Both refuse importing the authored flow: [changed](standalone-smoke.txt), +[base](standalone-baseline-smoke.txt). Build success is not a claim of standalone +runtime success. The Node CLI provider repro above is separate. + +## Intermediate checks retained for transparency + +regression.txt has the initial test's wrong journal-count expectation (it omitted +the deterministic f.done child). related.txt records the initial async signal +classification failure, fixed before related-pass.txt. Their commands were, +respectively, the regression command above and: + +```sh +npx vitest run tests/cli-probe.test.ts tests/preflight.test.ts tests/cli.test.ts tests/authored-parallel-agents.test.ts tests/communication-environment-preflight.test.ts +``` + +static-refusals.txt used: + +```sh +npx vitest run tests/authored-preflight.test.ts tests/cli-probe.test.ts tests/preflight.test.ts +``` + +Authentication material is not included. claude-auth.txt contains only the +output of: + +```sh +claude auth status | python -c 'import json,sys; d=json.load(sys.stdin); print(json.dumps({k:d.get(k) for k in ("loggedIn","authMethod")}))' +``` + +Captured stdout/stderr and patch files retain tool-generated whitespace verbatim. diff --git a/evidence/parallel-llm-fix/baseline-child-lease.txt b/evidence/parallel-llm-fix/baseline-child-lease.txt new file mode 100644 index 000000000..2744752f7 --- /dev/null +++ b/evidence/parallel-llm-fix/baseline-child-lease.txt @@ -0,0 +1,63 @@ + + RUN v2.1.9 /tmp/flows-lease-baseline/packages/sdk + + ❯ tests/authored-parallel-llm.test.ts (6 tests | 1 failed | 5 skipped) 55669ms + × parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 55668ms + → step_failed: journal step "llm-1" completed with lease_expired +Inspect: flows replay 01M36D0HK698MR5P0FHJ0YC4W6 --at llm-1 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight +AuthoredFlowExecutionError: step_failed: journal step "llm-1" completed with lease_expired +Inspect: flows replay 01M36D0HK698MR5P0FHJ0YC4W6 --at llm-1 + ❯ Module.readCompletedStepOutput src/authored-step-output.ts:85:11 + 83| ...edges, + 84| }); + 85| throw new AuthoredFlowExecutionError('step_failed', message, reaso… + | ^ + 86| } + 87| await recordAuthoredChild(journal, context.rootRunId, { + ❯ WorkerSlots.run src/worker-slots.ts:44:14 + ❯ Object.llm src/authored-worker-step.ts:212:22 + ❯ Module.observeStep src/progress.ts:48:20 + ❯ AuthoredFlowOperation.begin src/authored-flow-operation.ts:174:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + +⎯⎯⎯⎯⎯⎯ Unhandled Errors ⎯⎯⎯⎯⎯⎯ + +Vitest caught 1 unhandled error during the test run. +This might cause false positive tests. Resolve unhandled errors to make sure your tests are not affected. + +⎯⎯⎯⎯ Unhandled Rejection ⎯⎯⎯⎯⎯ +Error: Agent lease is already expired for 01M36D0HK698MR5P0FHJ0YC4W6/llm-1. + ❯ armExpiry src/worker-lease.ts:20:13 + 18| const remaining = deadline - Date.now(); + 19| if (!Number.isFinite(remaining) || remaining <= 0) { + 20| throw new Error(`Agent lease is already expired for ${dispatch.r… + | ^ + 21| } + 22| latestDeadline = deadline; + ❯ Module.withWorkerLease src/worker-lease.ts:47:5 + ❯ LlmWorker.execute src/llm-worker.ts:60:46 + ❯ JournalClient.onDispatch src/llm-worker.ts:49:26 + ❯ JournalClient.emit node:events:508:20 + ❯ JournalClient.onLine src/journal-client.ts:168:12 + ❯ JournalClient.onData src/journal-client.ts:144:33 + ❯ Socket. src/journal-client.ts:109:43 + ❯ Socket.emit node:events:508:20 + ❯ addChunk node:internal/streams/readable:564:12 + +This error originated in "tests/authored-parallel-llm.test.ts" test file. It doesn't mean the error was thrown inside the file itself, but while it was running. +The latest test that might've caused the error is "completes nine calls without expired child leases during slow preflight". It might mean one of the following: +- The error was thrown, while Vitest was running this test. +- If the error occurred after the test had been completed, this was the last documented test before it was thrown. +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ + + Test Files 1 failed (1) + Tests 1 failed | 5 skipped (6) + Errors 1 error + Start at 05:49:49 + Duration 56.91s (transform 599ms, setup 0ms, collect 1.06s, tests 55.67s, environment 0ms, prepare 46ms) + diff --git a/evidence/parallel-llm-fix/baseline-hosted.txt b/evidence/parallel-llm-fix/baseline-hosted.txt new file mode 100644 index 000000000..66c3d3282 --- /dev/null +++ b/evidence/parallel-llm-fix/baseline-hosted.txt @@ -0,0 +1,413 @@ + + RUN v2.1.9 /tmp/flows-lease-baseline/packages/sdk + + ❯ tests/hosted-extension-isolation.test.ts (22 tests | 15 failed) 286ms + × hosted extension capability isolation > executes the exact capability-only handler for a queued receipt 8ms + → bubblewrap is unavailable + × hosted extension capability isolation > executes the exact capability-only handler for a duplicate receipt 2ms + → bubblewrap is unavailable + × hosted extension capability isolation > launches through the captured process primitive after builtin export synchronization 4ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > ignores inherited launcher overrides and decodes manifests with the captured Buffer intrinsic 3ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > streams verified bytes when the live store is replaced and no writable staging path exists 207ms + → promise rejected "Error: Hosted extension sandbox exited wi… { code: '…' }" instead of resolving + × hosted extension capability isolation > mounts pinned private Surface bytes when the live package changes before launch 1ms + → ENOENT: no such file or directory, copyfile '/tmp/flows-lease-baseline/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-Mim52z/dist/flow.js' + × hosted extension capability isolation > refuses Surface runtime bytes that differ from the reviewed pin before launch 1ms + → ENOENT: no such file or directory, copyfile '/tmp/flows-lease-baseline/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-6b3BQt/dist/flow.js' + × hosted extension capability isolation > refuses oversized Surface files through the bounded descriptor reader 1ms + → ENOENT: no such file or directory, copyfile '/tmp/flows-lease-baseline/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-IqCOBL/dist/flow.js' + × hosted extension capability isolation > shields verified Surface files before async settlement 2ms + → ENOENT: no such file or directory, copyfile '/tmp/flows-lease-baseline/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-7aruFD/dist/flow.js' + × hosted extension capability isolation > preserves a typed host refusal while disclosing only a fixed marker to the child 5ms + → expected Error: bubblewrap is unavailable { code: '…' } to be Error: private Cloud policy detail { code: '…' } // Object.is equality + × hosted extension capability isolation > denies ambient credentials, host files, writes, network, subprocesses, and undeclared context verbs 7ms + → bubblewrap is unavailable + × hosted extension capability isolation > enforces OS address-space and data bounds on native Buffer allocation 8ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + × hosted extension capability isolation > blocks extra handler fields and authority-bearing receipt fields at the parent port 3ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension capability isolation > constructs adapter authority with the captured freeze intrinsic 3ms + → bubblewrap is unavailable + × hosted extension capability isolation > writes the Surface manifest and protocol without inherited toJSON behavior 3ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + +⎯⎯⎯⎯⎯⎯ Failed Tests 15 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > executes the exact capability-only handler for a queued receipt + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > executes the exact capability-only handler for a duplicate receipt +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:221:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > launches through the captured process primitive after builtin export synchronization +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:283:11 + 281| input: descriptor(), + 282| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 283| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 284| } finally { + 285| process.execPath = originalExecPath; + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:277:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > ignores inherited launcher overrides and decodes manifests with the captured Buffer intrinsic +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:373:11 + 371| input: descriptor('delivery-options'), + 372| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 373| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 374| } finally { + 375| Buffer.prototype.toString = bufferToString; + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:367:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > streams verified bytes when the live store is replaced and no writable staging path exists +AssertionError: promise rejected "Error: Hosted extension sandbox exited wi… { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:431:7 + 429| return { receiptId: 'receipt-1', status: 'queued' }; + 430| } }, + 431| })).resolves.toEqual({ completionReason: 'success', capabilityCall… + | ^ + 432| expect(calls).toBe(1); + 433| expect(readFileSync(join(installed.directory, 'babysitter.flow.ts'… + +Caused by: Error: Hosted extension sandbox exited without a valid completion (exit 1): /tmp/hosted-bwrap-wrapper-o4PuPs/bwrap-wrapper:20 +if (child.error) throw child.error; + ^ + +Error: spawnSync /usr/bin/bwrap ENOENT + at Object.spawnSync (node:internal/child_process:1103:20) + at spawnSync (node:child_process:911:24) + at Object. (/tmp/hosted-bwrap-wrapper-o4PuPs/bwrap-wrapper:19:15) + at Module._compile (node:internal/modules/cjs/loader:1809:14) + at Object..js (node:internal/modules/cjs/loader:1940:10) + at Module.load (node:internal/modules/cjs/loader:1530:32) + at Module._load (node:internal/modules/cjs/loader:1332:12) + at wrapModuleLoad (node:internal/modules/cjs/loader:255:19) + at Module.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:154:5) + at node:internal/main/run_main_module:33:47 { + errno: -2, + code: 'ENOENT', + syscall: 'spawnSync /usr/bin/bwrap', + path: '/usr/bin/bwrap', + spawnargs: [ + '--unshare-all', + '--die-with-parent', + '--new-session', + '--clearenv', + '--cap-drop', + 'ALL', + '--dir', + '/usr', + '--ro-bind', + '/usr/lib', + '/usr/lib', + '--ro-bind', + '/usr/lib64', + '/usr/lib64', + '--ro-bind', + '/usr/lib', + '/lib', + '--ro-bind', + '/usr/lib64', + '/lib64', + '--proc', + '/proc', + '--dev', + '/dev', + '--tmpfs', + '/tmp', + '--dir', + '/runtime', + '--dir', + '/extension', + '--dir', + '/extension/node_modules', + '--dir', + '/extension/node_modules/@relayflows', + '--dir', + '/extension/node_modules/@relayflows/surface', + '--dir', + '/extension/node_modules/@relayflows/surface/dist', + '--dir', + '/extension/node_modules/@relayflows/surface/dist/helpers', + '--dir', + '/extension/node_modules/@relayflows/surface/dist/triggers', + '--dir', + '/extension/src', + '--perms', + '0500', + '--ro-bind-data', + '4', + '/runtime/node', + '--perms', + '0400', + '--ro-bind-data', + '5', + '/runtime/runner.mjs', + '--perms', + '0400', + '--ro-bind-data', + '6', + '/extension/node_modules/@relayflows/surface/package.json', + '--perms', + '0400', + '--ro-bind-data', + '7', + '/extension/node_modules/@relayflows/surface/index.js', + '--perms', + '0400', + '--ro-bind-data', + '8', + '/extension/node_modules/@relayflows/surface/runtime.js', + '--perms', + '0400', + '--ro-bind-data', + '9', + '/extension/node_modules/@relayflows/surface/dist/flow.js', + '--perms', + '0400', + '--ro-bind-data', + '10', + '/extension/node_modules/@relayflows/surface/dist/helpers/providers.js', + '--perms', + '0400', + '--ro-bind-data', + '11', + '/extension/node_modules/@relayflows/surface/dist/provider-trigger.js', + '--perms', + '0400', + '--ro-bind-data', + '12', + '/extension/node_modules/@relayflows/surface/dist/schedule.js', + '--perms', + '0400', + '--ro-bind-data', + '13', + '/extension/node_modules/@relayflows/surface/dist/triggers.js', + '--perms', + '0400', + '--ro-bind-data', + '14', + '/extension/node_modules/@relayflows/surface/dist/triggers/github.js', + '--perms', + ... 27 more items + ] +} + +Node.js v25.6.0 + + ❯ Object. ../../../hosted-bwrap-wrapper-o4PuPs/bwrap-wrapper:19:15 + ❯ refuse src/hosted-extension-protocol.ts:135:21 + ❯ ChildProcess. src/hosted-extension-protocol.ts:234:21 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > mounts pinned private Surface bytes when the live package changes before launch +Error: ENOENT: no such file or directory, copyfile '/tmp/flows-lease-baseline/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-Mim52z/dist/flow.js' + ❯ surfaceFixture tests/hosted-extension-isolation.test.ts:109:5 + 107| const target = join(surfaceRoot, 'dist', file); + 108| mkdirSync(resolve(target, '..'), { recursive: true }); + 109| copyFileSync(join(sourceRoot, 'dist', file), target); + | ^ + 110| } + 111| return surfaceRoot; + ❯ tests/hosted-extension-isolation.test.ts:437:25 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > refuses Surface runtime bytes that differ from the reviewed pin before launch +Error: ENOENT: no such file or directory, copyfile '/tmp/flows-lease-baseline/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-6b3BQt/dist/flow.js' + ❯ surfaceFixture tests/hosted-extension-isolation.test.ts:109:5 + 107| const target = join(surfaceRoot, 'dist', file); + 108| mkdirSync(resolve(target, '..'), { recursive: true }); + 109| copyFileSync(join(sourceRoot, 'dist', file), target); + | ^ + 110| } + 111| return surfaceRoot; + ❯ tests/hosted-extension-isolation.test.ts:462:25 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[6/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > refuses oversized Surface files through the bounded descriptor reader +Error: ENOENT: no such file or directory, copyfile '/tmp/flows-lease-baseline/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-IqCOBL/dist/flow.js' + ❯ surfaceFixture tests/hosted-extension-isolation.test.ts:109:5 + 107| const target = join(surfaceRoot, 'dist', file); + 108| mkdirSync(resolve(target, '..'), { recursive: true }); + 109| copyFileSync(join(sourceRoot, 'dist', file), target); + | ^ + 110| } + 111| return surfaceRoot; + ❯ tests/hosted-extension-isolation.test.ts:483:25 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[7/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > shields verified Surface files before async settlement +Error: ENOENT: no such file or directory, copyfile '/tmp/flows-lease-baseline/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-7aruFD/dist/flow.js' + ❯ surfaceFixture tests/hosted-extension-isolation.test.ts:109:5 + 107| const target = join(surfaceRoot, 'dist', file); + 108| mkdirSync(resolve(target, '..'), { recursive: true }); + 109| copyFileSync(join(sourceRoot, 'dist', file), target); + | ^ + 110| } + 111| return surfaceRoot; + ❯ tests/hosted-extension-isolation.test.ts:507:25 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[8/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > preserves a typed host refusal while disclosing only a fixed marker to the child +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to be Error: private Cloud policy detail { code: '…' } // Object.is equality + +- Expected ++ Received + +- [Error: private Cloud policy detail] ++ [Error: bubblewrap is unavailable] + + ❯ tests/hosted-extension-isolation.test.ts:560:5 + 558| provider: 'github', eventType: 'pull_request.labeled', deliveryI… + 559| }); + 560| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 561| artifact: await artifact(source), manifest: validateFlowExtensio… + 562| babysitterTurn: { queue: async () => { throw refusal; } }, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[9/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > denies ambient credentials, host files, writes, network, subprocesses, and undeclared context verbs +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:624:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[10/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > enforces OS address-space and data bounds on native Buffer allocation +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + +- Expected ++ Received + +- Object { ++ PluginError { + "code": "plugin_unsupported", +- "message": StringMatching /(?:Failed to allocate memory|Array buffer allocation failed)/u, + } + + ❯ tests/hosted-extension-isolation.test.ts:646:5 + 644| }); + 645| let calls = 0; + 646| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 647| artifact: installed, + 648| manifest: validateFlowExtensionManifest(manifest()), + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[11/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > blocks extra handler fields and authority-bearing receipt fields at the parent port +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + +- Expected ++ Received + +- Object { +- "code": "plugin_event_unroutable", ++ PluginError { ++ "code": "plugin_unsupported", + } + + ❯ tests/hosted-extension-isolation.test.ts:675:5 + 673| }); + 674| let calls = 0; + 675| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 676| artifact: await artifact(source), manifest: validateFlowExtensio… + 677| babysitterTurn: { queue: async () => { calls += 1; return { rece… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[12/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > constructs adapter authority with the captured freeze intrinsic +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:745:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[13/15]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > writes the Surface manifest and protocol without inherited toJSON behavior +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:788:11 + 786| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 787| timeoutMs: 3_000, + 788| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 789| } finally { + 790| if (previous === undefined) delete (Object.prototype as { toJS… + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:781:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[14/15]⎯ + + Test Files 1 failed (1) + Tests 15 failed | 7 passed (22) + Start at 05:47:28 + Duration 2.51s (transform 1.11s, setup 0ms, collect 2.01s, tests 286ms, environment 0ms, prepare 76ms) + diff --git a/evidence/parallel-llm-fix/baseline-worktree.txt b/evidence/parallel-llm-fix/baseline-worktree.txt new file mode 100644 index 000000000..f7834fc95 --- /dev/null +++ b/evidence/parallel-llm-fix/baseline-worktree.txt @@ -0,0 +1,2 @@ +Preparing worktree (detached HEAD f6ece41) +HEAD is now at f6ece41 port(drive-cloud): emit the tick as a v2 kernel spec (#455) diff --git a/evidence/parallel-llm-fix/build.txt b/evidence/parallel-llm-fix/build.txt new file mode 100644 index 000000000..e83dbde7b --- /dev/null +++ b/evidence/parallel-llm-fix/build.txt @@ -0,0 +1,4 @@ + +> @relayflows/sdk@2.0.29 build +> tsc && node scripts/make-cli-executable.mjs + diff --git a/evidence/parallel-llm-fix/claude-auth.txt b/evidence/parallel-llm-fix/claude-auth.txt new file mode 100644 index 000000000..dad7b0d96 --- /dev/null +++ b/evidence/parallel-llm-fix/claude-auth.txt @@ -0,0 +1 @@ +{"loggedIn": true, "authMethod": "oauth_token"} diff --git a/evidence/parallel-llm-fix/diagnosis-post-fix.txt b/evidence/parallel-llm-fix/diagnosis-post-fix.txt new file mode 100644 index 000000000..6cd9a422e --- /dev/null +++ b/evidence/parallel-llm-fix/diagnosis-post-fix.txt @@ -0,0 +1,25 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +stdout | tests/zz-scratch-repro.test.ts > scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 +{ + "elapsedMs": 4716, + "probeCount": 11, + "byKind": { + "identify": 10, + "auth": 1 + }, + "completionReason": "success" +} + +stdout | tests/zz-scratch-repro.test.ts > scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 +lease_expired attempts: [] + + ✓ tests/zz-scratch-repro.test.ts (1 test) 4792ms + ✓ scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 4792ms + + Test Files 1 passed (1) + Tests 1 passed (1) + Start at 05:46:03 + Duration 5.97s (transform 576ms, setup 0ms, collect 1.01s, tests 4.79s, environment 0ms, prepare 46ms) + diff --git a/evidence/parallel-llm-fix/fetch.txt b/evidence/parallel-llm-fix/fetch.txt new file mode 100644 index 000000000..c67ee1129 --- /dev/null +++ b/evidence/parallel-llm-fix/fetch.txt @@ -0,0 +1,3 @@ +From https://github.com/agentworkforce/flows + * branch feat/examples-prompt-lab -> FETCH_HEAD + * [new branch] feat/examples-prompt-lab -> origin/feat/examples-prompt-lab diff --git a/evidence/parallel-llm-fix/final-focused.txt b/evidence/parallel-llm-fix/final-focused.txt new file mode 100644 index 000000000..ba2021626 --- /dev/null +++ b/evidence/parallel-llm-fix/final-focused.txt @@ -0,0 +1,35 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/preflight.test.ts (67 tests) 148ms + ✓ tests/cli.test.ts (70 tests) 2801ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 726ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 532ms + ✓ tests/cli-probe.test.ts (6 tests) 861ms + ✓ tests/bundle-preflight.test.ts (4 tests) 1443ms + ✓ bundle execution preflight > ignores surrounding cache configuration on a verified cache hit 626ms + ✓ bundle execution preflight > uses the built alias for a nameless flow even in a digest-only cache directory 786ms + ✓ tests/communication-preflight.test.ts (13 tests) 42ms + ✓ tests/authored-preflight.test.ts (3 tests) 98ms + ✓ tests/authored-parallel-agents.test.ts (8 tests) 15720ms + ✓ authored steps under local workers with capacity > runs more concurrent f.llm calls than the worker holds side by side, never more than its capacity 2201ms + ✓ authored steps under local workers with capacity > completes more concurrent f.agent calls than the worker holds: the overflow waits for a slot instead of parking 3688ms + ✓ authored steps under local workers with capacity > runs agents in distinct working directories side by side (the kernel carries cwd) 831ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a symlink alias of the same directory 2110ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a directory nested inside the other 1920ms + ✓ authored steps under local workers with capacity > never starts queued agents once the body has failed 1990ms + ✓ authored steps under local workers with capacity > never starts a queued agent when the agent holding the only slot fails 2053ms + ✓ authored steps under local workers with capacity > parks the overflow when the body is not told the capacity (the defect this closes) 927ms + ✓ tests/authored-parallel-llm.test.ts (6 tests) 119261ms + ✓ parallel llm capacity 1 > deduplicates concurrent preflight probes 5453ms + ✓ parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 8940ms + ✓ parallel llm capacity 1 > keeps the durable root lease alive across two cold models 47791ms + ✓ parallel llm capacity 4 > deduplicates concurrent preflight probes 3310ms + ✓ parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight 7300ms + ✓ parallel llm capacity 4 > keeps the durable root lease alive across two cold models 46466ms + + Test Files 8 passed (8) + Tests 177 passed (177) + Start at 05:50:30 + Duration 123.32s (transform 2.98s, setup 0ms, collect 10.31s, tests 140.37s, environment 1ms, prepare 555ms) + diff --git a/evidence/parallel-llm-fix/mutate.py b/evidence/parallel-llm-fix/mutate.py new file mode 100644 index 000000000..c585dbc5f --- /dev/null +++ b/evidence/parallel-llm-fix/mutate.py @@ -0,0 +1,44 @@ +"""Revert one fix, capture failure, restore byte-for-byte, capture success.""" +import pathlib +import subprocess +import sys + +root = pathlib.Path(__file__).resolve().parents[2] +sdk = root / 'packages/sdk' +mode = sys.argv[1] +if mode == 'cache': + path = sdk / 'src/authored-worker-step.ts' + before = 'await check(authoring)' + after = 'await authoredPreflight(flowPath)(authoring)' + pattern = 'deduplicates|completes nine' +elif mode == 'async': + path = sdk / 'src/cli/cli-probe.ts' + before = 'const sequence = probeSequence(...args);' + after = 'return probeCli(...args);\n const sequence = probeSequence(...args);' + pattern = 'parallel llm capacity 1.*keeps the durable' +else: + raise ValueError(mode) +original = path.read_bytes() +assert before.encode() in original + +def run(label): + command = ['npx', 'vitest', 'run', 'tests/authored-parallel-llm.test.ts', '-t', pattern] + log = root / 'evidence/parallel-llm-fix' / f'mutation-{mode}-{label}.txt' + with log.open('w') as out: + out.write(f'cwd: packages/sdk\ncommand: {command!r}\n') + out.flush() + result = subprocess.run(command, cwd=sdk, stdout=out, stderr=subprocess.STDOUT) + out.write(f'\nexit: {result.returncode}\n') + return result.returncode + +try: + path.write_bytes(original.replace(before.encode(), after.encode(), 1)) + failed = run('reverted') +finally: + path.write_bytes(original) +assert path.read_bytes() == original +print(f'{mode}: restored byte-for-byte', flush=True) +passed = run('restored') +assert failed != 0, 'mutation did not fail' +assert passed == 0, 'restored fix did not pass' +print(f'{mode}: reverted exit={failed}; restored exit={passed}', flush=True) diff --git a/evidence/parallel-llm-fix/mutation-async-restored.txt b/evidence/parallel-llm-fix/mutation-async-restored.txt new file mode 100644 index 000000000..8b8fe3d1f --- /dev/null +++ b/evidence/parallel-llm-fix/mutation-async-restored.txt @@ -0,0 +1,15 @@ +cwd: packages/sdk +command: ['npx', 'vitest', 'run', 'tests/authored-parallel-llm.test.ts', '-t', 'parallel llm capacity 1.*keeps the durable'] + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-parallel-llm.test.ts (6 tests | 5 skipped) 47947ms + ✓ parallel llm capacity 1 > keeps the durable root lease alive across two cold models 47946ms + + Test Files 1 passed (1) + Tests 1 passed | 5 skipped (6) + Start at 05:55:31 + Duration 49.09s (transform 558ms, setup 0ms, collect 978ms, tests 47.95s, environment 0ms, prepare 43ms) + + +exit: 0 diff --git a/evidence/parallel-llm-fix/mutation-async-reverted.txt b/evidence/parallel-llm-fix/mutation-async-reverted.txt new file mode 100644 index 000000000..fcf670840 --- /dev/null +++ b/evidence/parallel-llm-fix/mutation-async-reverted.txt @@ -0,0 +1,32 @@ +cwd: packages/sdk +command: ['npx', 'vitest', 'run', 'tests/authored-parallel-llm.test.ts', '-t', 'parallel llm capacity 1.*keeps the durable'] + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/authored-parallel-llm.test.ts (6 tests | 1 failed | 5 skipped) 91068ms + × parallel llm capacity 1 > keeps the durable root lease alive across two cold models 91067ms + → lease_conflict: attempt has no active worker lease + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > keeps the durable root lease alive across two cold models +JournalProtocolError: lease_conflict: attempt has no active worker lease + ❯ JournalClient.onLine src/journal-client.ts:165:27 + 163| if (pending.timer !== undefined) clearTimeout(pending.timer); + 164| if (res.ok) pending.resolve(res.result); + 165| else pending.reject(new JournalProtocolError(res.error.code, res… + | ^ + 166| } else { + 167| const ev = msg as ServerEvent; + ❯ JournalClient.onData src/journal-client.ts:144:33 + ❯ Socket. src/journal-client.ts:109:43 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 5 skipped (6) + Start at 05:53:59 + Duration 92.22s (transform 560ms, setup 0ms, collect 978ms, tests 91.07s, environment 0ms, prepare 46ms) + + +exit: 1 diff --git a/evidence/parallel-llm-fix/mutation-async.txt b/evidence/parallel-llm-fix/mutation-async.txt new file mode 100644 index 000000000..797559d89 --- /dev/null +++ b/evidence/parallel-llm-fix/mutation-async.txt @@ -0,0 +1,2 @@ +async: restored byte-for-byte +async: reverted exit=1; restored exit=0 diff --git a/evidence/parallel-llm-fix/mutation-cache-restored.txt b/evidence/parallel-llm-fix/mutation-cache-restored.txt new file mode 100644 index 000000000..2cd7ed98b --- /dev/null +++ b/evidence/parallel-llm-fix/mutation-cache-restored.txt @@ -0,0 +1,18 @@ +cwd: packages/sdk +command: ['npx', 'vitest', 'run', 'tests/authored-parallel-llm.test.ts', '-t', 'deduplicates|completes nine'] + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-parallel-llm.test.ts (6 tests | 2 skipped) 20071ms + ✓ parallel llm capacity 1 > deduplicates concurrent preflight probes 2799ms + ✓ parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 8690ms + ✓ parallel llm capacity 4 > deduplicates concurrent preflight probes 1311ms + ✓ parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight 7270ms + + Test Files 1 passed (1) + Tests 4 passed | 2 skipped (6) + Start at 05:53:37 + Duration 21.25s (transform 569ms, setup 0ms, collect 1.01s, tests 20.07s, environment 0ms, prepare 49ms) + + +exit: 0 diff --git a/evidence/parallel-llm-fix/mutation-cache-reverted.txt b/evidence/parallel-llm-fix/mutation-cache-reverted.txt new file mode 100644 index 000000000..2b544490f --- /dev/null +++ b/evidence/parallel-llm-fix/mutation-cache-reverted.txt @@ -0,0 +1,66 @@ +cwd: packages/sdk +command: ['npx', 'vitest', 'run', 'tests/authored-parallel-llm.test.ts', '-t', 'deduplicates|completes nine'] + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/authored-parallel-llm.test.ts (6 tests | 4 failed | 2 skipped) 22493ms + × parallel llm capacity 1 > deduplicates concurrent preflight probes 3643ms + → expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + × parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 10266ms + → expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + × parallel llm capacity 4 > deduplicates concurrent preflight probes 1273ms + → expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + × parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight 7311ms + → expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 4 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > deduplicates concurrent preflight probes + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 4 > deduplicates concurrent preflight probes +AssertionError: expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + +- Expected ++ Received + +- 1 ++ 9 + + ❯ Object.assertProbes tests/authored-parallel-llm.test.ts:61:27 + 59| } + 60| return { fixture, client, assertJournals, assertProbes(count: number… + 61| expect(lines(probes)).toHaveLength(count); + | ^ + 62| }, assertCapacity() { + 63| const intervals = lines(spans) as Array<{ start: number; end: numb… + ❯ tests/authored-parallel-llm.test.ts:86:10 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/4]⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight +AssertionError: expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + +- Expected ++ Received + +- 1 ++ 9 + + ❯ Object.assertProbes tests/authored-parallel-llm.test.ts:61:27 + 59| } + 60| return { fixture, client, assertJournals, assertProbes(count: number… + 61| expect(lines(probes)).toHaveLength(count); + | ^ + 62| }, assertCapacity() { + 63| const intervals = lines(spans) as Array<{ start: number; end: numb… + ❯ tests/authored-parallel-llm.test.ts:99:10 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/4]⎯ + + Test Files 1 failed (1) + Tests 4 failed | 2 skipped (6) + Start at 05:53:12 + Duration 23.80s (transform 596ms, setup 0ms, collect 1.13s, tests 22.49s, environment 0ms, prepare 42ms) + + +exit: 1 diff --git a/evidence/parallel-llm-fix/mutation-cache.txt b/evidence/parallel-llm-fix/mutation-cache.txt new file mode 100644 index 000000000..03abbfc85 --- /dev/null +++ b/evidence/parallel-llm-fix/mutation-cache.txt @@ -0,0 +1,2 @@ +cache: restored byte-for-byte +cache: reverted exit=1; restored exit=0 diff --git a/evidence/parallel-llm-fix/mutations.md b/evidence/parallel-llm-fix/mutations.md new file mode 100644 index 000000000..842945e9e --- /dev/null +++ b/evidence/parallel-llm-fix/mutations.md @@ -0,0 +1,180 @@ +# Mutation commands and literal captured output + +## cache + +Command: `python evidence/parallel-llm-fix/mutate.py cache` + +### reverted + +```text +cwd: packages/sdk +command: ['npx', 'vitest', 'run', 'tests/authored-parallel-llm.test.ts', '-t', 'deduplicates|completes nine'] + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/authored-parallel-llm.test.ts (6 tests | 4 failed | 2 skipped) 22493ms + × parallel llm capacity 1 > deduplicates concurrent preflight probes 3643ms + → expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + × parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 10266ms + → expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + × parallel llm capacity 4 > deduplicates concurrent preflight probes 1273ms + → expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + × parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight 7311ms + → expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 4 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > deduplicates concurrent preflight probes + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 4 > deduplicates concurrent preflight probes +AssertionError: expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + +- Expected ++ Received + +- 1 ++ 9 + + ❯ Object.assertProbes tests/authored-parallel-llm.test.ts:61:27 + 59| } + 60| return { fixture, client, assertJournals, assertProbes(count: number… + 61| expect(lines(probes)).toHaveLength(count); + | ^ + 62| }, assertCapacity() { + 63| const intervals = lines(spans) as Array<{ start: number; end: numb… + ❯ tests/authored-parallel-llm.test.ts:86:10 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/4]⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight +AssertionError: expected [ 'test-model', 'test-model', …(7) ] to have a length of 1 but got 9 + +- Expected ++ Received + +- 1 ++ 9 + + ❯ Object.assertProbes tests/authored-parallel-llm.test.ts:61:27 + 59| } + 60| return { fixture, client, assertJournals, assertProbes(count: number… + 61| expect(lines(probes)).toHaveLength(count); + | ^ + 62| }, assertCapacity() { + 63| const intervals = lines(spans) as Array<{ start: number; end: numb… + ❯ tests/authored-parallel-llm.test.ts:99:10 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/4]⎯ + + Test Files 1 failed (1) + Tests 4 failed | 2 skipped (6) + Start at 05:53:12 + Duration 23.80s (transform 596ms, setup 0ms, collect 1.13s, tests 22.49s, environment 0ms, prepare 42ms) + + +exit: 1 + +``` + +### restored + +```text +cwd: packages/sdk +command: ['npx', 'vitest', 'run', 'tests/authored-parallel-llm.test.ts', '-t', 'deduplicates|completes nine'] + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-parallel-llm.test.ts (6 tests | 2 skipped) 20071ms + ✓ parallel llm capacity 1 > deduplicates concurrent preflight probes 2799ms + ✓ parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 8690ms + ✓ parallel llm capacity 4 > deduplicates concurrent preflight probes 1311ms + ✓ parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight 7270ms + + Test Files 1 passed (1) + Tests 4 passed | 2 skipped (6) + Start at 05:53:37 + Duration 21.25s (transform 569ms, setup 0ms, collect 1.01s, tests 20.07s, environment 0ms, prepare 49ms) + + +exit: 0 + +``` + +Restoration check: + +```text +cache: restored byte-for-byte +cache: reverted exit=1; restored exit=0 + +``` + +## async + +Command: `python evidence/parallel-llm-fix/mutate.py async` + +### reverted + +```text +cwd: packages/sdk +command: ['npx', 'vitest', 'run', 'tests/authored-parallel-llm.test.ts', '-t', 'parallel llm capacity 1.*keeps the durable'] + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/authored-parallel-llm.test.ts (6 tests | 1 failed | 5 skipped) 91068ms + × parallel llm capacity 1 > keeps the durable root lease alive across two cold models 91067ms + → lease_conflict: attempt has no active worker lease + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > keeps the durable root lease alive across two cold models +JournalProtocolError: lease_conflict: attempt has no active worker lease + ❯ JournalClient.onLine src/journal-client.ts:165:27 + 163| if (pending.timer !== undefined) clearTimeout(pending.timer); + 164| if (res.ok) pending.resolve(res.result); + 165| else pending.reject(new JournalProtocolError(res.error.code, res… + | ^ + 166| } else { + 167| const ev = msg as ServerEvent; + ❯ JournalClient.onData src/journal-client.ts:144:33 + ❯ Socket. src/journal-client.ts:109:43 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 5 skipped (6) + Start at 05:53:59 + Duration 92.22s (transform 560ms, setup 0ms, collect 978ms, tests 91.07s, environment 0ms, prepare 46ms) + + +exit: 1 + +``` + +### restored + +```text +cwd: packages/sdk +command: ['npx', 'vitest', 'run', 'tests/authored-parallel-llm.test.ts', '-t', 'parallel llm capacity 1.*keeps the durable'] + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-parallel-llm.test.ts (6 tests | 5 skipped) 47947ms + ✓ parallel llm capacity 1 > keeps the durable root lease alive across two cold models 47946ms + + Test Files 1 passed (1) + Tests 1 passed | 5 skipped (6) + Start at 05:55:31 + Duration 49.09s (transform 558ms, setup 0ms, collect 978ms, tests 47.95s, environment 0ms, prepare 43ms) + + +exit: 0 + +``` + +Restoration check: + +```text +async: restored byte-for-byte +async: reverted exit=1; restored exit=0 + +``` diff --git a/evidence/parallel-llm-fix/prep.txt b/evidence/parallel-llm-fix/prep.txt new file mode 100644 index 000000000..c719b4217 --- /dev/null +++ b/evidence/parallel-llm-fix/prep.txt @@ -0,0 +1,5 @@ + +> @relayflows/sdk@2.0.29 test:prep +> ( cd ../../kernel && sh ../ops/cargo.sh build ) && ( [ ! -d ../../testdata/preflight ] || find ../../testdata/preflight -name '*-cli' -type f -exec chmod +x {} + ) + + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.07s diff --git a/evidence/parallel-llm-fix/prompt-lab-journals.txt b/evidence/parallel-llm-fix/prompt-lab-journals.txt new file mode 100644 index 000000000..b8d5c96a5 --- /dev/null +++ b/evidence/parallel-llm-fix/prompt-lab-journals.txt @@ -0,0 +1 @@ +{"directory": "/tmp/flows-prompt-lab-data", "journals": 29, "lease_expired": []} diff --git a/evidence/parallel-llm-fix/prompt-lab-parallel.patch b/evidence/parallel-llm-fix/prompt-lab-parallel.patch new file mode 100644 index 000000000..fda705b68 --- /dev/null +++ b/evidence/parallel-llm-fix/prompt-lab-parallel.patch @@ -0,0 +1,71 @@ +diff --git a/examples/prompt-lab/jobs/new-agency.ts b/examples/prompt-lab/jobs/new-agency.ts +index 75801b8..ee2bc65 100644 +--- a/examples/prompt-lab/jobs/new-agency.ts ++++ b/examples/prompt-lab/jobs/new-agency.ts +@@ -60,12 +60,11 @@ export async function newAgency(job: Job, input: NewAgencyInput): Promise + } + + // Run on fake visits — the background job; the grid fills when it finishes. +- const ran: { q: PiledQuestion; outputs: Record }[] = []; +- for (const c of plan.coverage) { // sequential: see runEngine ++ const ran = await Promise.all(plan.coverage.map(async c => { + const q = byId.get(c.questionId)!; + const shelf = c.patientIds.map((id) => patient.get(id)!); +- ran.push({ q, outputs: await runEngine(f, prompts.get(q.questionId)!.text, ask(q), shelf) }); +- } ++ return { q, outputs: await runEngine(f, prompts.get(q.questionId)!.text, ask(q), shelf) }; ++ })); + const rows = sortRows(ran.flatMap(({ q, outputs }) => Object.entries(outputs).map(([patientId, ai]): Row => ({ + questionId: q.questionId, patientId, pile: q.pile, + highlight: highlights(ai, q.pile, prompts.get(q.questionId)!.firstPass), +@@ -92,8 +91,7 @@ export async function newAgency(job: Job, input: NewAgencyInput): Promise + // are commit candidates (Prompt QA'd); shared/mismatch get a proposed rewrite only. + const changes = changed(edited); + const changedQs = [...new Set(changes.map((r) => r.questionId))].sort(); +- const rewrites: { q: PiledQuestion; rows: Row[]; prompt: string; passed: boolean }[] = []; +- for (const qid of changedQs) { // sequential: see runEngine ++ const rewrites = await Promise.all(changedQs.map(async qid => { + const q = byId.get(qid)!; + const rowsFor = changes.filter((r) => r.questionId === qid) + .map((r) => ({ patient: patient.get(r.patientId)! as Patient, ai: r.ai, target: r.target, notes: r.notes })); +@@ -102,14 +100,13 @@ export async function newAgency(job: Job, input: NewAgencyInput): Promise + const rows = changes.filter((r) => r.questionId === qid); + if (q.pile !== "agency-specific") { + const { prompt } = await llm<{ prompt: string }>(f, iterate(ask(q), existing, brief, rowsFor, [])); +- rewrites.push({ q, rows, prompt, passed: true }); +- continue; ++ return { q, rows, prompt, passed: true }; + } + const r = await untilQaPasses<{ prompt: string }>(f, + (findings) => iterate(ask(q), existing, brief, rowsFor, findings), + (w) => promptQa(w.prompt, ask(q), snap.guidelines, brief)); +- rewrites.push({ q, rows, prompt: r.value.prompt, passed: r.passed }); +- } ++ return { q, rows, prompt: r.value.prompt, passed: r.passed }; ++ })); + + // A first-pass prompt is a candidate only once it ran on a shelf patient and + // you reviewed the rows; a gap-only draft waits for a patient that covers it. +diff --git a/examples/prompt-lab/jobs/shared.ts b/examples/prompt-lab/jobs/shared.ts +index 732b0f5..165d23f 100644 +--- a/examples/prompt-lab/jobs/shared.ts ++++ b/examples/prompt-lab/jobs/shared.ts +@@ -66,16 +66,10 @@ export async function untilQaPasses( + /** + * Run one prompt on fake visits: the same engine nurses use. One llm step per patient. + * +- * Sequential on purpose. Every model call in this flow is independent and was +- * written as Promise.all, but relayflows 2.0.29 loses the run when concurrent +- * f.llm calls queue past their 30s lease (a stale completion becomes a fatal +- * protocol_error): see evidence/runtime-findings/runtime-parallel-llm-repro.txt. Restore +- * Promise.all here and in jobs/new-agency.ts when flows#561 and flows#560 ship. + */ + export async function runEngine(f: Ctx, promptText: string, ask: Ask, patients: readonly Patient[]): Promise> { +- const outputs: Record = {}; +- for (const p of patients) outputs[p.id] = await llm(f, engine(promptText, ask, p)); +- return outputs; ++ return Object.fromEntries(await Promise.all(patients.map(async p => ++ [p.id, await llm(f, engine(promptText, ask, p))] as const))); + } + + export interface Plan { coverage: { questionId: string; patientIds: string[] }[]; gaps: { questionId: string; brief: string }[] } diff --git a/evidence/parallel-llm-fix/prompt-lab-run/01-job1-run.txt b/evidence/parallel-llm-fix/prompt-lab-run/01-job1-run.txt new file mode 100644 index 000000000..fe29f9aff --- /dev/null +++ b/evidence/parallel-llm-fix/prompt-lab-run/01-job1-run.txt @@ -0,0 +1,86 @@ +$ node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js run --no-observer-link --data-dir /tmp/flows-prompt-lab-data --local-agent prompt-lab.flow.ts --input {"job":"new-agency","reviewer":"prompt-lab-reviewer","lab":"/tmp/flows-prompt-lab-proof/lab","agency":"sunrise","visitType":"soc"} +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.09s completionReason: success +○ llm-2 (llm) 0.00s +WAITING [worker_lease] Run "01M36CGZ7DHQPB1W0JJAA7J4QK" step "llm-2" (llm) is running under a worker lease until 1790142116388. +↻ llm-2 (llm) 2.11s +WAITING [worker_lease] Run "01M36CGZ7DHQPB1W0JJAA7J4QK" step "llm-2" (llm) is running under a worker lease until 1790142126392. +↻ llm-2 (llm) 12.12s +WAITING [worker_lease] Run "01M36CGZ7DHQPB1W0JJAA7J4QK" step "llm-2" (llm) is running under a worker lease until 1790142136397. +↻ llm-2 (llm) 22.12s +WAITING [worker_lease] Run "01M36CGZ7DHQPB1W0JJAA7J4QK" step "llm-2" (llm) is running under a worker lease until 1790142146407. +↻ llm-2 (llm) 32.15s +✓ llm-2 (llm) 36.16s completionReason: success +○ llm-3 (llm) 0.00s +WAITING [worker_lease] Run "01M36CJ0G6YRB25A7T31HS3VPE" step "llm-3" (llm) is running under a worker lease until 1790142150461. +↻ llm-3 (llm) 0.02s +✓ llm-3 (llm) 6.21s completionReason: success +○ run-4 (deterministic) 0.00s +✓ run-4 (deterministic) 0.10s completionReason: success +○ llm-5 (llm) 0.00s +WAITING [worker_lease] Run "01M36CJ6N8K5CE2DS40BS9Z7D6" step "llm-5" (llm) is running under a worker lease until 1790142156766. +↻ llm-5 (llm) 0.02s +WAITING [worker_lease] Run "01M36CJ6N8K5CE2DS40BS9Z7D6" step "llm-5" (llm) is running under a worker lease until 1790142166767. +↻ llm-5 (llm) 10.02s +WAITING [worker_lease] Run "01M36CJ6N8K5CE2DS40BS9Z7D6" step "llm-5" (llm) is running under a worker lease until 1790142176769. +↻ llm-5 (llm) 20.02s +✓ llm-5 (llm) 23.93s completionReason: success +○ llm-6 (llm) 0.00s +WAITING [worker_lease] Run "01M36CJY16AV5R77J4EZWVW4HF" step "llm-6" (llm) is running under a worker lease until 1790142180700. +↻ llm-6 (llm) 0.02s +✓ llm-6 (llm) 4.56s completionReason: success +○ run-7 (deterministic) 0.00s +✓ run-7 (deterministic) 0.09s completionReason: success +○ llm-8 (llm) 0.00s +WAITING [worker_lease] Run "01M36CK2J805CEYS2GY3ZQCMXR" step "llm-8" (llm) is running under a worker lease until 1790142185343. +↻ llm-8 (llm) 0.01s +WAITING [worker_lease] Run "01M36CK2J805CEYS2GY3ZQCMXR" step "llm-8" (llm) is running under a worker lease until 1790142195346. +↻ llm-8 (llm) 10.01s +✓ llm-8 (llm) 18.39s completionReason: success +○ run-9 (deterministic) 0.00s +✓ run-9 (deterministic) 0.09s completionReason: success +○ llm-10 (llm) 0.00s +○ llm-11 (llm) 0.00s +○ llm-12 (llm) 0.00s +○ llm-13 (llm) 0.00s +○ llm-14 (llm) 0.00s +○ llm-15 (llm) 0.00s +○ llm-16 (llm) 0.00s +○ llm-17 (llm) 0.00s +○ llm-18 (llm) 0.00s +WAITING [worker_lease] Run "01M36CKMKNARNYRM1MR44ZZZTS" step "llm-10" (llm) is running under a worker lease until 1790142203819. +↻ llm-10 (llm) 0.05s +WAITING [worker_lease] Run "01M36CKMKWDXZKEPC20SAYR7A4" step "llm-11" (llm) is running under a worker lease until 1790142203828. +↻ llm-11 (llm) 0.05s +WAITING [worker_lease] Run "01M36CKMM3C5D595A9GN2R4728" step "llm-12" (llm) is running under a worker lease until 1790142203835. +↻ llm-12 (llm) 0.05s +WAITING [worker_lease] Run "01M36CKMMB96EFV04W6YMC1SNP" step "llm-13" (llm) is running under a worker lease until 1790142203844. +↻ llm-13 (llm) 0.05s +✓ llm-13 (llm) 2.42s completionReason: success +WAITING [worker_lease] Run "01M36CKPZ4YF1RQZAEFD0NFZY4" step "llm-14" (llm) is running under a worker lease until 1790142206235. +↻ llm-14 (llm) 2.43s +✓ llm-12 (llm) 3.25s completionReason: success +WAITING [worker_lease] Run "01M36CKQS75YSENHNFMA9TFP6A" step "llm-15" (llm) is running under a worker lease until 1790142207070. +↻ llm-15 (llm) 3.27s +✓ llm-11 (llm) 3.40s completionReason: success +WAITING [worker_lease] Run "01M36CKQXZKW8JBET2SW0ZCC6Y" step "llm-16" (llm) is running under a worker lease until 1790142207221. +↻ llm-16 (llm) 3.42s +✓ llm-14 (llm) 5.53s completionReason: success +WAITING [worker_lease] Run "01M36CKT0AJB4PY0Z4VGEVN0S4" step "llm-17" (llm) is running under a worker lease until 1790142209351. +↻ llm-17 (llm) 5.60s +✓ llm-15 (llm) 6.50s completionReason: success +WAITING [worker_lease] Run "01M36CKTYT88664BJ884HWWTS1" step "llm-18" (llm) is running under a worker lease until 1790142210323. +↻ llm-18 (llm) 6.52s +✓ llm-16 (llm) 7.14s completionReason: success +✓ llm-18 (llm) 9.18s completionReason: success +✓ llm-10 (llm) 9.54s completionReason: success +✓ llm-17 (llm) 9.85s completionReason: success +○ run-19 (deterministic) 0.00s +✓ run-19 (deterministic) 0.09s completionReason: success +○ human-20 (deterministic) 0.00s +⏸ human-20 (human) 0.01s +PARKED [run_parked] Run "01M36CGX2HC59PAVGKV0SCFW6S" is waiting for prompt-lab-reviewer to answer human-20: "Config workbench · sunrise soc: 9 rows on 3 questions (6 highlighted, listed first); 1 gap brief(s) queued for the test patient manager.\nEdit target answer / confidence / explanation / notes in /tmp/flows-prompt-lab-proof/lab/work/sunrise-soc/e4a1ccb3/grid.json. Your first pass persists as the target for each question × patient.\nyes = persist targets and run iteration on every changed row; no = stop without persisting." +Answer with: flows answer --data-dir /tmp/flows-prompt-lab-data 01M36CGX2HC59PAVGKV0SCFW6S human-20 yes|no +Then continue with: flows resume --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +RUN 01M36CGX2HC59PAVGKV0SCFW6S parked +exit=3 diff --git a/evidence/parallel-llm-fix/prompt-lab-run/02-reviewer-edit.diff b/evidence/parallel-llm-fix/prompt-lab-run/02-reviewer-edit.diff new file mode 100644 index 000000000..9ba0e3939 --- /dev/null +++ b/evidence/parallel-llm-fix/prompt-lab-run/02-reviewer-edit.diff @@ -0,0 +1,13 @@ +# reviewer edit: /tmp/flows-prompt-lab-proof/lab/work/sunrise-soc/e4a1ccb3/grid.json +@@ -144,9 +144,9 @@ + "target": { + "answer": "Ongoing", + "confidence": "High", +- "explanation": "Today's SOC visit documents the left heel as an open area 2.0 x 1.5 cm with moderate serous drainage and a pink wound bed with yellow slough, so the wound is not healed despite the discharge summary stating it was closed. The current direct assessment supersedes the stale referral statement." ++ "explanation": "Today's visit notes describe an open 2.0 x 1.5 cm left heel wound with serous drainage and slough; the discharge summary's closed status is outdated." + }, +- "notes": "" ++ "notes": "The prompt makes the referral win over what the nurse saw today. Today's notes must win (shared guideline 1)." + }, + { + "questionId": "wound-status", diff --git a/evidence/parallel-llm-fix/prompt-lab-run/03-answer.txt b/evidence/parallel-llm-fix/prompt-lab-run/03-answer.txt new file mode 100644 index 000000000..cc4b074d6 --- /dev/null +++ b/evidence/parallel-llm-fix/prompt-lab-run/03-answer.txt @@ -0,0 +1,4 @@ +$ node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js answer --data-dir /tmp/flows-prompt-lab-data 01M36CGX2HC59PAVGKV0SCFW6S human-20 yes --by prompt-lab-reviewer +ANSWERED 01M36CGX2HC59PAVGKV0SCFW6S human-20 yes +Continue with: flows resume --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +exit=0 diff --git a/evidence/parallel-llm-fix/prompt-lab-run/04-resume.txt b/evidence/parallel-llm-fix/prompt-lab-run/04-resume.txt new file mode 100644 index 000000000..df359df5a --- /dev/null +++ b/evidence/parallel-llm-fix/prompt-lab-run/04-resume.txt @@ -0,0 +1,62 @@ +$ node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js resume --no-observer-link --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.01s completionReason: success +○ llm-2 (llm) 0.00s +✓ llm-2 (llm) 1.84s completionReason: success +○ llm-3 (llm) 0.00s +✓ llm-3 (llm) 0.01s completionReason: success +○ run-4 (deterministic) 0.00s +✓ run-4 (deterministic) 0.01s completionReason: success +○ llm-5 (llm) 0.00s +✓ llm-5 (llm) 0.01s completionReason: success +○ llm-6 (llm) 0.00s +✓ llm-6 (llm) 0.01s completionReason: success +○ run-7 (deterministic) 0.00s +✓ run-7 (deterministic) 0.01s completionReason: success +○ llm-8 (llm) 0.00s +✓ llm-8 (llm) 0.01s completionReason: success +○ run-9 (deterministic) 0.00s +✓ run-9 (deterministic) 0.01s completionReason: success +○ llm-10 (llm) 0.00s +○ llm-11 (llm) 0.00s +○ llm-12 (llm) 0.00s +○ llm-13 (llm) 0.00s +○ llm-14 (llm) 0.00s +○ llm-15 (llm) 0.00s +○ llm-16 (llm) 0.00s +○ llm-17 (llm) 0.00s +○ llm-18 (llm) 0.00s +✓ llm-10 (llm) 0.04s completionReason: success +✓ llm-11 (llm) 0.04s completionReason: success +✓ llm-12 (llm) 0.05s completionReason: success +✓ llm-13 (llm) 0.05s completionReason: success +✓ llm-14 (llm) 0.09s completionReason: success +✓ llm-15 (llm) 0.09s completionReason: success +✓ llm-16 (llm) 0.10s completionReason: success +✓ llm-17 (llm) 0.10s completionReason: success +✓ llm-18 (llm) 0.11s completionReason: success +○ run-19 (deterministic) 0.00s +✓ run-19 (deterministic) 0.02s completionReason: success +○ human-20 (deterministic) 0.00s +✓ human-20 (deterministic) 0.03s completionReason: success +○ run-21 (deterministic) 0.00s +✓ run-21 (deterministic) 0.09s completionReason: success +○ run-22 (deterministic) 0.00s +✓ run-22 (deterministic) 0.09s completionReason: success +○ llm-23 (llm) 0.00s +WAITING [worker_lease] Run "01M36CM1MRES6V92VT42Q0RZ7H" step "llm-23" (llm) is running under a worker lease until 1790142217166. +↻ llm-23 (llm) 0.02s +WAITING [worker_lease] Run "01M36CM1MRES6V92VT42Q0RZ7H" step "llm-23" (llm) is running under a worker lease until 1790142227169. +↻ llm-23 (llm) 10.05s +✓ llm-23 (llm) 10.37s completionReason: success +○ run-24 (deterministic) 0.00s +✓ run-24 (deterministic) 0.10s completionReason: success +○ run-25 (deterministic) 0.00s +✓ run-25 (deterministic) 0.10s completionReason: success +○ human-26 (deterministic) 0.00s +⏸ human-26 (human) 0.01s +PARKED [run_parked] Run "01M36CGX2HC59PAVGKV0SCFW6S" is waiting for prompt-lab-reviewer to answer human-26: "Commit output · sunrise: agency-specific prompts ready to go live in Apricot: living-situation.\nSent to the question manager (frozen here): wound-status → config-sunrise-wound-status-7309e97c.\nHeld as drafts until a shelf patient covers them: ostomy-supplies.\nTo commit only some, edit /tmp/flows-prompt-lab-proof/lab/work/sunrise-soc/e4a1ccb3/commit.json: {\"mode\":\"only\"|\"except\",\"questions\":[...]}.\nyes = commit (Done = live, no promote step); no = leave them as drafts." +Answer with: flows answer --data-dir /tmp/flows-prompt-lab-data 01M36CGX2HC59PAVGKV0SCFW6S human-26 yes|no +Then continue with: flows resume --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +RUN 01M36CGX2HC59PAVGKV0SCFW6S parked +exit=3 diff --git a/evidence/parallel-llm-fix/prompt-lab-run/05-answer.txt b/evidence/parallel-llm-fix/prompt-lab-run/05-answer.txt new file mode 100644 index 000000000..3bf84768b --- /dev/null +++ b/evidence/parallel-llm-fix/prompt-lab-run/05-answer.txt @@ -0,0 +1,4 @@ +$ node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js answer --data-dir /tmp/flows-prompt-lab-data 01M36CGX2HC59PAVGKV0SCFW6S human-26 yes --by prompt-lab-reviewer +ANSWERED 01M36CGX2HC59PAVGKV0SCFW6S human-26 yes +Continue with: flows resume --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +exit=0 diff --git a/evidence/parallel-llm-fix/prompt-lab-run/06-resume.txt b/evidence/parallel-llm-fix/prompt-lab-run/06-resume.txt new file mode 100644 index 000000000..9898998f3 --- /dev/null +++ b/evidence/parallel-llm-fix/prompt-lab-run/06-resume.txt @@ -0,0 +1,60 @@ +$ node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js resume --no-observer-link --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.08s completionReason: success +○ llm-2 (llm) 0.00s +✓ llm-2 (llm) 1.85s completionReason: success +○ llm-3 (llm) 0.00s +✓ llm-3 (llm) 0.02s completionReason: success +○ run-4 (deterministic) 0.00s +✓ run-4 (deterministic) 0.02s completionReason: success +○ llm-5 (llm) 0.00s +✓ llm-5 (llm) 0.02s completionReason: success +○ llm-6 (llm) 0.00s +✓ llm-6 (llm) 0.02s completionReason: success +○ run-7 (deterministic) 0.00s +✓ run-7 (deterministic) 0.02s completionReason: success +○ llm-8 (llm) 0.00s +✓ llm-8 (llm) 0.02s completionReason: success +○ run-9 (deterministic) 0.00s +✓ run-9 (deterministic) 0.02s completionReason: success +○ llm-10 (llm) 0.00s +○ llm-11 (llm) 0.00s +○ llm-12 (llm) 0.00s +○ llm-13 (llm) 0.00s +○ llm-14 (llm) 0.00s +○ llm-15 (llm) 0.00s +○ llm-16 (llm) 0.00s +○ llm-17 (llm) 0.00s +○ llm-18 (llm) 0.00s +✓ llm-10 (llm) 0.05s completionReason: success +✓ llm-11 (llm) 0.06s completionReason: success +✓ llm-12 (llm) 0.06s completionReason: success +✓ llm-13 (llm) 0.07s completionReason: success +✓ llm-14 (llm) 0.12s completionReason: success +✓ llm-15 (llm) 0.12s completionReason: success +✓ llm-16 (llm) 0.13s completionReason: success +✓ llm-17 (llm) 0.14s completionReason: success +✓ llm-18 (llm) 0.15s completionReason: success +○ run-19 (deterministic) 0.00s +✓ run-19 (deterministic) 0.02s completionReason: success +○ human-20 (deterministic) 0.00s +✓ human-20 (deterministic) 0.02s completionReason: success +○ run-21 (deterministic) 0.00s +✓ run-21 (deterministic) 0.02s completionReason: success +○ run-22 (deterministic) 0.00s +✓ run-22 (deterministic) 0.02s completionReason: success +○ llm-23 (llm) 0.00s +✓ llm-23 (llm) 0.02s completionReason: success +○ run-24 (deterministic) 0.00s +✓ run-24 (deterministic) 0.02s completionReason: success +○ run-25 (deterministic) 0.00s +✓ run-25 (deterministic) 0.02s completionReason: success +○ human-26 (deterministic) 0.00s +✓ human-26 (deterministic) 0.04s completionReason: success +○ run-27 (deterministic) 0.00s +✓ run-27 (deterministic) 0.10s completionReason: success +○ run-28 (deterministic) 0.00s +✓ run-28 (deterministic) 0.10s completionReason: success +FAILED [protocol_error] relayflowd could not complete the resume request: unawaited_step: flow "prompt-lab" returned with unawaited steps: llm-10 (f.llm), llm-11 (f.llm), llm-13 (f.llm), llm-14 (f.llm) +RUN 01M36CGX2HC59PAVGKV0SCFW6S unknown +exit=1 diff --git a/evidence/parallel-llm-fix/prompt-lab.txt b/evidence/parallel-llm-fix/prompt-lab.txt new file mode 100644 index 000000000..553cede0b --- /dev/null +++ b/evidence/parallel-llm-fix/prompt-lab.txt @@ -0,0 +1,23 @@ +{"seeded":"/tmp/flows-prompt-lab-proof/lab"} +== Job 1: new agency sunrise / soc +Answer with: flows answer --data-dir /tmp/flows-prompt-lab-data 01M36CGX2HC59PAVGKV0SCFW6S human-20 yes|no +Then continue with: flows resume --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +RUN 01M36CGX2HC59PAVGKV0SCFW6S parked +exit=3 +$ node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js answer --data-dir /tmp/flows-prompt-lab-data 01M36CGX2HC59PAVGKV0SCFW6S human-20 yes --by prompt-lab-reviewer +ANSWERED 01M36CGX2HC59PAVGKV0SCFW6S human-20 yes +Continue with: flows resume --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +exit=0 +Answer with: flows answer --data-dir /tmp/flows-prompt-lab-data 01M36CGX2HC59PAVGKV0SCFW6S human-26 yes|no +Then continue with: flows resume --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +RUN 01M36CGX2HC59PAVGKV0SCFW6S parked +exit=3 +$ node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js answer --data-dir /tmp/flows-prompt-lab-data 01M36CGX2HC59PAVGKV0SCFW6S human-26 yes --by prompt-lab-reviewer +ANSWERED 01M36CGX2HC59PAVGKV0SCFW6S human-26 yes +Continue with: flows resume --data-dir /tmp/flows-prompt-lab-data --local-agent 01M36CGX2HC59PAVGKV0SCFW6S +exit=0 +✓ run-28 (deterministic) 0.10s completionReason: success +FAILED [protocol_error] relayflowd could not complete the resume request: unawaited_step: flow "prompt-lab" returned with unawaited steps: llm-10 (f.llm), llm-11 (f.llm), llm-13 (f.llm), llm-14 (f.llm) +RUN 01M36CGX2HC59PAVGKV0SCFW6S unknown +exit=1 +prove: /tmp/flows-prompt-lab-proof/06-resume.txt exited 1, expected 0 diff --git a/evidence/parallel-llm-fix/prove-fixed.sh b/evidence/parallel-llm-fix/prove-fixed.sh new file mode 100644 index 000000000..a37df7233 --- /dev/null +++ b/evidence/parallel-llm-fix/prove-fixed.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# End-to-end proof of the Prompt Lab relayflow, locally, with real model calls. +# +# Seeds a fresh lab from fixtures/, then drives all three jobs through the real +# kernel with `flows run --local-agent`. At every parked `f.human` gate this +# script plays the reviewer named in the input: it applies the documented edit +# (captured as a diff), answers with `flows answer`, and `flows resume`s. +# Every command is captured with its literal output and exit code, and the +# script stops on the first exit it did not expect. +# +# ./prove.sh [out-dir] default: evidence/run +set -uo pipefail +cd "$(dirname "$0")" +OUT=${1:-evidence/run} +LAB=$OUT/lab +DD=${FLOWS_DATA_DIR:-$(mktemp -d)} +REVIEWER=prompt-lab-reviewer +N=0 +STATUS= +FILE= + +die() { echo "prove: $*" >&2; exit 1; } +[ -e "$LAB" ] && die "refusing: $LAB exists" +mkdir -p "$OUT" +node --no-warnings --experimental-strip-types store.ts "$LAB" seed fixtures || die "seed failed" + +# capture -- : run it, write "$ cmd", its +# output and exit code to $OUT/NN-name.txt, and stop unless the exit is one of +# the expected ones (0 completed, 3 parked on a gate). +capture() { + local want=() + while [ "$1" != "--" ]; do want+=("$1"); shift; done + shift + local name=$1 + shift + N=$((N + 1)) + FILE=$(printf '%s/%02d-%s.txt' "$OUT" "$N" "$name") + { echo "\$ $*"; "$@" 2>&1; STATUS=$?; echo "exit=$STATUS"; } > "$FILE" + grep -v 'WAITING\|↻\|○' "$FILE" | tail -4 | cut -c1-240 + local w + for w in "${want[@]}"; do [ "$STATUS" = "$w" ] && return 0; done + die "$FILE exited $STATUS, expected ${want[*]}" +} +run_id() { grep -o 'RUN [0-9A-Z]\{26\}' "$FILE" | tail -1 | cut -d' ' -f2; } +wait_id() { grep -o ' human-[0-9]* yes|no' "$FILE" | tail -1 | awk '{print $1}'; } +parked_on() { grep -q "PARKED.*$1" "$FILE" || die "$FILE did not park on \"$1\""; } +gate_path() { grep -o "$LAB/work/[^ ]*\.json" "$FILE" | head -1; } +# edit : the reviewer's change, captured as a diff +edit() { + local file=$1 expr=$2 before + [ -f "$file" ] || die "no file to edit: $file" + before=$(mktemp) + cp "$file" "$before" + node -e "const fs=require('fs');const v=JSON.parse(fs.readFileSync('$file','utf8'));$expr;fs.writeFileSync('$file',JSON.stringify(v,null,2)+'\n')" \ + || die "reviewer edit failed on $file" + N=$((N + 1)) + { echo "# reviewer edit: $file"; diff -u "$before" "$file" | tail -n +3; } > "$(printf '%s/%02d-reviewer-edit.diff' "$OUT" "$N")" +} +# answer_and_resume : 3 when another gate follows, 0 when the run should complete. +answer_and_resume() { + local run wait + run=$(run_id) + wait=$(wait_id) + [ -n "$run" ] && [ -n "$wait" ] || die "no parked gate in $FILE" + capture 0 -- answer node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js answer --data-dir "$DD" "$run" "$wait" yes --by "$REVIEWER" + capture "$1" -- resume node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js resume --no-observer-link --data-dir "$DD" --local-agent "$run" +} +flow() { capture 3 -- "$1" node /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js run --no-observer-link --data-dir "$DD" --local-agent prompt-lab.flow.ts --input "$2"; } + +echo "== Job 1: new agency sunrise / soc" +flow job1-run "{\"job\":\"new-agency\",\"reviewer\":\"$REVIEWER\",\"lab\":\"$LAB\",\"agency\":\"sunrise\",\"visitType\":\"soc\"}" +parked_on "Config workbench" +# The reviewer's first pass: Pat's wound is open today, whatever the discharge summary said. +edit "$(gate_path)" 'const r=v.find(r=>r.questionId==="wound-status"&&r.patientId==="pat");r.target={answer:"Ongoing",confidence:"High",explanation:"Today'"'"'s visit notes describe an open 2.0 x 1.5 cm left heel wound with serous drainage and slough; the discharge summary'"'"'s closed status is outdated."};r.notes="The prompt makes the referral win over what the nurse saw today. Today'"'"'s notes must win (shared guideline 1)."' +answer_and_resume 3 +parked_on "Commit output" +# Only covered prompts are offered; ostomy-supplies (no shelf patient yet) is held as a draft. +grep -q "Held as drafts until a shelf patient covers them: ostomy-supplies" "$FILE" || die "ostomy-supplies was offered for commit" +answer_and_resume 0 # commit all offered + +echo "== Test patient creator: the ostomy gap brief Job 1 queued" +flow patient-run "{\"job\":\"patient\",\"reviewer\":\"$REVIEWER\",\"lab\":\"$LAB\",\"briefId\":\"gap-ostomy-supplies-soc\"}" +parked_on "Test patient creator" +answer_and_resume 0 # kick generate; Patient QA locks it, nobody approves the chart + +echo "== Job 2: fix wound-status from the config-send issue" +ISSUE=$(node -e "const q=require('./$LAB/queue/issues.json');console.log(q.find(i=>i.questionIds[0]==='wound-status').id)") \ + || die "no wound-status issue was sent to the question manager" +flow job2-run "{\"job\":\"fix\",\"reviewer\":\"$REVIEWER\",\"lab\":\"$LAB\",\"issueId\":\"$ISSUE\"}" +parked_on "Question workbench" +answer_and_resume 3 # gold as prefilled: Pat's config-level target travelled with the issue +parked_on "Re-run and score" +answer_and_resume 0 # mark done = live, for every agency using it + +echo "== Final lab state" +capture 0 -- lab-state node -e " +const b=require('./$LAB/bank.json'); +for (const [q,v] of Object.entries(b.questions)) console.log(q.padEnd(17),'live',String(v.livePromptId).padEnd(30),'draft',v.draftPromptId??'-'); +console.log('shelf', require('fs').readdirSync('$LAB/shelf').join(' ')); +console.log('issues', JSON.stringify(require('./$LAB/queue/issues.json').map(i=>[i.id,i.status]))); +console.log('patient briefs', JSON.stringify(require('./$LAB/queue/patient-briefs.json').map(i=>[i.id,i.status]))); +console.log('gold', Object.keys(require('./$LAB/gold.json')).join(' '));" +echo "data dir: $DD" diff --git a/evidence/parallel-llm-fix/regression-pass.txt b/evidence/parallel-llm-fix/regression-pass.txt new file mode 100644 index 000000000..7b7a4429a --- /dev/null +++ b/evidence/parallel-llm-fix/regression-pass.txt @@ -0,0 +1,16 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-parallel-llm.test.ts (6 tests) 114270ms + ✓ parallel llm capacity 1 > deduplicates concurrent preflight probes 3122ms + ✓ parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 8803ms + ✓ parallel llm capacity 1 > keeps the durable root lease alive across two cold models 47739ms + ✓ parallel llm capacity 4 > deduplicates concurrent preflight probes 1178ms + ✓ parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight 7141ms + ✓ parallel llm capacity 4 > keeps the durable root lease alive across two cold models 46286ms + + Test Files 1 passed (1) + Tests 6 passed (6) + Start at 05:40:14 + Duration 115.45s (transform 576ms, setup 0ms, collect 1.00s, tests 114.27s, environment 0ms, prepare 50ms) + diff --git a/evidence/parallel-llm-fix/regression.txt b/evidence/parallel-llm-fix/regression.txt new file mode 100644 index 000000000..106d88f2c --- /dev/null +++ b/evidence/parallel-llm-fix/regression.txt @@ -0,0 +1,87 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/authored-parallel-llm.test.ts (6 tests | 6 failed) 113509ms + × parallel llm capacity 1 > deduplicates concurrent preflight probes 2515ms + → expected [ …(10) ] to have a length of 9 but got 10 + × parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 8547ms + → expected [ …(10) ] to have a length of 9 but got 10 + × parallel llm capacity 1 > keeps the durable root lease alive across two cold models 47711ms + → expected [ …(11) ] to have a length of 10 but got 11 + × parallel llm capacity 4 > deduplicates concurrent preflight probes 1189ms + → expected [ …(10) ] to have a length of 9 but got 10 + × parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight 7166ms + → expected [ …(10) ] to have a length of 9 but got 10 + × parallel llm capacity 4 > keeps the durable root lease alive across two cold models 46380ms + → expected [ …(11) ] to have a length of 10 but got 11 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 6 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > deduplicates concurrent preflight probes + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 4 > deduplicates concurrent preflight probes +AssertionError: expected [ …(10) ] to have a length of 9 but got 10 + +- Expected ++ Received + +- 9 ++ 10 + + ❯ Object.assertJournals tests/authored-parallel-llm.test.ts:51:17 + 49| async function assertJournals(expected: number) { + 50| const ids = readdirSync(join(fixture.data, 'runs')).filter(name =>… + 51| expect(ids).toHaveLength(expected); + | ^ + 52| const expired: unknown[] = []; + 53| for (const name of ids) { + ❯ tests/authored-parallel-llm.test.ts:88:16 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/6]⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight +AssertionError: expected [ …(10) ] to have a length of 9 but got 10 + +- Expected ++ Received + +- 9 ++ 10 + + ❯ Object.assertJournals tests/authored-parallel-llm.test.ts:51:17 + 49| async function assertJournals(expected: number) { + 50| const ids = readdirSync(join(fixture.data, 'runs')).filter(name =>… + 51| expect(ids).toHaveLength(expected); + | ^ + 52| const expired: unknown[] = []; + 53| for (const name of ids) { + ❯ tests/authored-parallel-llm.test.ts:97:16 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/6]⎯ + + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 1 > keeps the durable root lease alive across two cold models + FAIL tests/authored-parallel-llm.test.ts > parallel llm capacity 4 > keeps the durable root lease alive across two cold models +AssertionError: expected [ …(11) ] to have a length of 10 but got 11 + +- Expected ++ Received + +- 10 ++ 11 + + ❯ Object.assertJournals tests/authored-parallel-llm.test.ts:51:17 + 49| async function assertJournals(expected: number) { + 50| const ids = readdirSync(join(fixture.data, 'runs')).filter(name =>… + 51| expect(ids).toHaveLength(expected); + | ^ + 52| const expired: unknown[] = []; + 53| for (const name of ids) { + ❯ tests/authored-parallel-llm.test.ts:119:16 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/6]⎯ + + Test Files 1 failed (1) + Tests 6 failed (6) + Start at 05:37:50 + Duration 114.67s (transform 568ms, setup 0ms, collect 985ms, tests 113.51s, environment 0ms, prepare 50ms) + diff --git a/evidence/parallel-llm-fix/related-pass.txt b/evidence/parallel-llm-fix/related-pass.txt new file mode 100644 index 000000000..4ba8caee0 --- /dev/null +++ b/evidence/parallel-llm-fix/related-pass.txt @@ -0,0 +1,24 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/preflight.test.ts (67 tests) 117ms + ✓ tests/cli-probe.test.ts (6 tests) 858ms + ✓ tests/communication-environment-preflight.test.ts (6 tests) 4ms + ✓ tests/cli.test.ts (70 tests) 1884ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 440ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 420ms + ✓ tests/authored-parallel-agents.test.ts (8 tests) 12632ms + ✓ authored steps under local workers with capacity > runs more concurrent f.llm calls than the worker holds side by side, never more than its capacity 1195ms + ✓ authored steps under local workers with capacity > completes more concurrent f.agent calls than the worker holds: the overflow waits for a slot instead of parking 2956ms + ✓ authored steps under local workers with capacity > runs agents in distinct working directories side by side (the kernel carries cwd) 637ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a symlink alias of the same directory 1926ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a directory nested inside the other 1046ms + ✓ authored steps under local workers with capacity > never starts queued agents once the body has failed 1915ms + ✓ authored steps under local workers with capacity > never starts a queued agent when the agent holding the only slot fails 1974ms + ✓ authored steps under local workers with capacity > parks the overflow when the body is not told the capacity (the defect this closes) 982ms + + Test Files 5 passed (5) + Tests 157 passed (157) + Start at 05:39:44 + Duration 13.97s (transform 1.47s, setup 0ms, collect 4.15s, tests 15.49s, environment 1ms, prepare 254ms) + diff --git a/evidence/parallel-llm-fix/related.txt b/evidence/parallel-llm-fix/related.txt new file mode 100644 index 000000000..32ff28569 --- /dev/null +++ b/evidence/parallel-llm-fix/related.txt @@ -0,0 +1,50 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/preflight.test.ts (67 tests) 104ms + ❯ tests/cli-probe.test.ts (6 tests | 1 failed) 812ms + × reports signal termination in both drivers 101ms + → expected Error: CLI probe failed { detail: 's…' } to match object { detail: 'signal:SIGTERM' } + ✓ tests/communication-environment-preflight.test.ts (6 tests) 5ms + ✓ tests/cli.test.ts (70 tests) 1710ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 330ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 389ms + ✓ tests/authored-parallel-agents.test.ts (8 tests) 11090ms + ✓ authored steps under local workers with capacity > runs more concurrent f.llm calls than the worker holds side by side, never more than its capacity 1130ms + ✓ authored steps under local workers with capacity > completes more concurrent f.agent calls than the worker holds: the overflow waits for a slot instead of parking 2560ms + ✓ authored steps under local workers with capacity > runs agents in distinct working directories side by side (the kernel carries cwd) 626ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a symlink alias of the same directory 1038ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a directory nested inside the other 1033ms + ✓ authored steps under local workers with capacity > never starts queued agents once the body has failed 1914ms + ✓ authored steps under local workers with capacity > never starts a queued agent when the agent holding the only slot fails 1915ms + ✓ authored steps under local workers with capacity > parks the overflow when the body is not told the capacity (the defect this closes) 872ms + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/cli-probe.test.ts > reports signal termination in both drivers +AssertionError: expected Error: CLI probe failed { detail: 's…' } to match object { detail: 'signal:SIGTERM' } + +- Expected ++ Received + +- Object { +- "detail": "signal:SIGTERM", ++ CliProbeError { ++ "detail": "spawn_failed", + } + + ❯ tests/cli-probe.test.ts:61:3 + 59| const { path, directory } = wrapper(identify + "process.kill(process… + 60| expect(() => probeCli(path, directory, 'test-model')).toThrow(expect… + 61| await expect(probeCliAsync(path, directory, 'test-model')).rejects.t… + | ^ + 62| }); + 63| + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed | 4 passed (5) + Tests 1 failed | 156 passed (157) + Start at 05:39:17 + Duration 12.24s (transform 1.21s, setup 0ms, collect 3.62s, tests 13.72s, environment 1ms, prepare 202ms) + diff --git a/evidence/parallel-llm-fix/repro-capacity1.txt b/evidence/parallel-llm-fix/repro-capacity1.txt new file mode 100644 index 000000000..9e486d9c2 --- /dev/null +++ b/evidence/parallel-llm-fix/repro-capacity1.txt @@ -0,0 +1,41 @@ +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.02s completionReason: success +○ llm-2 (llm) 0.00s +○ llm-3 (llm) 0.00s +○ llm-4 (llm) 0.00s +○ llm-5 (llm) 0.00s +○ llm-6 (llm) 0.00s +○ llm-7 (llm) 0.00s +○ llm-8 (llm) 0.00s +○ llm-9 (llm) 0.00s +○ llm-10 (llm) 0.00s +WAITING [worker_lease] Run "01M36CGEMJRJG2HRVZJ9C7CQB2" step "llm-2" (llm) is running under a worker lease until 1790142099406. +↻ llm-2 (llm) 2.29s +✓ llm-2 (llm) 4.25s completionReason: success +WAITING [worker_lease] Run "01M36CGGJD3NJP04H66RJW8MDE" step "llm-3" (llm) is running under a worker lease until 1790142101380. +↻ llm-3 (llm) 4.26s +✓ llm-3 (llm) 6.57s completionReason: success +WAITING [worker_lease] Run "01M36CGJV2FPXESM9MNPAHSMXA" step "llm-4" (llm) is running under a worker lease until 1790142103704. +↻ llm-4 (llm) 6.59s +✓ llm-4 (llm) 9.27s completionReason: success +WAITING [worker_lease] Run "01M36CGNFHQ2EH4GH0TRR9XGYB" step "llm-5" (llm) is running under a worker lease until 1790142106407. +↻ llm-5 (llm) 9.29s +✓ llm-5 (llm) 11.70s completionReason: success +WAITING [worker_lease] Run "01M36CGQV9QAK5Q8R2NT0Z83KE" step "llm-6" (llm) is running under a worker lease until 1790142108833. +↻ llm-6 (llm) 11.72s +✓ llm-6 (llm) 14.47s completionReason: success +WAITING [worker_lease] Run "01M36CGTHWWRQRJHYXFHP8MCVR" step "llm-7" (llm) is running under a worker lease until 1790142111602. +↻ llm-7 (llm) 14.48s +✓ llm-7 (llm) 17.31s completionReason: success +WAITING [worker_lease] Run "01M36CGXASCMH6YZDPW98WG2NN" step "llm-8" (llm) is running under a worker lease until 1790142114446. +↻ llm-8 (llm) 17.33s +✓ llm-8 (llm) 19.75s completionReason: success +WAITING [worker_lease] Run "01M36CGZPRH7QYGD3ESRAFY544" step "llm-9" (llm) is running under a worker lease until 1790142116878. +↻ llm-9 (llm) 19.76s +✓ llm-9 (llm) 22.22s completionReason: success +WAITING [worker_lease] Run "01M36CH241FY5H47NJ7JHT3ZBZ" step "llm-10" (llm) is running under a worker lease until 1790142119351. +↻ llm-10 (llm) 22.23s +✓ llm-10 (llm) 25.01s completionReason: success +○ run-11 (deterministic) 0.00s +✓ run-11 (deterministic) 0.02s completionReason: success +RUN 01M36CGCC7T4X6P1XRRKM3BBSF completed (12 steps) completionReason: success diff --git a/evidence/parallel-llm-fix/repro-capacity4.txt b/evidence/parallel-llm-fix/repro-capacity4.txt new file mode 100644 index 000000000..e67a6265d --- /dev/null +++ b/evidence/parallel-llm-fix/repro-capacity4.txt @@ -0,0 +1,41 @@ +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.02s completionReason: success +○ llm-2 (llm) 0.00s +○ llm-3 (llm) 0.00s +○ llm-4 (llm) 0.00s +○ llm-5 (llm) 0.00s +○ llm-6 (llm) 0.00s +○ llm-7 (llm) 0.00s +○ llm-8 (llm) 0.00s +○ llm-9 (llm) 0.00s +○ llm-10 (llm) 0.00s +WAITING [worker_lease] Run "01M36CF7BTGJEMYX64R70QA85F" step "llm-2" (llm) is running under a worker lease until 1790142059197. +↻ llm-2 (llm) 3.18s +WAITING [worker_lease] Run "01M36CF7C24EJ27MX7Q0GRHB3R" step "llm-3" (llm) is running under a worker lease until 1790142059198. +↻ llm-3 (llm) 3.18s +WAITING [worker_lease] Run "01M36CF7CADB4QKY9MY3ANKYC2" step "llm-4" (llm) is running under a worker lease until 1790142059205. +↻ llm-4 (llm) 3.18s +WAITING [worker_lease] Run "01M36CF7CMQFZWYJGC0JQM0AMR" step "llm-5" (llm) is running under a worker lease until 1790142059210. +↻ llm-5 (llm) 3.18s +✓ llm-5 (llm) 5.60s completionReason: success +WAITING [worker_lease] Run "01M36CF9RWTK7DTRCE76JMQYCX" step "llm-6" (llm) is running under a worker lease until 1790142061650. +↻ llm-6 (llm) 5.61s +✓ llm-2 (llm) 5.98s completionReason: success +✓ llm-3 (llm) 5.99s completionReason: success +WAITING [worker_lease] Run "01M36CFA4TRJSAMKXCNF073WH0" step "llm-7" (llm) is running under a worker lease until 1790142062033. +↻ llm-7 (llm) 6.00s +WAITING [worker_lease] Run "01M36CFA5CWS9RR3N5X7T7K7YS" step "llm-8" (llm) is running under a worker lease until 1790142062050. +↻ llm-8 (llm) 6.01s +✓ llm-4 (llm) 6.02s completionReason: success +WAITING [worker_lease] Run "01M36CFA640PMA3JP60HNHXQ1Z" step "llm-9" (llm) is running under a worker lease until 1790142062075. +↻ llm-9 (llm) 6.03s +✓ llm-6 (llm) 7.71s completionReason: success +WAITING [worker_lease] Run "01M36CFBV0WH9DNMJBQAW05J9D" step "llm-10" (llm) is running under a worker lease until 1790142063766. +↻ llm-10 (llm) 7.72s +✓ llm-9 (llm) 8.61s completionReason: success +✓ llm-7 (llm) 8.98s completionReason: success +✓ llm-8 (llm) 9.14s completionReason: success +✓ llm-10 (llm) 10.13s completionReason: success +○ run-11 (deterministic) 0.00s +✓ run-11 (deterministic) 0.02s completionReason: success +RUN 01M36CF44N8Q424XQZSGEA9EKJ completed (12 steps) completionReason: success diff --git a/evidence/parallel-llm-fix/repro-final-capacity1.txt b/evidence/parallel-llm-fix/repro-final-capacity1.txt new file mode 100644 index 000000000..024e93d1b --- /dev/null +++ b/evidence/parallel-llm-fix/repro-final-capacity1.txt @@ -0,0 +1,41 @@ +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.03s completionReason: success +○ llm-2 (llm) 0.00s +○ llm-3 (llm) 0.00s +○ llm-4 (llm) 0.00s +○ llm-5 (llm) 0.00s +○ llm-6 (llm) 0.00s +○ llm-7 (llm) 0.00s +○ llm-8 (llm) 0.00s +○ llm-9 (llm) 0.00s +○ llm-10 (llm) 0.00s +WAITING [worker_lease] Run "01M36DBY2Y33WH7GG9ZW4HN16K" step "llm-2" (llm) is running under a worker lease until 1790143000156. +↻ llm-2 (llm) 2.36s +✓ llm-2 (llm) 5.17s completionReason: success +WAITING [worker_lease] Run "01M36DC12SFN3XRP3XZS5DZPMY" step "llm-3" (llm) is running under a worker lease until 1790143003027. +↻ llm-3 (llm) 5.17s +✓ llm-3 (llm) 7.56s completionReason: success +WAITING [worker_lease] Run "01M36DC3EHT0GA8EC3RS4ZZZ2A" step "llm-4" (llm) is running under a worker lease until 1790143005448. +↻ llm-4 (llm) 7.56s +✓ llm-4 (llm) 10.55s completionReason: success +WAITING [worker_lease] Run "01M36DC6CQDQ7BB1FWV1JCMEK6" step "llm-5" (llm) is running under a worker lease until 1790143008461. +↻ llm-5 (llm) 10.55s +✓ llm-5 (llm) 12.98s completionReason: success +WAITING [worker_lease] Run "01M36DC8S3ARXF65D6ENCDH4HP" step "llm-6" (llm) is running under a worker lease until 1790143010905. +↻ llm-6 (llm) 12.98s +✓ llm-6 (llm) 15.31s completionReason: success +WAITING [worker_lease] Run "01M36DCB2DMASR0GX519PGSW1P" step "llm-7" (llm) is running under a worker lease until 1790143013253. +↻ llm-7 (llm) 15.31s +✓ llm-7 (llm) 17.31s completionReason: success +WAITING [worker_lease] Run "01M36DCD1DZGDCRES8X3G0JTWB" step "llm-8" (llm) is running under a worker lease until 1790143015267. +↻ llm-8 (llm) 17.30s +✓ llm-8 (llm) 19.45s completionReason: success +WAITING [worker_lease] Run "01M36DCF55HRFS7V80RYS5TY2X" step "llm-9" (llm) is running under a worker lease until 1790143017435. +↻ llm-9 (llm) 19.46s +✓ llm-9 (llm) 21.72s completionReason: success +WAITING [worker_lease] Run "01M36DCHCC4F42WYPBZPA7E8YY" step "llm-10" (llm) is running under a worker lease until 1790143019714. +↻ llm-10 (llm) 21.72s +✓ llm-10 (llm) 24.56s completionReason: success +○ run-11 (deterministic) 0.00s +✓ run-11 (deterministic) 0.02s completionReason: success +RUN 01M36DBVZG7SWRRDC9VVXW7GV9 completed (12 steps) completionReason: success diff --git a/evidence/parallel-llm-fix/repro-final-capacity4.txt b/evidence/parallel-llm-fix/repro-final-capacity4.txt new file mode 100644 index 000000000..7857ecf50 --- /dev/null +++ b/evidence/parallel-llm-fix/repro-final-capacity4.txt @@ -0,0 +1,41 @@ +○ run-1 (deterministic) 0.00s +✓ run-1 (deterministic) 0.03s completionReason: success +○ llm-2 (llm) 0.00s +○ llm-3 (llm) 0.00s +○ llm-4 (llm) 0.00s +○ llm-5 (llm) 0.00s +○ llm-6 (llm) 0.00s +○ llm-7 (llm) 0.00s +○ llm-8 (llm) 0.00s +○ llm-9 (llm) 0.00s +○ llm-10 (llm) 0.00s +WAITING [worker_lease] Run "01M36DBY9XFPVX5W815P9G0WSE" step "llm-2" (llm) is running under a worker lease until 1790143000261. +↻ llm-2 (llm) 2.73s +WAITING [worker_lease] Run "01M36DBYDBR8D7G9M1M73T7R0A" step "llm-3" (llm) is running under a worker lease until 1790143000349. +↻ llm-3 (llm) 2.70s +WAITING [worker_lease] Run "01M36DBYFKDZ5KAWKE539EHNXD" step "llm-4" (llm) is running under a worker lease until 1790143000440. +↻ llm-4 (llm) 2.68s +WAITING [worker_lease] Run "01M36DBYKEEQ3SBSQGQ8WP7JQK" step "llm-5" (llm) is running under a worker lease until 1790143000562. +↻ llm-5 (llm) 2.66s +✓ llm-3 (llm) 4.95s completionReason: success +WAITING [worker_lease] Run "01M36DC0XBC820MN9ZCDVAT5MK" step "llm-6" (llm) is running under a worker lease until 1790143002850. +↻ llm-6 (llm) 4.92s +✓ llm-2 (llm) 5.13s completionReason: success +WAITING [worker_lease] Run "01M36DC122A2T5FJG8ZA502F1Y" step "llm-7" (llm) is running under a worker lease until 1790143003003. +↻ llm-7 (llm) 5.04s +✓ llm-5 (llm) 5.59s completionReason: success +WAITING [worker_lease] Run "01M36DC1JH4KG5NEZTJJMSNV74" step "llm-8" (llm) is running under a worker lease until 1790143003537. +↻ llm-8 (llm) 5.56s +✓ llm-4 (llm) 5.72s completionReason: success +WAITING [worker_lease] Run "01M36DC1NY00KRCDVGTXP44KKQ" step "llm-9" (llm) is running under a worker lease until 1790143003638. +↻ llm-9 (llm) 5.65s +✓ llm-7 (llm) 7.46s completionReason: success +WAITING [worker_lease] Run "01M36DC3E3QA97KQWBFKWEYKZQ" step "llm-10" (llm) is running under a worker lease until 1790143005434. +↻ llm-10 (llm) 7.43s +✓ llm-6 (llm) 7.59s completionReason: success +✓ llm-8 (llm) 7.89s completionReason: success +✓ llm-9 (llm) 8.24s completionReason: success +✓ llm-10 (llm) 9.69s completionReason: success +○ run-11 (deterministic) 0.00s +✓ run-11 (deterministic) 0.02s completionReason: success +RUN 01M36DBVZYQGPS3GKMPTV3X0Y2 completed (12 steps) completionReason: success diff --git a/evidence/parallel-llm-fix/repro-final-journals.txt b/evidence/parallel-llm-fix/repro-final-journals.txt new file mode 100644 index 000000000..a58e62555 --- /dev/null +++ b/evidence/parallel-llm-fix/repro-final-journals.txt @@ -0,0 +1,2 @@ +{"directory": "/tmp/flows-verified-capacity1", "journals": 13, "lease_expired": []} +{"directory": "/tmp/flows-verified-capacity4", "journals": 13, "lease_expired": []} diff --git a/evidence/parallel-llm-fix/repro-journals.txt b/evidence/parallel-llm-fix/repro-journals.txt new file mode 100644 index 000000000..9edbe8ad0 --- /dev/null +++ b/evidence/parallel-llm-fix/repro-journals.txt @@ -0,0 +1,2 @@ +{"directory": "/tmp/flows-repro-capacity1", "journals": 13, "lease_expired": []} +{"directory": "/tmp/flows-repro-capacity4", "journals": 13, "lease_expired": []} diff --git a/evidence/parallel-llm-fix/scan-journals.py b/evidence/parallel-llm-fix/scan-journals.py new file mode 100644 index 000000000..7ac321e39 --- /dev/null +++ b/evidence/parallel-llm-fix/scan-journals.py @@ -0,0 +1,15 @@ +import glob +import json +import sqlite3 +import sys + +for directory in sys.argv[1:]: + paths = glob.glob(directory + '/runs/*.sqlite3') + expired = [] + for path in paths: + with sqlite3.connect(path) as db: + for step, payload in db.execute("select step_id, payload from entries where entry_type='step.completed'"): + if json.loads(payload).get('completionReason') == 'lease_expired': + expired.append({'journal': path, 'step': step}) + print(json.dumps({'directory': directory, 'journals': len(paths), 'lease_expired': expired})) + assert paths and not expired diff --git a/evidence/parallel-llm-fix/sdk-suite.txt b/evidence/parallel-llm-fix/sdk-suite.txt new file mode 100644 index 000000000..347f0f4a4 --- /dev/null +++ b/evidence/parallel-llm-fix/sdk-suite.txt @@ -0,0 +1,1691 @@ + +> @relayflows/sdk@2.0.29 test +> sh scripts/test.sh + + +> @relayflows/sdk@2.0.29 test:prep +> ( cd ../../kernel && sh ../ops/cargo.sh build ) && ( [ ! -d ../../testdata/preflight ] || find ../../testdata/preflight -name '*-cli' -type f -exec chmod +x {} + ) + + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.07s + +> @relayflows/sdk@2.0.29 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + + +> @relayflows/sdk@2.0.29 build +> tsc && node scripts/make-cli-executable.mjs + + +> @relayflows/sdk@2.0.29 typecheck:tests +> tsc -p tsconfig.tests.json + + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd +LIVE_KERNEL flows=/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js + + ✓ tests/preflight.test.ts (67 tests) 108ms + ✓ tests/cloud-read.test.ts (41 tests) 42ms + ❯ tests/hosted-extension-isolation.test.ts (22 tests | 15 failed) 308ms + × hosted extension capability isolation > executes the exact capability-only handler for a queued receipt 13ms + → bubblewrap is unavailable + × hosted extension capability isolation > executes the exact capability-only handler for a duplicate receipt 4ms + → bubblewrap is unavailable + × hosted extension capability isolation > launches through the captured process primitive after builtin export synchronization 12ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > ignores inherited launcher overrides and decodes manifests with the captured Buffer intrinsic 4ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > streams verified bytes when the live store is replaced and no writable staging path exists 202ms + → promise rejected "Error: Hosted extension sandbox exited wi… { code: '…' }" instead of resolving + × hosted extension capability isolation > mounts pinned private Surface bytes when the live package changes before launch 1ms + → ENOENT: no such file or directory, copyfile '/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-48daMa/dist/flow.js' + × hosted extension capability isolation > refuses Surface runtime bytes that differ from the reviewed pin before launch 1ms + → ENOENT: no such file or directory, copyfile '/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-nbb4yF/dist/flow.js' + × hosted extension capability isolation > refuses oversized Surface files through the bounded descriptor reader 1ms + → ENOENT: no such file or directory, copyfile '/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-0JhPEY/dist/flow.js' + × hosted extension capability isolation > shields verified Surface files before async settlement 1ms + → ENOENT: no such file or directory, copyfile '/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-SpYDeT/dist/flow.js' + × hosted extension capability isolation > preserves a typed host refusal while disclosing only a fixed marker to the child 4ms + → expected Error: bubblewrap is unavailable { code: '…' } to be Error: private Cloud policy detail { code: '…' } // Object.is equality + × hosted extension capability isolation > denies ambient credentials, host files, writes, network, subprocesses, and undeclared context verbs 7ms + → bubblewrap is unavailable + × hosted extension capability isolation > enforces OS address-space and data bounds on native Buffer allocation 4ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + × hosted extension capability isolation > blocks extra handler fields and authority-bearing receipt fields at the parent port 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension capability isolation > constructs adapter authority with the captured freeze intrinsic 3ms + → bubblewrap is unavailable + × hosted extension capability isolation > writes the Surface manifest and protocol without inherited toJSON behavior 3ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ✓ tests/cli.test.ts (70 tests) 1836ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 422ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 415ms + ✓ tests/plugin-extension.test.ts (90 tests) 437ms + ✓ tests/cloud-transcript-codex.test.ts (39 tests) 20ms + ✓ tests/cloud-sync.test.ts (40 tests) 772ms + ✓ tests/observer-link.test.ts (39 tests) 144ms +(node:24071) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-flow.test.ts (34 tests) 788ms + ✓ tests/agent-transcript.test.ts (29 tests) 283ms + ✓ tests/cli-status.test.ts (27 tests) 1011ms + ✓ flows status > resolves the run with no arguments from inside a worker-spawned agent 769ms + ✓ tests/cloud-run.test.ts (58 tests) 679ms + ✓ tests/babysitter-native-extension.test.ts (41 tests | 1 skipped) 1479ms + ✓ tests/relay-cli-surface.test.ts (75 tests) 39ms + ✓ tests/worker-cli.test.ts (18 tests) 24985ms + ✓ registered CLI model defaults > passes the same priced Claude default to the real provider invocation 396ms + ✓ step discovery environment > names the run, step, attempt and an absolute data dir for a direct agent spawn 419ms + ✓ step discovery environment > exports none of the four without a data dir, even when the worker inherited them 433ms + ✓ wrapper discovery environment > sets the four names from the dispatch and still refuses ambient values and other secrets 414ms + ✓ wrapper discovery environment > exports none of the four to a wrapper without a data dir, even when the worker inherited them 342ms + ✓ custom wrapper execution identity > passes an explicit safe environment at identification and execution 414ms + ✓ custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 368ms + ✓ custom wrapper execution identity > bounds wrapper execution after acknowledgement 384ms + ✓ custom wrapper execution identity > bounds captured wrapper output 307ms + ✓ custom wrapper execution identity > refuses a duplicate execute protocol frame 337ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 1906ms + ✓ custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 1907ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3597ms + ✓ custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11569ms + ✓ custom wrapper execution bounds are reader-owned > accepts an execute token and an over-8KiB payload flushed in one write 349ms + ✓ custom wrapper execution bounds are reader-owned > accepts the same over-8KiB payload whether or not it coalesces with the execute token 1139ms + ✓ custom wrapper execution bounds are reader-owned > still bounds an un-terminated handshake buffer and names the bound 370ms + ✓ delivers the journaled memory pack to the real wrapper and excludes its charge from completion usage 332ms + ✓ tests/step-failure-diagnostic.test.ts (25 tests) 58ms + ✓ tests/daemon-lifecycle.test.ts (42 tests) 45ms + ✓ tests/stop-process-group.test.ts (9 tests) 12759ms + ✓ every stop reaches the process group, not just the direct child > exits the run after an execution-timeout stop 775ms + ✓ every stop reaches the process group, not just the direct child > exits the run after a protocol terminate stop 395ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after a protocol terminate stop 1666ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after an execution-timeout stop 2033ms + ✓ every stop reaches the process group, not just the direct child > holds the loop open long enough for the escalation to run 1092ms + ✓ a wrapper that exits with no execution deadline still drains > reports the wrapper result and reaps a grandchild holding its pipes 656ms + ✓ a wrapper that exits with no execution deadline still drains > reaps a SIGTERM-deaf grandchild holding its pipes 1621ms + ✓ a wrapper that exits with no execution deadline still drains > settles on its own deadline when an escaped holder withholds close 4268ms + ✓ tests/run-state.test.ts (21 tests) 12ms + ✓ tests/cloud-deploy.test.ts (40 tests) 932ms + ❯ tests/hosted-extension-protocol.test.ts (24 tests | 8 failed) 40093ms + × hosted extension hostile protocol > uses captured JSON intrinsics for the complete parent boundary 7ms + → bubblewrap is unavailable + × hosted extension hostile protocol > rejects an import-time different PR frame with zero adapter calls 8ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension hostile protocol > rejects an import-time different delivery frame with zero adapter calls 3ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension hostile protocol > rejects an import-time different event frame with zero adapter calls 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension hostile protocol > rejects two forged calls after the authoritative first outcome settles 10003ms + → hostile child did not invoke the adapter + × hosted extension hostile protocol > waits for a pending adapter to reject after a forged child error 10005ms + → hostile child did not invoke the adapter + × hosted extension hostile protocol > waits for a pending adapter to resolve after a forged child error 10005ms + → hostile child did not invoke the adapter + × hosted extension hostile protocol > returns a typed adapter rejection even when the hostile child hangs 10006ms + → hostile child did not invoke the adapter + ✓ tests/hosted-base-snapshot.test.ts (18 tests) 1935ms + ✓ hosted base private snapshot > stops streaming project entries at the shared count limit 1854ms +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=25410 run=01M36CVPD1JG5S41C76P3C5D53 while step=two state=Running + + ❯ tests/live-kernel.test.ts (31 tests | 9 failed) 52134ms + ✓ built flows CLI against live relayflowd > twenty-six-step reuses 25 durable completions after editing the failed final step 2204ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 2659ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32477ms + ✓ built flows CLI against live relayflowd > follows a live worker dispatch through flows run 573ms + ✓ built flows CLI against live relayflowd > runs an agent CLI end to end through the SDK worker 323ms + ✓ built flows CLI against live relayflowd > f.agent lowers to a real agent step and dispatches through a live worker 473ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5593ms + ✓ built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 429ms + × built flows CLI against live relayflowd > runs hn-monitor analyze-story end-to-end via a stub agent CLI (gate 2 clause 2 demo) 425ms + → expected { …(12) } to match object { output: { …(3) }, …(1) } +(22 matching properties omitted from actual) + × built flows CLI against live relayflowd > hn-monitor analyze-story FAILS verification when the CLI omits required schema fields 385ms + → expected { …(12) } to match object { …(3) } +(21 matching properties omitted from actual) + × built flows CLI against live relayflowd > agent step preserves the CliResult wrapper as output when the CLI emits non-JSON text 402ms + → expected null not to be null + × built flows CLI against live relayflowd > AgentWorker exposes wake_context to the CLI via RELAYFLOW_WAKE_CONTEXT env var (real analyzer prerequisite) 346ms + → Cannot read properties of null (reading 'story_title') + × built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_WAKE_CONTEXT UNSET when the run has no wake_context (undefined-vs-null pin) 382ms + → Cannot read properties of null (reading 'env_present') + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 494ms + ✓ built flows CLI against live relayflowd > AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL 435ms + ✓ built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag 388ms + ✓ built flows CLI against live relayflowd > AgentWorker executes the raw codex adapter with its real model flag 365ms + × built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model 380ms + → Cannot read properties of null (reading 'story_title') + × built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 27ms + → LIVE_ANALYZER_UNAVAILABLE: "/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/preflight/analyze-story-claude-cli" does not identify as relayflows-agent-cli-v1 — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ✓ built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket 882ms + × built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir 527ms + → WARNING [unprovable_effects] Step "greet" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "shout" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +REFUSED [relayflowd_not_found] No relayflowd binary could be found. Install the runtime package for this host (@relayflows/runtime-linux-x64), or set RELAYFLOWD_BIN to a relayflowd executable. Tried: /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/relayflowd. +: expected 2 to be +0 // Object.is equality + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 881ms + × a relayflow can be scheduled: tick source against live relayflowd > a tick spawns a real run whose step reports the SCHEDULED instant 387ms + → expected null to deeply equal { schedule_id: 'heartbeat-1m', …(3) } + ✓ tests/journal-client.test.ts (17 tests) 82ms + ✓ tests/authored-root.test.ts (13 tests) 162ms + ✓ tests/cloud-connect.test.ts (24 tests) 2926ms + ✓ hosted verbs connect before they submit > flows run --cloud submits once the prompt connected the integration 2122ms + ✓ tests/flow-extension-compose.test.ts (22 tests) 3788ms + ✓ composing flow extensions onto a base flow > composes two extensions in declaration order, and the order is the lockfile order 412ms + ✓ composing flow extensions onto a base flow > flows check reports the composition and keeps the composed triggers deliverable 727ms + ❯ tests/authored-node-runtime.test.ts (14 tests | 14 skipped) 11ms + ❯ tests/mcp.test.ts (30 tests | 4 skipped) 9603ms + ✓ MCP preflight and transports > flows check refuses an undeclared server with exit 2 and no daemon 579ms + ✓ MCP preflight and transports > flows check reports a refusing server and leaves no PID 778ms + ✓ MCP preflight and transports > kills a SIGTERM-resistant silent child after a parent-owned handshake deadline 1313ms + ✓ MCP preflight and transports > reaps a SIGTERM-resistant descendant with inherit stdio before cleanup finishes 1110ms + ✓ MCP preflight and transports > reaps a SIGTERM-resistant descendant with ignore stdio before cleanup finishes 2061ms + ✓ MCP preflight and transports > reports malformed connection configuration as config_invalid 598ms + ✓ tests/bundle.test.ts (26 tests) 9311ms + ✓ immutable bundles > returns exit 2 naming a byte-flipped payload and refuses to reuse corruption 525ms + ✓ immutable bundles > verifies with --verify in any position and answers --json with one object 737ms + ✓ immutable bundles > refuses --out with --verify rather than ignoring the destination 367ms + ✓ immutable bundles > builds and verifies the canonical YAML fixture through the compiled CLI 1183ms + ✓ immutable bundles > emits the ephemeral warning on CLI stderr and uses the default output directory 746ms + ✓ immutable bundles > refuses build-provable CLI resolution errors without environment probes 386ms + ✓ immutable bundles > builds a standalone TS fixture twice with identical executable hashes 2442ms + ✓ immutable bundles > refuses to label installed dependency drift with lockfile pins 386ms + ✓ immutable bundles > refuses invalid CLI arguments %j 394ms + ✓ immutable bundles > refuses invalid CLI arguments "--out" 430ms + ✓ immutable bundles > refuses invalid CLI arguments "--verify" 578ms + ✓ immutable bundles > refuses invalid CLI arguments "--verify" 553ms + ✓ immutable bundles > refuses invalid CLI arguments "--out" 451ms + ✓ tests/validate.test.ts (68 tests) 25ms + ✓ tests/verb-field-lint.test.ts (96 tests) 320ms + ✓ tests/close-pr-flow.test.ts (28 tests) 10414ms + ✓ close-pr journaled repair loop > reads an existing Bugbot finding, repairs in the same worktree, pushes and re-verifies before merging 2239ms + ✓ close-pr journaled repair loop > opens a PR and feeds failed CI logs into the repair agent 1798ms + ✓ close-pr journaled repair loop > parks after exactly three nonconverging repairs, with accumulated blockers 1949ms + ✓ close-pr journaled repair loop > can converge on the third repair 2080ms + ✓ close-pr journaled repair loop > executes the deterministic commit and force-push steps against a local Git remote, including a no-op repair 2098ms + ✓ tests/tick-source.test.ts (33 tests) 25ms + ✓ tests/authored-step-graph.test.ts (25 tests) 815ms + ✓ tests/agent-relay-transport.test.ts (16 tests) 2236ms + ✓ Relay completion at the journal boundary > does not complete at readiness and journals exact output, receipt, and priced accounting 1006ms + ✓ Relay completion at the journal boundary > aborts polling on rejected renewal and never writes a stale completion 1002ms + ✓ tests/pr-review-post.test.ts (21 tests) 2129ms + ✓ tests/step-failure-excerpt.test.ts (42 tests) 193ms + ✓ tests/authored-flow-lifecycle-executor.test.ts (27 tests) 570ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1724ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 698ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 481ms + ✓ tests/tick-runner.test.ts (22 tests) 2288ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms fractional as an invocation error 389ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms exponent notation as an invocation error 370ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms hex as an invocation error 368ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms trailing text as an invocation error 385ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms empty as an invocation error 372ms + ✓ CLI argument parsing refuses coercion rather than accepting it > accepts an exact integer and proceeds past parsing 378ms + ✓ tests/authored-step-index.test.ts (17 tests) 20ms +(node:29919) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/flow-executor-chain.test.ts (14 tests) 10301ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 836ms + ✓ flow executor LLM and output-binding chain > runs a dollar-budgeted authored Claude agent with the same default used by preflight 474ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 360ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: null 510ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: "hello" 324ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1492ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3251ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 757ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1084ms + ✓ tests/cli-replay.test.ts (37 tests) 1302ms + ✓ flows replay > --json is byte-identical across two CLI invocations (diff) 792ms + ✓ tests/authored-node-result.test.ts (39 tests) 16ms + ✓ tests/gate-contract.test.ts (20 tests) 152ms + ✓ tests/authored-human.test.ts (13 tests) 84ms + ✓ tests/wrapper-execution-duration.test.ts (7 tests) 10889ms + ✓ keeps the handshake deadline independent of the removed execution deadline 10052ms + ✓ still lets a lease abort stop an unlimited wrapper before it produces output 510ms + ✓ tests/direct-input.test.ts (6 tests) 5651ms + ✓ direct .flow.ts input through the built CLI and live runtime > returns exit 3 for an authored human handoff and persists its outcome 569ms + ✓ direct .flow.ts input through the built CLI and live runtime > returns exit 1 for an authored step_failed verdict and persists its outcome 597ms + ✓ direct .flow.ts input through the built CLI and live runtime > executes inline and file JSON input through relayflowd 2033ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses missing and malformed input before contacting relayflowd 1533ms + ✓ direct .flow.ts input through the built CLI and live runtime > does not run the authored body before daemon availability 490ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses oversized file input before contacting relayflowd 429ms + ✓ tests/authored-run-failure-evidence.test.ts (9 tests) 1031ms + ✓ the child index after the process that wrote it is gone > still names every child, with its own run id, after a daemon restart 572ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 94ms + ✓ tests/cloud-schedule.test.ts (17 tests) 5614ms + ✓ schedule lowering > marks a non-grid cron as Cloud-only rather than approximating it, with a silence budget from its own cadence 3307ms + ✓ flows check prints declared schedules > shows the lowering for a fixed interval and the Cloud-only note for a real cron 1810ms + ✓ tests/daemon-lifecycle-live.test.ts (9 tests) 6432ms + ✓ flows run against a data dir with no daemon (§6 test 7) > cold start spawns exactly one daemon, the run succeeds, and the daemon outlives the CLI 473ms + ✓ flows run against a data dir with no daemon (§6 test 7) > polls, bounded, for a daemon that holds the lock before it binds 1365ms + ✓ flows run against a data dir with no daemon (§6 test 7) > attaches to a serving daemon that has not published a connection file 436ms + ✓ flows run against a data dir with no daemon (§6 test 7) > a second run attaches to the daemon the first one started, spawning nothing 840ms + ✓ flows run against a data dir with no daemon (§6 test 7) > detects a stale connection file left by a hard kill and starts a fresh daemon 935ms + ✓ concurrent invocations against one empty data dir (§6 test 15) > ends with exactly one daemon owning the socket, and both runs succeed 1080ms + ✓ refusals from a spawn that cannot produce a daemon > names relayflowd_not_found rather than falling through to PATH 393ms + ✓ refusals from a spawn that cannot produce a daemon > names daemon_start_failed and quotes the daemon log when startup dies 455ms + ✓ refusals from a spawn that cannot produce a daemon > refuses a daemon speaking another protocol version instead of binding over it 454ms +(node:31388) Warning: Transcript tail for run-9/analyze attempt 1 (stdout) could not be written; the step continues without it: EACCES: permission denied, mkdir '/tmp/transcript-tail-t1B4pL/runs/run-9/steps' +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/transcript-tail.test.ts (11 tests) 753ms + ✓ direct agent spawn > tees stdout and stderr into tail files that name the dispatch 322ms + ✓ direct agent spawn > completes the step when the tail directory cannot be created 352ms + ✓ tests/authored-agent-artifacts.test.ts (4 tests) 311ms + ✓ tests/authored-helpers.test.ts (6 tests) 2932ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1529ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 500ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 499ms + ✓ tests/authored-flow-operation.test.ts (24 tests) 546ms + ✓ tests/flow-requirements.test.ts (14 tests) 466ms + ✓ tests/backlog-picker.test.ts (14 tests) 47ms + ✓ tests/plugin-store-bounds.test.ts (11 tests) 75ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 267ms + ✓ tests/hosted-extension-protocol-intrinsics.test.ts (6 tests) 13ms + ✓ tests/preflight-permissions-unenforced.test.ts (17 tests) 215ms + ✓ tests/wrapper-exit-drain.test.ts (8 tests) 2642ms + ✓ reports a signalled wrapper death while a pipe is held, with its output intact 383ms + ✓ refuses a duplicate execute frame found at finalization, despite a clean exit 310ms + ✓ lets a lease abort outrank a successful exit still being drained 527ms + ❯ tests/stuck-run-triage.test.ts (22 tests | 22 failed) 70ms + × stuck-run-triage input validation > refuses an 8-character run-id prefix: Cloud has no prefix lookup 5ms + → expected [Function] to throw error matching /not full Cloud run ids: c649fe14/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage input validation > refuses the whole batch when any id is invalid, rather than dropping it 1ms + → expected [Function] to throw error matching /not full Cloud run ids: nope!/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage input validation > refuses an empty batch 1ms + → expected [Function] to throw error matching /needs runIds/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage input validation > refuses a batch too large for the edge step lease 1ms + → expected [Function] to throw error matching /exceeds the 8 that fit/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage input validation > accepts eight ids — the incident batch is inside the bound 3ms + → promise rejected "TypeError: expected an @relayflows/surfac…" instead of resolving + × stuck-run-triage apiUrl > refuses to send the Cloud bearer token to an unapproved origin 0ms + → expected [Function] to throw error matching /refusing to send the Cloud bearer to…/\ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage apiUrl > refuses a non-URL apiUrl 0ms + → expected [Function] to throw error matching /is not a URL/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage apiUrl > allows an approved origin and uses it in the curl 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage apiUrl > defaults to production Cloud 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage apiUrl > never publishes a run record the fetch did not produce 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > names the Worker on every wrangler invocation 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > accepts caller-supplied Workers and rejects option-shaped ones 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > falls back when GNU timeout is absent, as it is on macOS 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > runs the tails concurrently so wall time does not scale with the batch 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > records wrangler's own exit status rather than head's 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage shell text > parses under both sh and bash 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage shell text > collects tails with no GNU timeout on PATH, as on a stock macOS 53ms + → expected an @relayflows/surface flow handle + × stuck-run-triage agents > declares read-only permissions on every agent 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage agents > tells the forensics agents their evidence is untrusted 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage fan-out > refuses a duplicate run id: two tails would share one evidence file 1ms + → expected [Function] to throw error matching /duplicate runIds: c649fe14-0c2e-4e51-…/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage fan-out > refuses a duplicate Worker name for the same reason 0ms + → expected [Function] to throw error matching /duplicate workers: w-one/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage fan-out > bounds ids x workers, not just ids 0ms + → expected [Function] to throw error matching /24 concurrent tails, over the 16/ but got 'expected an @relayflows/surface flow …' + ✓ tests/worker-transcript.test.ts (5 tests) 180ms + ✓ tests/hosted-extension-routing.test.ts (7 tests) 7ms + ✓ tests/artifact-gates.test.ts (7 tests) 175ms + ✓ tests/webhook.test.ts (9 tests) 807ms + ✓ webhook ingress > checks TS declarations against flows.json without invoking handlers 709ms + ✓ tests/agent-artifacts-live.test.ts (5 tests) 43638ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > journals the files the agent wrote, including under a dot-directory, and every artifact gate passes on that journal 1107ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > fails the run when the artifact_exists gate names a file the agent did not write 13048ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > fails the run with the author reason when a predicate gate returns false, journaling the verdict 14268ms + ✓ review follow-ups > applies a predicate gate on a helper step too, and journals its verdict 14180ms + ✓ review follow-ups > records predicate verdicts on the root run so a resume reuses them instead of re-running the closure 1033ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 45166ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 14668ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 14652ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 773ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 15073ms + ✓ tests/human-live.test.ts (3 tests) 6241ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3714ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1561ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 964ms + ✓ tests/authored-step-failed.test.ts (10 tests) 37ms + ✓ tests/cli-watch.test.ts (10 tests) 15881ms + ✓ flows check --watch > rechecks syntax errors, clears once, and returns the last refusal on Ctrl-C 1293ms + ✓ flows check --watch > streams JSON lines without ANSI, recovers after atomic saves, and exits zero after repair 1948ms + ✓ flows check --watch > coalesces 20 concurrent saves into at most two rechecks 1857ms + ✓ flows check --watch > watches transitive relative use imports, cycles, and nearest config changes 2361ms + ✓ flows check --watch > refreshes the import graph and notices missing imports being created 2334ms + ✓ flows check --watch > reloads authored TypeScript instead of reusing the first imported definition 1553ms + ✓ flows check --watch > detects a nearer config appearing and falls back after it is deleted 1910ms + ✓ flows check --watch > keeps watching after the target is deleted and recreated 1851ms + ✓ flows check --watch > queues changes during a slow check without overlapping checks 773ms + ✓ tests/budget-preflight.test.ts (25 tests) 16ms + ✓ tests/authored-parallel-agents.test.ts (8 tests) 11090ms + ✓ authored steps under local workers with capacity > runs more concurrent f.llm calls than the worker holds side by side, never more than its capacity 1127ms + ✓ authored steps under local workers with capacity > completes more concurrent f.agent calls than the worker holds: the overflow waits for a slot instead of parking 2399ms + ✓ authored steps under local workers with capacity > runs agents in distinct working directories side by side (the kernel carries cwd) 632ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a symlink alias of the same directory 1109ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a directory nested inside the other 1050ms + ✓ authored steps under local workers with capacity > never starts queued agents once the body has failed 1925ms + ✓ authored steps under local workers with capacity > never starts a queued agent when the agent holding the only slot fails 1916ms + ✓ authored steps under local workers with capacity > parks the overflow when the body is not told the capacity (the defect this closes) 930ms + ✓ tests/budget-unmetered-live.test.ts (3 tests) 1343ms + ✓ unmetered budget spend through the live kernel > runs an unpriced step under a dollar budget without tripping it, journaling unknown dollars 496ms + ✓ unmetered budget spend through the live kernel > still counts an unpriced step toward a token budget 568ms + ✓ tests/provider-trigger-contract.test.ts (7 tests) 526ms + ✓ provider trigger contract > fails `flows check` before deployment and passes once the event is real 325ms + ✓ tests/work-package-consumer.test.ts (13 tests) 107ms + ✓ tests/spec-parity.test.ts (31 tests) 361ms + ✓ tests/helpers-fanout.test.ts (96 tests) 227ms +(node:35350) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-step-graph-live.test.ts (1 test) 694ms + ✓ the authored step DAG through the live kernel > carries labels and predecessors on every index record and journal step, ids unchanged 694ms + ✓ tests/generate-triggers.test.ts (7 tests) 1076ms + ✓ discovers new adapters, preserves exact event names, and prefers adapter-local mappings 346ms + ❯ tests/webhook-live.test.ts (6 tests | 6 failed) 62434ms + × executes and deduplicates 'app_mention' only for its provider and matching payload 10413ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × executes and deduplicates 'reaction_added' only for its provider and matching payload 10409ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × executes and deduplicates 'pull_request' only for its provider and matching payload 10411ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × flows serve-webhook writes JSON before the daemon starts, then journals and archives exactly once 10400ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × replays a dropped file after SIGKILL before spawn 10408ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × resumes the same journal after SIGKILL after spawn and before acknowledgement 10392ms + → webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ✓ tests/webhook-hardening.test.ts (11 tests) 62ms + ✓ tests/human-to.test.ts (8 tests) 10ms + ✓ tests/pty-sidechannel.test.ts (11 tests) 5715ms + ✓ view attach preserves worker completion and marks only drive 782ms + ✓ drive attach preserves worker completion and marks only drive 371ms + ✓ passthrough attach preserves worker completion and marks only drive 799ms + ✓ none attach preserves worker completion and marks only drive 778ms + ✓ none subscriber lets an unattended CLI read EOF 412ms + ✓ view subscriber lets an unattended CLI read EOF 369ms + ✓ passthrough subscriber lets an unattended CLI read EOF 409ms + ✓ incomplete subscriber lets an unattended CLI read EOF 412ms + ✓ rejects drive after EOF without marking human intervention 730ms + ✓ delivers all drive bytes in order across child stdin backpressure 650ms + ✓ tests/plugin-loader.test.ts (9 tests) 192ms + ✓ tests/worker-cli-result-exit.test.ts (5 tests) 32887ms + ✓ a Claude agent step completes on its result, not only on process exit > settles a hung, successful run within the grace and stops its whole tree 31615ms + ✓ a Claude agent step completes on its result, not only on process exit > maps an error result on a hung run to a failed exit 31616ms + ✓ a Claude agent step completes on its result, not only on process exit > leaves a hang before any result to the existing stops 32010ms + ✓ an agent tree does not outlive the process that spawned it > kills the agent group when the run process is terminated by SIGTERM 797ms + ✓ tests/worker-lease.test.ts (7 tests) 12ms + ✓ tests/yaml-helpers.test.ts (33 tests) 75ms + ✓ tests/authored-agent-permissions.test.ts (26 tests) 780ms + ✓ tests/babysitter-catalog-export.test.ts (14 tests) 784ms + ✓ Babysitter catalog artifact export > CLI refuses an existing output and leaves no file on validation failure 624ms + ✓ tests/redact.test.ts (35 tests) 11ms + ✓ tests/communication.test.ts (10 tests) 14ms + ✓ tests/typed-output.test.ts (14 tests) 212ms + ❯ tests/canonical-software-factory.test.ts (3 tests | 3 failed) 19ms + × canonical software-factory metadata contract > opens the actual catalog flow with the ticket title and exactly one GitHub closing line 12ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > normalizes whitespace and caps the title at 240 Unicode code points 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > fails closed before push when GitHub identity is missing or the final body duplicates its closing line 0ms + → expected an @relayflows/surface flow handle + ✓ tests/budget-attribution.test.ts (5 tests) 7ms + ✓ tests/deploy.test.ts (11 tests) 5282ms + ✓ flows deploy file buckets > publishes the full signed layout byte-for-byte and redeploys as a noop 845ms + ✓ flows deploy file buckets > answers --json with one object per outcome 802ms + ✓ flows deploy file buckets > reports a refusal as JSON under --json 405ms + ✓ flows deploy file buckets > refuses a missing local bundle before creating the bucket 396ms + ✓ flows deploy file buckets > refuses an unreachable bucket before copying 417ms + ✓ flows deploy file buckets > refuses an unwritable bucket 386ms + ✓ flows deploy file buckets > refuses local tampering of spec.canonical.json 385ms + ✓ flows deploy file buckets > refuses local tampering of identity.json 395ms + ✓ flows deploy file buckets > refuses asset bundles instead of using daemon-relative files 414ms + ✓ flows deploy file buckets > never labels a corrupt existing deployment as a noop 812ms + ✓ tests/effect-channel.test.ts (5 tests) 349ms + ✓ tests/json-schema-bound.test.ts (71 tests) 2452ms + ✓ JSON Schema termination bound > walks a deep schema with an explicit stack rather than recursion 2010ms + ✓ tests/mcp-lifecycle.test.ts (4 tests) 12ms + ✓ tests/model-selection.test.ts (10 tests) 17ms + ✓ tests/f-memory.test.ts (7 tests) 828ms + ✓ tests/relayflowd-path.test.ts (10 tests) 5ms + ✓ tests/agent-artifacts.test.ts (9 tests) 19ms + ✓ tests/step-lease.test.ts (36 tests) 66546ms + ✓ f.run leases against the live kernel > enforces 10000 ms for 'sleep 5; printf ok' 5093ms + ✓ f.run leases against the live kernel > enforces 40000 ms for 'sleep 31; printf ok' 31084ms + ✓ f.run leases against the live kernel > enforces 30000 ms for 'sleep 31; printf ok' 30123ms + ✓ tests/authored-plugin-effect.test.ts (6 tests) 55ms + ✓ tests/yaml-local-agent-live.test.ts (7 tests) 4580ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked step CLI and model and journals done 641ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked named CLI and model and journals done 677ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked flow CLI and model and journals done 915ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked project CLI and model and journals done 623ms + ✓ YAML --local-agent through the built CLI and real daemon > still parks without --local-agent 534ms + ✓ YAML --local-agent through the built CLI and real daemon > reports the agent process failure 622ms + ✓ YAML --local-agent through the built CLI and real daemon > preserves declared workspace surfaces that the local worker cannot pin 568ms + ✓ tests/worker-slots.test.ts (7 tests) 5ms + ✓ tests/local-dev-ux.test.ts (8 tests) 63ms + ↓ tests/relay-cli-surface-live.test.ts (3 tests | 3 skipped) + ✓ tests/authored-declined.test.ts (13 tests) 48ms + ✓ tests/resume-failure.test.ts (2 tests) 5ms + ✓ tests/dependency-validation.test.ts (6 tests) 800ms + ✓ dependency validation > accepts a valid 10,000-step reverse chain through every direct public boundary 430ms + ✓ tests/authored-hooks.test.ts (5 tests) 7ms + ✓ tests/local-agent-live.test.ts (5 tests) 66377ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 863ms + ✓ built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 36934ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 1091ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 12868ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 14621ms + ✓ tests/input-binding.test.ts (12 tests) 419ms + ✓ tests/communication-review.test.ts (5 tests) 354ms + ✓ tests/yaml-helper-effect.test.ts (4 tests) 143ms + ✓ tests/deterministic-llm.test.ts (5 tests) 96ms + ✓ tests/scope-preflight.test.ts (6 tests) 8ms + ✓ tests/build-gate.test.ts (3 tests) 1683ms + ✓ flows build gates on flows check green (#318) > refuses a flow with an unresolvable named-agent CLI and leaves no artifacts 675ms + ✓ flows build gates on flows check green (#318) > --json emits one CheckReport object on stdout on refusal, exits 2, no artifacts 432ms + ✓ flows build gates on flows check green (#318) > builds the bundle on success (regression: gate must not block valid flows) 576ms + ✓ tests/scope-compiler.test.ts (25 tests) 11ms + ✓ tests/bin.test.ts (7 tests) 3653ms + ✓ built flows binary > refuses through a symlink to the built artifact 816ms + ✓ built flows binary > refuses through a symlinked directory component 671ms + ✓ built flows binary > classifies a signal-terminated auth probe as probe_failed 675ms + ✓ built flows binary > classifies an unavailable PATH resolver as probe_failed 592ms + ✓ built flows binary > does not describe a present non-executable CLI as missing 488ms + ✓ built flows binary > runs one auth probe for three steps sharing a flow CLI 408ms + ✓ tests/communication-worker.test.ts (15 tests) 1493ms + ✓ tests/hn-poller.test.ts (6 tests) 7ms + ✓ tests/plugin-add.test.ts (7 tests) 1883ms + ✓ installs a real offline npm fixture and includes declarations 476ms + ✓ typechecks the augmented verb and rejects unknown namespaces 1394ms + ✓ tests/authored-step-failed-exit.test.ts (3 tests) 8ms + ✓ tests/direct-run-failure.test.ts (8 tests) 11ms + ✓ tests/run-from-digest.test.ts (6 tests) 5598ms + ✓ flows run digest input > submits the sealed canonical spec through the normal journal path without checkout 512ms + ✓ flows run digest input > uses a verified cache hit even after the bucket is removed 437ms + ✓ flows run digest input > resolves deploy.bucket from flows.json and honors explicit override 1496ms + ✓ flows run digest input > refuses an unconfigured bucket 1215ms + ✓ flows run digest input > refuses tampered spec.canonical.json before creating run data 1095ms + ✓ flows run digest input > refuses tampered identity.json before creating run data 842ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 4ms + ✓ tests/model-pricing.test.ts (10 tests) 5ms + ❯ tests/provider-trigger-executor.test.ts (4 tests | 3 failed) 18ms + × the kernel executes compiled 'app_mention' subscriptions with provider isolation and durable dedupe 10ms + → spawnSync /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × the kernel executes compiled 'reaction_added' subscriptions with provider isolation and durable dedupe 3ms + → spawnSync /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + × the kernel executes compiled 'pull_request' subscriptions with provider isolation and durable dedupe 3ms + → spawnSync /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ✓ tests/transcript-tail-close.test.ts (2 tests) 913ms + ✓ a stalled transcript-tail close > does not hold the spawn open past its bounded window 476ms + ✓ a stalled tail close beside a transcript that finished > still journals the transcript pointer 437ms + ✓ tests/wrapper-artifacts-cwd.test.ts (2 tests) 77ms + ✓ tests/yaml-helper-live.test.ts (1 test) 1735ms + ✓ runs compiled YAML helpers through the built CLI and kernel effect journal 1735ms + ✓ tests/hello-deterministic.test.ts (5 tests) 17ms + ✓ tests/cli-adapter.test.ts (4 tests) 5ms + ✓ tests/transcript-exclusion-timeout.test.ts (1 test) 187ms + ✓ tests/work-package-validator.test.ts (7 tests) 6ms + ❯ tests/communication-mixed-resume.test.ts (1 test | 1 failed) 13ms + × resumes mixed ordinary and linked agents through the real daemon without stealing peer capacity 13ms + → ENOENT: no such file or directory, open '/tmp/communication-resume-vfA6bi/data/connection.json' + ✓ tests/authored-use-loader.test.ts (5 tests) 709ms + ✓ tests/cli-probe.test.ts (6 tests) 983ms + ✓ tests/cli-answer.test.ts (15 tests) 9ms + ✓ tests/authored-declined-live.test.ts (1 test) 1800ms + ✓ runs an input guard and resumes its completed declined root without repeated effects 1800ms + ✓ tests/agent-relay-hardening.test.ts (12 tests) 12ms + ✓ tests/bundle-preflight.test.ts (4 tests) 892ms + ✓ bundle execution preflight > ignores surrounding cache configuration on a verified cache hit 447ms + ✓ bundle execution preflight > uses the built alias for a nameless flow even in a digest-only cache directory 409ms + ✓ tests/communication-preflight.test.ts (13 tests) 34ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/memoization.test.ts (57 tests) 51ms + ✓ tests/fs-descriptor.test.ts (1 test) 6ms + ✓ tests/classify-outcome.test.ts (2 tests) 2161ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2007ms + ✓ tests/authored-preflight.test.ts (3 tests) 111ms + ✓ tests/parse-json-output.test.ts (7 tests) 5ms + ✓ tests/journal-client-completion.test.ts (4 tests) 102ms + ✓ tests/communication-environment-preflight.test.ts (6 tests) 8ms + ✓ tests/budget-authored-live.test.ts (2 tests) 307ms + ✓ tests/slack-writeback.test.ts (1 test) 258ms + ✓ tests/authored-surface-authority.test.ts (2 tests) 225ms + ✓ tests/adapters/claude.test.ts (7 tests) 5ms + ✓ tests/worker-cli-abort.test.ts (2 tests) 2649ms + ✓ stops claude and its process group when lease ownership is lost 1301ms + ✓ stops wrapper.mjs and its process group when lease ownership is lost 1347ms + ✓ tests/worker-cli-cwd.test.ts (2 tests) 265ms + ✓ tests/adapters/codex.test.ts (7 tests) 4ms + ✓ tests/communication-history.test.ts (1 test) 3ms + ✓ tests/adapters/registry.test.ts (4 tests) 4ms + ✓ tests/slack-block-kit.test.ts (5 tests) 13ms + ✓ tests/promise-ancestry.test.ts (2 tests) 311ms + ✓ reachableTargets > handles more than 32 targets and a chain too deep for recursion 308ms + ✓ tests/authored-declined-report.test.ts (6 tests) 8ms + ✓ tests/communication-refusal.test.ts (1 test) 15ms + ✓ tests/agent-cwd-validation.test.ts (2 tests) 424ms + ✓ declarative agent cwd > is refused by `flows check` on a YAML flow before anything runs 421ms + ✓ tests/catalog-plugins.test.ts (2 tests) 3ms + ✓ tests/check-command-cwd.test.ts (1 test) 13ms + ✓ tests/communication-lazy.test.ts (1 test) 4ms + ✓ tests/cli-progress-wait.test.ts (2 tests) 3ms + ↓ tests/run-digest-live.test.ts (1 test | 1 skipped) + ✓ tests/placement.test.ts (54 tests) 16ms + ✓ tests/bundle-transport.test.ts (20 tests) 2599ms + ✓ digest references > accepts and deploys the build output for hello 430ms + ✓ digest references > accepts and deploys the build output for Hello 420ms + ✓ digest references > accepts and deploys the build output for hello.world 415ms + ✓ digest references > accepts and deploys the build output for hello_world 418ms + ✓ digest references > accepts and deploys the build output for 123 492ms + ✓ digest references > accepts and deploys the build output for A_b.c-1 422ms + ✓ tests/canonical-tree.test.ts (1 test) 2ms + ✓ tests/communication-tools.test.ts (1 test) 77ms + ✓ tests/authored-admission.test.ts (2 tests) 3ms + ✓ tests/memory.test.ts (18 tests) 9ms + ✓ tests/worker-platform.test.ts (1 test) 3ms + ✓ tests/run-digest.test.ts (4 tests) 1610ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {invalid json 405ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{}} 397ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{"bucket":123}} 406ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{"bucket":""}} 401ms + ✓ tests/authored-parallel-llm.test.ts (6 tests) 117134ms + ✓ parallel llm capacity 1 > deduplicates concurrent preflight probes 3035ms + ✓ parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 9012ms + ✓ parallel llm capacity 1 > keeps the durable root lease alive across two cold models 48242ms + ✓ parallel llm capacity 4 > deduplicates concurrent preflight probes 1998ms + ✓ parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight 8082ms + ✓ parallel llm capacity 4 > keeps the durable root lease alive across two cold models 46754ms + +⎯⎯⎯⎯⎯⎯ Failed Suites 2 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-node-runtime.test.ts [ tests/authored-node-runtime.test.ts ] +AssertionError: expected '1.3.6' to be '1.4.0' // Object.is equality + +Expected: "1.4.0" +Received: "1.3.6" + + ❯ tests/authored-node-runtime.test.ts:18:77 + 16| + 17| beforeAll(() => { + 18| expect(spawnSync(bun, ['--version'], { encoding: 'utf8' }).stdout.tr… + | ^ + 19| expect(existsSync(daemon), 'build the current kernel or set RELAYFLO… + 20| stage = mkdtempSync(join(tmpdir(), 'authored-standalone-build-')); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/69]⎯ + + FAIL tests/mcp.test.ts > authored MCP effects against the real kernel +Error: journal client: connect failed: connect ENOENT /tmp/relayflowd-1c26badc8d89.sock + ❯ Socket.onError src/journal-client.ts:102:16 + 100| socket.removeAllListeners(); + 101| this.failAll(err); + 102| reject(new Error(`journal client: connect failed: ${err.messag… + | ^ + 103| }; + 104| socket.once('error', onError); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/69]⎯ + +⎯⎯⎯⎯⎯⎯ Failed Tests 67 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > opens the actual catalog flow with the ticket title and exactly one GitHub closing line +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:69:22 + 67| }; + 68| + 69| const definition = getFlowDefinition(softwareFactory); + | ^ + 70| return definition.body(context as never, { issue, approver: 'khaliq'… + 71| root, + ❯ tests/canonical-software-factory.test.ts:82:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/69]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > normalizes whitespace and caps the title at 240 Unicode code points +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:69:22 + 67| }; + 68| + 69| const definition = getFlowDefinition(softwareFactory); + | ^ + 70| return definition.body(context as never, { issue, approver: 'khaliq'… + 71| root, + ❯ tests/canonical-software-factory.test.ts:99:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/69]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > fails closed before push when GitHub identity is missing or the final body duplicates its closing line +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ tests/canonical-software-factory.test.ts:108:24 + 106| + 107| it('fails closed before push when GitHub identity is missing or the … + 108| const definition = getFlowDefinition(softwareFactory); + | ^ + 109| const commands: string[] = []; + 110| let completionReason = ''; + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/69]⎯ + + FAIL tests/communication-mixed-resume.test.ts > resumes mixed ordinary and linked agents through the real daemon without stealing peer capacity +Error: ENOENT: no such file or directory, open '/tmp/communication-resume-vfA6bi/data/connection.json' + ❯ tests/communication-mixed-resume.test.ts:54:35 + 52| } finally { + 53| clearTimeout(timeout); state.release(); client.close(); + 54| try { process.kill(JSON.parse(readFileSync(join(dataDir, 'connecti… + | ^ + 55| finally { rmSync(root, { recursive: true, force: true }); } + 56| } + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[6/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > executes the exact capability-only handler for a queued receipt + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > executes the exact capability-only handler for a duplicate receipt +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:221:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[7/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > launches through the captured process primitive after builtin export synchronization +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:283:11 + 281| input: descriptor(), + 282| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 283| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 284| } finally { + 285| process.execPath = originalExecPath; + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:277:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[8/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > ignores inherited launcher overrides and decodes manifests with the captured Buffer intrinsic +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:373:11 + 371| input: descriptor('delivery-options'), + 372| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 373| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 374| } finally { + 375| Buffer.prototype.toString = bufferToString; + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:367:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[9/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > streams verified bytes when the live store is replaced and no writable staging path exists +AssertionError: promise rejected "Error: Hosted extension sandbox exited wi… { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:431:7 + 429| return { receiptId: 'receipt-1', status: 'queued' }; + 430| } }, + 431| })).resolves.toEqual({ completionReason: 'success', capabilityCall… + | ^ + 432| expect(calls).toBe(1); + 433| expect(readFileSync(join(installed.directory, 'babysitter.flow.ts'… + +Caused by: Error: Hosted extension sandbox exited without a valid completion (exit 1): /tmp/hosted-bwrap-wrapper-VmLyve/bwrap-wrapper:20 +if (child.error) throw child.error; + ^ + +Error: spawnSync /usr/bin/bwrap ENOENT + at Object.spawnSync (node:internal/child_process:1103:20) + at spawnSync (node:child_process:911:24) + at Object. (/tmp/hosted-bwrap-wrapper-VmLyve/bwrap-wrapper:19:15) + at Module._compile (node:internal/modules/cjs/loader:1809:14) + at Object..js (node:internal/modules/cjs/loader:1940:10) + at Module.load (node:internal/modules/cjs/loader:1530:32) + at Module._load (node:internal/modules/cjs/loader:1332:12) + at wrapModuleLoad (node:internal/modules/cjs/loader:255:19) + at Module.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:154:5) + at node:internal/main/run_main_module:33:47 { + errno: -2, + code: 'ENOENT', + syscall: 'spawnSync /usr/bin/bwrap', + path: '/usr/bin/bwrap', + spawnargs: [ + '--unshare-all', + '--die-with-parent', + '--new-session', + '--clearenv', + '--cap-drop', + 'ALL', + '--dir', + '/usr', + '--ro-bind', + '/usr/lib', + '/usr/lib', + '--ro-bind', + '/usr/lib64', + '/usr/lib64', + '--ro-bind', + '/usr/lib', + '/lib', + '--ro-bind', + '/usr/lib64', + '/lib64', + '--proc', + '/proc', + '--dev', + '/dev', + '--tmpfs', + '/tmp', + '--dir', + '/runtime', + '--dir', + '/extension', + '--dir', + '/extension/node_modules', + '--dir', + '/extension/node_modules/@relayflows', + '--dir', + '/extension/node_modules/@relayflows/surface', + '--dir', + '/extension/node_modules/@relayflows/surface/dist', + '--dir', + '/extension/node_modules/@relayflows/surface/dist/helpers', + '--dir', + '/extension/node_modules/@relayflows/surface/dist/triggers', + '--dir', + '/extension/src', + '--perms', + '0500', + '--ro-bind-data', + '4', + '/runtime/node', + '--perms', + '0400', + '--ro-bind-data', + '5', + '/runtime/runner.mjs', + '--perms', + '0400', + '--ro-bind-data', + '6', + '/extension/node_modules/@relayflows/surface/package.json', + '--perms', + '0400', + '--ro-bind-data', + '7', + '/extension/node_modules/@relayflows/surface/index.js', + '--perms', + '0400', + '--ro-bind-data', + '8', + '/extension/node_modules/@relayflows/surface/runtime.js', + '--perms', + '0400', + '--ro-bind-data', + '9', + '/extension/node_modules/@relayflows/surface/dist/flow.js', + '--perms', + '0400', + '--ro-bind-data', + '10', + '/extension/node_modules/@relayflows/surface/dist/helpers/providers.js', + '--perms', + '0400', + '--ro-bind-data', + '11', + '/extension/node_modules/@relayflows/surface/dist/provider-trigger.js', + '--perms', + '0400', + '--ro-bind-data', + '12', + '/extension/node_modules/@relayflows/surface/dist/schedule.js', + '--perms', + '0400', + '--ro-bind-data', + '13', + '/extension/node_modules/@relayflows/surface/dist/triggers.js', + '--perms', + '0400', + '--ro-bind-data', + '14', + '/extension/node_modules/@relayflows/surface/dist/triggers/github.js', + '--perms', + ... 27 more items + ] +} + +Node.js v25.6.0 + + ❯ Object. ../../../../../../../tmp/hosted-bwrap-wrapper-VmLyve/bwrap-wrapper:19:15 + ❯ refuse src/hosted-extension-protocol.ts:135:21 + ❯ ChildProcess. src/hosted-extension-protocol.ts:234:21 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[10/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > mounts pinned private Surface bytes when the live package changes before launch +Error: ENOENT: no such file or directory, copyfile '/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-48daMa/dist/flow.js' + ❯ surfaceFixture tests/hosted-extension-isolation.test.ts:109:5 + 107| const target = join(surfaceRoot, 'dist', file); + 108| mkdirSync(resolve(target, '..'), { recursive: true }); + 109| copyFileSync(join(sourceRoot, 'dist', file), target); + | ^ + 110| } + 111| return surfaceRoot; + ❯ tests/hosted-extension-isolation.test.ts:437:25 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[11/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > refuses Surface runtime bytes that differ from the reviewed pin before launch +Error: ENOENT: no such file or directory, copyfile '/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-nbb4yF/dist/flow.js' + ❯ surfaceFixture tests/hosted-extension-isolation.test.ts:109:5 + 107| const target = join(surfaceRoot, 'dist', file); + 108| mkdirSync(resolve(target, '..'), { recursive: true }); + 109| copyFileSync(join(sourceRoot, 'dist', file), target); + | ^ + 110| } + 111| return surfaceRoot; + ❯ tests/hosted-extension-isolation.test.ts:462:25 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[12/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > refuses oversized Surface files through the bounded descriptor reader +Error: ENOENT: no such file or directory, copyfile '/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-0JhPEY/dist/flow.js' + ❯ surfaceFixture tests/hosted-extension-isolation.test.ts:109:5 + 107| const target = join(surfaceRoot, 'dist', file); + 108| mkdirSync(resolve(target, '..'), { recursive: true }); + 109| copyFileSync(join(sourceRoot, 'dist', file), target); + | ^ + 110| } + 111| return surfaceRoot; + ❯ tests/hosted-extension-isolation.test.ts:483:25 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[13/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > shields verified Surface files before async settlement +Error: ENOENT: no such file or directory, copyfile '/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/surface/dist/flow.js' -> '/tmp/hosted-surface-test-SpYDeT/dist/flow.js' + ❯ surfaceFixture tests/hosted-extension-isolation.test.ts:109:5 + 107| const target = join(surfaceRoot, 'dist', file); + 108| mkdirSync(resolve(target, '..'), { recursive: true }); + 109| copyFileSync(join(sourceRoot, 'dist', file), target); + | ^ + 110| } + 111| return surfaceRoot; + ❯ tests/hosted-extension-isolation.test.ts:507:25 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[14/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > preserves a typed host refusal while disclosing only a fixed marker to the child +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to be Error: private Cloud policy detail { code: '…' } // Object.is equality + +- Expected ++ Received + +- [Error: private Cloud policy detail] ++ [Error: bubblewrap is unavailable] + + ❯ tests/hosted-extension-isolation.test.ts:560:5 + 558| provider: 'github', eventType: 'pull_request.labeled', deliveryI… + 559| }); + 560| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 561| artifact: await artifact(source), manifest: validateFlowExtensio… + 562| babysitterTurn: { queue: async () => { throw refusal; } }, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[15/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > denies ambient credentials, host files, writes, network, subprocesses, and undeclared context verbs +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:624:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[16/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > enforces OS address-space and data bounds on native Buffer allocation +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + +- Expected ++ Received + +- Object { ++ PluginError { + "code": "plugin_unsupported", +- "message": StringMatching /(?:Failed to allocate memory|Array buffer allocation failed)/u, + } + + ❯ tests/hosted-extension-isolation.test.ts:646:5 + 644| }); + 645| let calls = 0; + 646| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 647| artifact: installed, + 648| manifest: validateFlowExtensionManifest(manifest()), + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[17/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > blocks extra handler fields and authority-bearing receipt fields at the parent port +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + +- Expected ++ Received + +- Object { +- "code": "plugin_event_unroutable", ++ PluginError { ++ "code": "plugin_unsupported", + } + + ❯ tests/hosted-extension-isolation.test.ts:675:5 + 673| }); + 674| let calls = 0; + 675| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 676| artifact: await artifact(source), manifest: validateFlowExtensio… + 677| babysitterTurn: { queue: async () => { calls += 1; return { rece… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[18/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > constructs adapter authority with the captured freeze intrinsic +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:745:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[19/69]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > writes the Surface manifest and protocol without inherited toJSON behavior +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:788:11 + 786| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 787| timeoutMs: 3_000, + 788| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 789| } finally { + 790| if (previous === undefined) delete (Object.prototype as { toJS… + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-isolation.test.ts:781:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[20/69]⎯ + + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > uses captured JSON intrinsics for the complete parent boundary +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-protocol.test.ts:309:24 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[21/69]⎯ + + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects an import-time different PR frame with zero adapter calls + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects an import-time different delivery frame with zero adapter calls + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects an import-time different event frame with zero adapter calls +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + +- Expected ++ Received + +- Object { +- "code": "plugin_event_unroutable", ++ PluginError { ++ "code": "plugin_unsupported", + } + + ❯ tests/hosted-extension-protocol.test.ts:430:5 + 428| ])('rejects an import-time %s frame with zero adapter calls', async … + 429| let calls = 0; + 430| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 431| artifact: await artifact(hostileImport([frame, { type: 'error', … + 432| manifest: validateFlowExtensionManifest(manifest()), dispatch: d… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[22/69]⎯ + + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects two forged calls after the authoritative first outcome settles + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > waits for a pending adapter to reject after a forged child error + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > waits for a pending adapter to resolve after a forged child error + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > returns a typed adapter rejection even when the hostile child hangs +Error: hostile child did not invoke the adapter + ❯ Timeout._onTimeout tests/hosted-extension-protocol.test.ts:118:45 + 116| async function waitForInvocation(invoked: Promise): Promise((resolve, reject) => { + 118| const timeout = setTimeout(() => reject(new Error('hostile child d… + | ^ + 119| void invoked.then(() => { clearTimeout(timeout); resolve(); }, rej… + 120| }); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[23/69]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > runs hn-monitor analyze-story end-to-end via a stub agent CLI (gate 2 clause 2 demo) +AssertionError: expected { …(12) } to match object { output: { …(3) }, …(1) } +(22 matching properties omitted from actual) + +- Expected ++ Received + + Object { +- "output": Object { +- "reasoning": "stub agent runtime — deterministic output for gate-2 clause-2 demo", +- "relevance_score": 5, +- "story_title": "stub", +- }, ++ "output": null, + "verification": Object { +- "gate": "json_schema", +- "verdict": "pass", ++ "gate": "execution", ++ "verdict": "fail", + }, + } + + ❯ tests/live-kernel.test.ts:657:36 + 655| && (entry as { step_id?: string }).step_id === 'analyze-story', + 656| ) as { payload: { output: unknown; verification: unknown } } | und… + 657| expect(stepCompleted?.payload).toMatchObject({ + | ^ + 658| output: { + 659| story_title: 'stub', + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[24/69]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story FAILS verification when the CLI omits required schema fields +AssertionError: expected { …(12) } to match object { …(3) } +(21 matching properties omitted from actual) + +- Expected ++ Received + + Object { +- "completionReason": "retries_exhausted", ++ "completionReason": "worker_error", + "output": null, + "verification": Object { +- "gate": "json_schema", ++ "gate": "execution", + "verdict": "fail", + }, + } + + ❯ tests/live-kernel.test.ts:752:36 + 750| // its verification record names the json_schema rejection. The re… + 751| // parsed value is nulled before the completion is persisted. + 752| expect(stepCompleted?.payload).toMatchObject({ + | ^ + 753| completionReason: 'retries_exhausted', + 754| output: null, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[25/69]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > agent step preserves the CliResult wrapper as output when the CLI emits non-JSON text +AssertionError: expected null not to be null + ❯ tests/live-kernel.test.ts:823:24 + 821| // here (parseJsonOutput returned null on non-JSON stdout) and + 822| // these assertions would all fail. + 823| expect(output).not.toBeNull(); + | ^ + 824| expect(output.exit_code).toBe(0); + 825| expect(output.stdout_tail).toContain('looked at the story'); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[26/69]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker exposes wake_context to the CLI via RELAYFLOW_WAKE_CONTEXT env var (real analyzer prerequisite) +TypeError: Cannot read properties of null (reading 'story_title') + ❯ tests/live-kernel.test.ts:891:42 + 889| ) as { payload: { output: { story_title: string; reasoning: string… + 890| expect(stepCompleted).toBeDefined(); + 891| expect(stepCompleted!.payload.output.story_title).toBe(`echoed:${s… + | ^ + 892| expect(stepCompleted!.payload.output.reasoning).toContain(String(s… + 893| + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[27/69]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_WAKE_CONTEXT UNSET when the run has no wake_context (undefined-vs-null pin) +TypeError: Cannot read properties of null (reading 'env_present') + ❯ tests/live-kernel.test.ts:958:38 + 956| ) as { payload: { output: { env_present: boolean } } } | undefined; + 957| expect(completed).toBeDefined(); + 958| expect(completed!.payload.output.env_present).toBe(false); + | ^ + 959| + 960| delete process.env.RELAYFLOW_WAKE_CONTEXT; + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[28/69]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model +TypeError: Cannot read properties of null (reading 'story_title') + ❯ tests/live-kernel.test.ts:1194:38 + 1192| expect(completed).toBeDefined(); + 1193| // UNSET, not EMPTY and not the leaked parent value. + 1194| expect(completed!.payload.output.story_title).toBe('model:UNSET'); + | ^ + 1195| + 1196| delete process.env.RELAYFLOW_MODEL; + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[29/69]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +Error: LIVE_ANALYZER_UNAVAILABLE: "/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/preflight/analyze-story-claude-cli" does not identify as relayflows-agent-cli-v1 — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ❯ tests/live-kernel.test.ts:1223:15 + 1221| const notice = `LIVE_ANALYZER_UNAVAILABLE: ${readiness.detail}`; + 1222| if (process.env['RELAYFLOWS_ALLOW_ANALYZER_SKIP'] !== '1') { + 1223| throw new Error( + | ^ + 1224| `${notice} — failing because gate-2 acceptance requires the … + 1225| + 'Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is … + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[30/69]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir +AssertionError: WARNING [unprovable_effects] Step "greet" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [unprovable_effects] Step "shout" command "echo" resolves, but its effects cannot be proven before execution. +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +REFUSED [relayflowd_not_found] No relayflowd binary could be found. Install the runtime package for this host (@relayflows/runtime-linux-x64), or set RELAYFLOWD_BIN to a relayflowd executable. Tried: /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/relayflowd. +: expected 2 to be +0 // Object.is equality + +- Expected ++ Received + +- 0 ++ 2 + + ❯ tests/live-kernel.test.ts:1388:40 + 1386| ]); + 1387| + 1388| expect(first.status, first.stderr).toBe(0); + | ^ + 1389| expect(second.status, second.stderr).toBe(0); + 1390| expect(first.stdout).toContain('completionReason: success'); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[31/69]⎯ + + FAIL tests/live-kernel.test.ts > a relayflow can be scheduled: tick source against live relayflowd > a tick spawns a real run whose step reports the SCHEDULED instant +AssertionError: expected null to deeply equal { schedule_id: 'heartbeat-1m', …(3) } + +- Expected: +Object { + "lag_ms": 43000, + "schedule_id": "heartbeat-1m", + "scheduled_for_ms": 1764000000000, + "slot": 29400000, +} + ++ Received: +null + + ❯ tests/live-kernel.test.ts:1665:39 + 1663| // The bound: the run reports the grid instant and its own lag, so… + 1664| // backfilled run can tell it is running for a slot from the past. + 1665| expect(completed!.payload.output).toEqual({ + | ^ + 1666| schedule_id: 'heartbeat-1m', + 1667| slot: 29_400_000, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[32/69]⎯ + + FAIL tests/provider-trigger-executor.test.ts > the kernel executes compiled 'app_mention' subscriptions with provider isolation and durable dedupe + FAIL tests/provider-trigger-executor.test.ts > the kernel executes compiled 'reaction_added' subscriptions with provider isolation and durable dedupe + FAIL tests/provider-trigger-executor.test.ts > the kernel executes compiled 'pull_request' subscriptions with provider isolation and durable dedupe +Error: spawnSync /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ submit tests/provider-trigger-executor.test.ts:43:89 + 41| steps: [{ id: 'effect', type: 'deterministic', command: `printf ac… + 42| })))); + 43| const submit = (envelope: unknown, key: string, executor = source.na… + | ^ + 44| '--data-dir', dir, 'run', spec, '--event', JSON.stringify({ type: … + 45| ], { encoding: 'utf8', stdio: 'pipe' })) as { matched: boolean; dedu… + ❯ tests/provider-trigger-executor.test.ts:50:12 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[33/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > refuses an 8-character run-id prefix: Cloud has no prefix lookup +AssertionError: expected [Function] to throw error matching /not full Cloud run ids: c649fe14/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/not full Cloud run ids: c649fe14/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[34/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > refuses the whole batch when any id is invalid, rather than dropping it +AssertionError: expected [Function] to throw error matching /not full Cloud run ids: nope!/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/not full Cloud run ids: nope!/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[35/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > refuses an empty batch +AssertionError: expected [Function] to throw error matching /needs runIds/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/needs runIds/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[36/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > refuses a batch too large for the edge step lease +AssertionError: expected [Function] to throw error matching /exceeds the 8 that fit/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/exceeds the 8 that fit/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[37/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > accepts eight ids — the incident batch is inside the bound +AssertionError: promise rejected "TypeError: expected an @relayflows/surfac…" instead of resolving + ❯ tests/stuck-run-triage.test.ts:62:40 + 60| it('accepts eight ids — the incident batch is inside the bound', asy… + 61| const ids = Array.from({ length: 8 }, (_, i) => `${ID_A.slice(0, -… + 62| await expect(drive({ runIds: ids })).resolves.toBeDefined(); + | ^ + 63| }); + 64| }); + +Caused by: TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + ❯ tests/stuck-run-triage.test.ts:62:18 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[38/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > refuses to send the Cloud bearer token to an unapproved origin +AssertionError: expected [Function] to throw error matching /refusing to send the Cloud bearer to…/\ but got 'expected an @relayflows/surface flow …' + +- Expected: +/refusing to send the Cloud bearer token to https:\/\/evil\.example/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[39/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > refuses a non-URL apiUrl +AssertionError: expected [Function] to throw error matching /is not a URL/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/is not a URL/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[40/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > allows an approved origin and uses it in the curl +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:77:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[41/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > defaults to production Cloud +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:82:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[42/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > never publishes a run record the fetch did not produce +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:89:34 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[43/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > names the Worker on every wrangler invocation +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:98:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[44/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > accepts caller-supplied Workers and rejects option-shaped ones +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:107:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[45/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > falls back when GNU timeout is absent, as it is on macOS +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:115:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[46/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > runs the tails concurrently so wall time does not scale with the batch +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:123:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[47/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > records wrangler's own exit status rather than head's +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:129:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[48/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage shell text > parses under both sh and bash +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:137:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[49/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage shell text > collects tails with no GNU timeout on PATH, as on a stock macOS +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:157:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[50/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage agents > declares read-only permissions on every agent +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:176:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[51/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage agents > tells the forensics agents their evidence is untrusted +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:149:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:182:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[52/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage fan-out > refuses a duplicate run id: two tails would share one evidence file +AssertionError: expected [Function] to throw error matching /duplicate runIds: c649fe14-0c2e-4e51-…/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/duplicate runIds: c649fe14-0c2e-4e51-9a6a-4f0d1b0f77aa/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[53/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage fan-out > refuses a duplicate Worker name for the same reason +AssertionError: expected [Function] to throw error matching /duplicate workers: w-one/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/duplicate workers: w-one/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[54/69]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage fan-out > bounds ids x workers, not just ids +AssertionError: expected [Function] to throw error matching /24 concurrent tails, over the 16/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/24 concurrent tails, over the 16/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[55/69]⎯ + + FAIL tests/webhook-live.test.ts > executes and deduplicates 'app_mention' only for its provider and matching payload + FAIL tests/webhook-live.test.ts > executes and deduplicates 'reaction_added' only for its provider and matching payload + FAIL tests/webhook-live.test.ts > executes and deduplicates 'pull_request' only for its provider and matching payload +Error: webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ until tests/webhook-live.test.ts:39:9 + 37| const deadline = Date.now() + 10_000; + 38| while (Date.now() < deadline) { if (await predicate()) return; await… + 39| throw new Error(`webhook integration timed out: ${detail()}`); + | ^ + 40| } + 41| async function daemon(dir: string): Promise { + ❯ daemon tests/webhook-live.test.ts:43:3 + ❯ tests/webhook-live.test.ts:100:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[56/69]⎯ + + FAIL tests/webhook-live.test.ts > flows serve-webhook writes JSON before the daemon starts, then journals and archives exactly once +Error: webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ until tests/webhook-live.test.ts:39:9 + 37| const deadline = Date.now() + 10_000; + 38| while (Date.now() < deadline) { if (await predicate()) return; await… + 39| throw new Error(`webhook integration timed out: ${detail()}`); + | ^ + 40| } + 41| async function daemon(dir: string): Promise { + ❯ daemon tests/webhook-live.test.ts:43:3 + ❯ tests/webhook-live.test.ts:121:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[57/69]⎯ + + FAIL tests/webhook-live.test.ts > replays a dropped file after SIGKILL before spawn +Error: webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ until tests/webhook-live.test.ts:39:9 + 37| const deadline = Date.now() + 10_000; + 38| while (Date.now() < deadline) { if (await predicate()) return; await… + 39| throw new Error(`webhook integration timed out: ${detail()}`); + | ^ + 40| } + 41| async function daemon(dir: string): Promise { + ❯ daemon tests/webhook-live.test.ts:43:3 + ❯ tests/webhook-live.test.ts:137:17 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[58/69]⎯ + + FAIL tests/webhook-live.test.ts > resumes the same journal after SIGKILL after spawn and before acknowledgement +Error: webhook integration timed out: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/debug/relayflowd ENOENT + ❯ until tests/webhook-live.test.ts:39:9 + 37| const deadline = Date.now() + 10_000; + 38| while (Date.now() < deadline) { if (await predicate()) return; await… + 39| throw new Error(`webhook integration timed out: ${detail()}`); + | ^ + 40| } + 41| async function daemon(dir: string): Promise { + ❯ daemon tests/webhook-live.test.ts:43:3 + ❯ tests/webhook-live.test.ts:150:17 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[59/69]⎯ + +⎯⎯⎯⎯⎯⎯ Unhandled Errors ⎯⎯⎯⎯⎯⎯ + +Vitest caught 2 unhandled errors during the test run. +This might cause false positive tests. Resolve unhandled errors to make sure your tests are not affected. + +⎯⎯⎯⎯ Unhandled Rejection ⎯⎯⎯⎯⎯ +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:488:9 + 486| + 487| function unsupported(message: string): never { + 488| throw new PluginError('plugin_unsupported', message); + | ^ + 489| } + 490| + ❯ executable src/hosted-extension-sandbox.ts:469:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:160:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:173:16 + ❯ tests/hosted-extension-protocol.test.ts:454:17 + ❯ node_modules/@vitest/runner/dist/index.js:533:5 + ❯ runTest node_modules/@vitest/runner/dist/index.js:1056:11 + ❯ runSuite node_modules/@vitest/runner/dist/index.js:1205:15 + ❯ runSuite node_modules/@vitest/runner/dist/index.js:1205:15 + ❯ runFiles node_modules/@vitest/runner/dist/index.js:1262:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +This error originated in "tests/hosted-extension-protocol.test.ts" test file. It doesn't mean the error was thrown inside the file itself, but while it was running. +The latest test that might've caused the error is "rejects two forged calls after the authoritative first outcome settles". It might mean one of the following: +- The error was thrown, while Vitest was running this test. +- If the error occurred after the test had been completed, this was the last documented test before it was thrown. + +⎯⎯⎯⎯⎯ Uncaught Exception ⎯⎯⎯⎯⎯ +Error: spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/release/relayflowd ENOENT + ❯ Process.ChildProcess._handle.onexit node:internal/child_process:285:19 + ❯ onErrorNT node:internal/child_process:483:16 + ❯ processTicksAndRejections node:internal/process/task_queues:90:21 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { errno: -2, code: 'ENOENT', syscall: 'spawn /home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/release/relayflowd', path: '/home/daytona/.relayflow-v2-supervisor/durable/repository/kernel/target/release/relayflowd', spawnargs: [ '--data-dir', '/tmp/flows-mcp-daemon-xHMNd5', 'serve' ] } +This error originated in "tests/mcp.test.ts" test file. It doesn't mean the error was thrown inside the file itself, but while it was running. +The latest test that might've caused the error is "authored MCP effects against the real kernel". It might mean one of the following: +- The error was thrown, while Vitest was running this test. +- If the error occurred after the test had been completed, this was the last documented test before it was thrown. +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ + + Test Files 10 failed | 172 passed | 3 skipped (185) + Tests 67 failed | 2810 passed | 26 skipped (2903) + Errors 2 errors + Start at 05:46:25 + Duration 294.12s (transform 3.04s, setup 0ms, collect 47.25s, tests 766.88s, environment 25ms, prepare 8.05s) + diff --git a/evidence/parallel-llm-fix/standalone-baseline-build.txt b/evidence/parallel-llm-fix/standalone-baseline-build.txt new file mode 100644 index 000000000..1001208c9 --- /dev/null +++ b/evidence/parallel-llm-fix/standalone-baseline-build.txt @@ -0,0 +1,6 @@ +Bundled 757 modules in 52ms + + authored-node.mjs 3.1 MB (entry point) + + [116ms] bundle 690 modules + [3ms] compile /tmp/flows-parallel-base diff --git a/evidence/parallel-llm-fix/standalone-baseline-smoke.txt b/evidence/parallel-llm-fix/standalone-baseline-smoke.txt new file mode 100644 index 000000000..99aacb98b --- /dev/null +++ b/evidence/parallel-llm-fix/standalone-baseline-smoke.txt @@ -0,0 +1 @@ +REFUSED [invalid_spec] Flow "/tmp/flows-prompt-lab/examples/prompt-lab/evidence/runtime-findings/runtime-parallel-llm-repro.flow.ts" could not be imported: unknown authored-flow error diff --git a/evidence/parallel-llm-fix/standalone-build-final.txt b/evidence/parallel-llm-fix/standalone-build-final.txt new file mode 100644 index 000000000..1936c935f --- /dev/null +++ b/evidence/parallel-llm-fix/standalone-build-final.txt @@ -0,0 +1,6 @@ +Bundled 759 modules in 54ms + + authored-node.mjs 3.0 MB (entry point) + + [133ms] bundle 692 modules + [8ms] compile /tmp/flows-parallel-fixed diff --git a/evidence/parallel-llm-fix/standalone-build.txt b/evidence/parallel-llm-fix/standalone-build.txt new file mode 100644 index 000000000..f0c247cf8 --- /dev/null +++ b/evidence/parallel-llm-fix/standalone-build.txt @@ -0,0 +1,6 @@ +Bundled 759 modules in 51ms + + authored-node.mjs 3.0 MB (entry point) + + [72ms] bundle 692 modules + [3ms] compile /tmp/flows-parallel-fixed diff --git a/evidence/parallel-llm-fix/standalone-smoke.txt b/evidence/parallel-llm-fix/standalone-smoke.txt new file mode 100644 index 000000000..99aacb98b --- /dev/null +++ b/evidence/parallel-llm-fix/standalone-smoke.txt @@ -0,0 +1 @@ +REFUSED [invalid_spec] Flow "/tmp/flows-prompt-lab/examples/prompt-lab/evidence/runtime-findings/runtime-parallel-llm-repro.flow.ts" could not be imported: unknown authored-flow error diff --git a/evidence/parallel-llm-fix/static-refusals.txt b/evidence/parallel-llm-fix/static-refusals.txt new file mode 100644 index 000000000..309cc56f0 --- /dev/null +++ b/evidence/parallel-llm-fix/static-refusals.txt @@ -0,0 +1,12 @@ + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-preflight.test.ts (3 tests) 120ms + ✓ tests/cli-probe.test.ts (6 tests) 866ms + ✓ tests/preflight.test.ts (67 tests) 108ms + + Test Files 3 passed (3) + Tests 76 passed (76) + Start at 05:46:01 + Duration 1.68s (transform 1.10s, setup 0ms, collect 2.60s, tests 1.09s, environment 0ms, prepare 141ms) + diff --git a/evidence/parallel-llm-fix/test-types-final.txt b/evidence/parallel-llm-fix/test-types-final.txt new file mode 100644 index 000000000..b349a0934 --- /dev/null +++ b/evidence/parallel-llm-fix/test-types-final.txt @@ -0,0 +1,4 @@ + +> @relayflows/sdk@2.0.29 typecheck:tests +> tsc -p tsconfig.tests.json + diff --git a/evidence/parallel-llm-fix/test-types.txt b/evidence/parallel-llm-fix/test-types.txt new file mode 100644 index 000000000..b349a0934 --- /dev/null +++ b/evidence/parallel-llm-fix/test-types.txt @@ -0,0 +1,4 @@ + +> @relayflows/sdk@2.0.29 typecheck:tests +> tsc -p tsconfig.tests.json + diff --git a/evidence/parallel-llm-fix/typecheck-final.txt b/evidence/parallel-llm-fix/typecheck-final.txt new file mode 100644 index 000000000..302fb33bd --- /dev/null +++ b/evidence/parallel-llm-fix/typecheck-final.txt @@ -0,0 +1,4 @@ + +> @relayflows/sdk@2.0.29 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + diff --git a/evidence/parallel-llm-fix/typecheck.txt b/evidence/parallel-llm-fix/typecheck.txt new file mode 100644 index 000000000..302fb33bd --- /dev/null +++ b/evidence/parallel-llm-fix/typecheck.txt @@ -0,0 +1,4 @@ + +> @relayflows/sdk@2.0.29 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + diff --git a/evidence/parallel-llm-fix/worktree-merge.txt b/evidence/parallel-llm-fix/worktree-merge.txt new file mode 100644 index 000000000..7993e7d08 --- /dev/null +++ b/evidence/parallel-llm-fix/worktree-merge.txt @@ -0,0 +1 @@ +fatal: refusing to merge unrelated histories diff --git a/evidence/parallel-llm-fix/worktree.txt b/evidence/parallel-llm-fix/worktree.txt new file mode 100644 index 000000000..46a66689d --- /dev/null +++ b/evidence/parallel-llm-fix/worktree.txt @@ -0,0 +1,2 @@ +Preparing worktree (detached HEAD 735f2e0) +HEAD is now at 735f2e0 fix(examples): address prompt-lab review — store lock, menus, visit type, commit candidates diff --git a/evidence/parallel-llm-lease/README.md b/evidence/parallel-llm-lease/README.md new file mode 100644 index 000000000..c67f6d115 --- /dev/null +++ b/evidence/parallel-llm-lease/README.md @@ -0,0 +1,62 @@ +# Diagnosis evidence: concurrent `f.llm` dispatched after its lease expired + +Backs `plan.md` at the repo root. This directory records how the reported +failure was **reproduced and localized before the fix**, not how the fix was +verified. Post-fix commands and captured outputs live in +[../parallel-llm-fix/README.md](../parallel-llm-fix/README.md). + +## Harness + +`diagnosis-harness.test.ts` is a throwaway vitest file that was run from +`packages/sdk/tests/` and then removed. It is kept here verbatim so the +numbers below can be reproduced: copy it back to +`packages/sdk/tests/zz-scratch-repro.test.ts` and run the commands recorded in +each transcript. + +It builds on the existing `chainFixture` (`packages/sdk/tests/flow-chain-fixture.ts`), +which starts a real `relayflowd`, attaches a real `AgentWorker` + `LlmWorker`, +and points `flows.json` at a fake wrapper CLI. The only change is that the fake +CLI's **preflight probe branches** (`--relayflows-adapter-v1` identification and +`auth status`) log every invocation with a timestamp, and the `auth` branch +sleeps `SCRATCH_PROBE_MS`. Step *execution* is untouched and fast. + +Preconditions: `cd kernel && sh ../ops/cargo.sh build` and `npm run build` in +`packages/sdk`. + +## What each transcript shows + +### `probe-count-capacity1.txt` — the repeated work + +`SCRATCH_PROBE_MS=0`, 9 concurrent `f.llm`, capacity 1. The run succeeds, and +the probe log holds: + + "probeCount": 27, + "byKind": { "identify": 18, "auth": 9 } + +18 of those 27 are preflight (9 identification + 9 `auth status`); the other 9 +identifications are the steps actually executing. One preflight probe round per +`f.llm` call, all of it `spawnSync`. + +### `lease-expired-capacity1.txt` — the consequence + +Same flow, `SCRATCH_PROBE_MS=4000`. The run fails: + + "elapsedMs": 36694, + "failure": "step_failed: journal step \"llm-1\" completed with lease_expired" + + Error: Agent lease is already expired for 01M36BDH5G5CAFP4YVVW6BZ2MY/llm-1. + ❯ armExpiry src/worker-lease.ts:20:13 + ❯ Module.withWorkerLease src/worker-lease.ts:47:5 + ❯ LlmWorker.execute src/llm-worker.ts:60:46 + ❯ JournalClient.onDispatch src/llm-worker.ts:49:26 + +Same error string and same stack as the issue report. Elapsed ≈ 9 × 4 s: the +event loop spent the whole run inside `spawnSync`. `llm-1` — the *first* call, +the only one that ever held a lease at capacity 1 — is the attempt that expired. + +### `lease-expired-capacity4.txt` — capacity is not the variable + +Identical run at the default capacity 4: same failure, same step, elapsed +36801 ms. The probe train runs *before* `WorkerSlots.run` is ever reached, so +admission capacity cannot bound it. This is why the issue reproduces at +`--agent-capacity 4` and `1` alike. diff --git a/evidence/parallel-llm-lease/diagnosis-harness.test.ts b/evidence/parallel-llm-lease/diagnosis-harness.test.ts new file mode 100644 index 000000000..cf53cdb6a --- /dev/null +++ b/evidence/parallel-llm-lease/diagnosis-harness.test.ts @@ -0,0 +1,94 @@ +import { readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { flow } from '@relayflows/surface'; +import { executeAuthoredFlow } from '../src/authored-flow-executor.js'; +import { attachLocalAgent } from '../src/local-agent.js'; +import { JournalClient } from '../src/journal-client.js'; +import { LlmWorker } from '../src/llm-worker.js'; +import { socketPathFor } from '../src/daemon-connection.js'; +import { chainFixture } from './flow-chain-fixture.js'; + +const closes: Array<() => Promise> = []; +afterEach(async () => { for (const close of closes.splice(0).reverse()) await close(); }); + +const PROBE_MS = Number(process.env['SCRATCH_PROBE_MS'] ?? '4000'); +const N = Number(process.env['SCRATCH_N'] ?? '9'); + +async function fixtureWithSlowProbe(capacity: number) { + const fixture = chainFixture(); + closes.push(() => fixture.close()); + const probes = join(fixture.root, 'probes.jsonl'); + writeFileSync(fixture.wrapper, `#!/usr/bin/env node +import { receiveWrapperRequest } from ${JSON.stringify(resolve('../../testdata/preflight/wrapper-session.mjs'))}; +import { appendFileSync } from 'node:fs'; +if (process.argv[2] === 'auth') { + appendFileSync(${JSON.stringify(probes)}, JSON.stringify({ kind: 'auth', at: Date.now() }) + '\\n'); + await new Promise(r => setTimeout(r, ${PROBE_MS})); + process.exit(0); +} +if (process.argv[2] === '--relayflows-adapter-v1') { + appendFileSync(${JSON.stringify(probes)}, JSON.stringify({ kind: 'identify', at: Date.now() }) + '\\n'); +} +const request = await receiveWrapperRequest(); +if (request) process.stdout.write('ok'); +`); + const client = await fixture.connect(); + const agent = await attachLocalAgent(client, undefined, undefined, undefined, capacity); + closes.push(() => agent.close()); + const llmClient = new JournalClient(socketPathFor(fixture.data)); + await llmClient.connect(); + await llmClient.hello('scratch-llm-worker'); + closes.push(async () => { llmClient.close(); }); + const llm = new LlmWorker(llmClient, `${agent.stream}-llm`, capacity); + await llm.attach(); + closes.push(() => llm.close()); + const readProbes = () => (existsSync(probes) ? readFileSync(probes, 'utf8').trim().split('\n') + .filter(Boolean).map(l => JSON.parse(l) as { kind: string; at: number }) : []); + return { fixture, client, agent, readProbes }; +} + +const many = flow('many-summaries', async f => { + await Promise.all(Array.from({ length: N }, (_, i) => f.llm`Summarize ${String(i)}`)); + f.done('success'); +}); + +describe('scratch: repeated per-call preflight probes', () => { + it(`runs ${N} concurrent f.llm at capacity 1`, async () => { + const CAP = Number(process.env['SCRATCH_CAP'] ?? '1'); + const { fixture, client, agent, readProbes } = await fixtureWithSlowProbe(CAP); + let failure: unknown; + const started = Date.now(); + let result: Awaited> | undefined; + try { + result = await executeAuthoredFlow(many, client, undefined, { + flowPath: fixture.flowPath, localAgentStream: agent.stream, workerCapacity: CAP, + }); + } catch (error) { failure = error; } + const elapsed = Date.now() - started; + const probes = readProbes(); + // eslint-disable-next-line no-console + console.log(JSON.stringify({ + elapsedMs: elapsed, + probeCount: probes.length, + byKind: probes.reduce((acc, p) => ({ ...acc, [p.kind]: (acc[p.kind] ?? 0) + 1 }), {} as Record), + failure: failure instanceof Error ? failure.message : failure, + completionReason: result?.completionReason, + }, null, 2)); + + // Scan every child run journal for a lease_expired attempt. + const runIds = (result?.journalSteps ?? []).map(s => s.runId); + const expired: string[] = []; + for (const runId of runIds) { + const { entries } = await client.journalRead(runId, 1, 500); + for (const raw of entries as Array<{ entry_type: string; step_id?: string; payload?: { completionReason?: string } }>) { + if (raw.entry_type === 'step.completed' && raw.payload?.completionReason === 'lease_expired') { + expired.push(`${runId}/${raw.step_id}`); + } + } + } + // eslint-disable-next-line no-console + console.log('lease_expired attempts: ' + JSON.stringify(expired)); + expect(true).toBe(true); + }, 300_000); +}); diff --git a/evidence/parallel-llm-lease/lease-expired-capacity1.txt b/evidence/parallel-llm-lease/lease-expired-capacity1.txt new file mode 100644 index 000000000..20c5d607d --- /dev/null +++ b/evidence/parallel-llm-lease/lease-expired-capacity1.txt @@ -0,0 +1,55 @@ +$ SCRATCH_PROBE_MS=4000 SCRATCH_N=9 SCRATCH_CAP=1 npx vitest run tests/zz-scratch-repro.test.ts + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +stdout | tests/zz-scratch-repro.test.ts > scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 +{ + "elapsedMs": 36694, + "probeCount": 19, + "byKind": { + "identify": 10, + "auth": 9 + }, + "failure": "step_failed: journal step \"llm-1\" completed with lease_expired\nInspect: flows replay 01M36BDH5G5CAFP4YVVW6BZ2MY --at llm-1" +} +lease_expired attempts: [] + + ✓ tests/zz-scratch-repro.test.ts (1 test) 36750ms + ✓ scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 36749ms + +⎯⎯⎯⎯⎯⎯ Unhandled Errors ⎯⎯⎯⎯⎯⎯ + +Vitest caught 1 unhandled error during the test run. +This might cause false positive tests. Resolve unhandled errors to make sure your tests are not affected. + +⎯⎯⎯⎯ Unhandled Rejection ⎯⎯⎯⎯⎯ +Error: Agent lease is already expired for 01M36BDH5G5CAFP4YVVW6BZ2MY/llm-1. + ❯ armExpiry src/worker-lease.ts:20:13 + 18| const remaining = deadline - Date.now(); + 19| if (!Number.isFinite(remaining) || remaining <= 0) { + 20| throw new Error(`Agent lease is already expired for ${dispatch.r… + | ^ + 21| } + 22| latestDeadline = deadline; + ❯ Module.withWorkerLease src/worker-lease.ts:47:5 + ❯ LlmWorker.execute src/llm-worker.ts:60:46 + ❯ JournalClient.onDispatch src/llm-worker.ts:49:26 + ❯ JournalClient.emit node:events:508:20 + ❯ JournalClient.onLine src/journal-client.ts:168:12 + ❯ JournalClient.onData src/journal-client.ts:144:33 + ❯ Socket. src/journal-client.ts:109:43 + ❯ Socket.emit node:events:508:20 + ❯ addChunk node:internal/streams/readable:564:12 + +This error originated in "tests/zz-scratch-repro.test.ts" test file. It doesn't mean the error was thrown inside the file itself, but while it was running. +The latest test that might've caused the error is "runs 9 concurrent f.llm at capacity 1". It might mean one of the following: +- The error was thrown, while Vitest was running this test. +- If the error occurred after the test had been completed, this was the last documented test before it was thrown. +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ + + Test Files 1 passed (1) + Tests 1 passed (1) + Errors 1 error + Start at 05:21:59 + Duration 37.90s (transform 545ms, setup 0ms, collect 971ms, tests 36.75s, environment 0ms, prepare 44ms) + diff --git a/evidence/parallel-llm-lease/lease-expired-capacity4.txt b/evidence/parallel-llm-lease/lease-expired-capacity4.txt new file mode 100644 index 000000000..42343a538 --- /dev/null +++ b/evidence/parallel-llm-lease/lease-expired-capacity4.txt @@ -0,0 +1,55 @@ +$ SCRATCH_PROBE_MS=4000 SCRATCH_N=9 SCRATCH_CAP=4 npx vitest run tests/zz-scratch-repro.test.ts + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +stdout | tests/zz-scratch-repro.test.ts > scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 +{ + "elapsedMs": 36801, + "probeCount": 25, + "byKind": { + "identify": 16, + "auth": 9 + }, + "failure": "step_failed: journal step \"llm-1\" completed with lease_expired\nInspect: flows replay 01M36BETK0AMZ3DXPJW2MVCV51 --at llm-1" +} +lease_expired attempts: [] + + ✓ tests/zz-scratch-repro.test.ts (1 test) 36866ms + ✓ scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 36865ms + +⎯⎯⎯⎯⎯⎯ Unhandled Errors ⎯⎯⎯⎯⎯⎯ + +Vitest caught 1 unhandled error during the test run. +This might cause false positive tests. Resolve unhandled errors to make sure your tests are not affected. + +⎯⎯⎯⎯ Unhandled Rejection ⎯⎯⎯⎯⎯ +Error: Agent lease is already expired for 01M36BETK0AMZ3DXPJW2MVCV51/llm-1. + ❯ armExpiry src/worker-lease.ts:20:13 + 18| const remaining = deadline - Date.now(); + 19| if (!Number.isFinite(remaining) || remaining <= 0) { + 20| throw new Error(`Agent lease is already expired for ${dispatch.r… + | ^ + 21| } + 22| latestDeadline = deadline; + ❯ Module.withWorkerLease src/worker-lease.ts:47:5 + ❯ LlmWorker.execute src/llm-worker.ts:60:46 + ❯ JournalClient.onDispatch src/llm-worker.ts:49:26 + ❯ JournalClient.emit node:events:508:20 + ❯ JournalClient.onLine src/journal-client.ts:168:12 + ❯ JournalClient.onData src/journal-client.ts:144:33 + ❯ Socket. src/journal-client.ts:109:43 + ❯ Socket.emit node:events:508:20 + ❯ addChunk node:internal/streams/readable:564:12 + +This error originated in "tests/zz-scratch-repro.test.ts" test file. It doesn't mean the error was thrown inside the file itself, but while it was running. +The latest test that might've caused the error is "runs 9 concurrent f.llm at capacity 1". It might mean one of the following: +- The error was thrown, while Vitest was running this test. +- If the error occurred after the test had been completed, this was the last documented test before it was thrown. +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ + + Test Files 1 passed (1) + Tests 1 passed (1) + Errors 1 error + Start at 05:22:42 + Duration 38.06s (transform 576ms, setup 0ms, collect 1.02s, tests 36.87s, environment 0ms, prepare 53ms) + diff --git a/evidence/parallel-llm-lease/probe-count-capacity1.txt b/evidence/parallel-llm-lease/probe-count-capacity1.txt new file mode 100644 index 000000000..b3d5e1170 --- /dev/null +++ b/evidence/parallel-llm-lease/probe-count-capacity1.txt @@ -0,0 +1,26 @@ +$ SCRATCH_PROBE_MS=0 SCRATCH_N=9 SCRATCH_CAP=1 npx vitest run tests/zz-scratch-repro.test.ts + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +stdout | tests/zz-scratch-repro.test.ts > scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 +{ + "elapsedMs": 1182, + "probeCount": 27, + "byKind": { + "identify": 18, + "auth": 9 + }, + "completionReason": "success" +} + +stdout | tests/zz-scratch-repro.test.ts > scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 +lease_expired attempts: [] + + ✓ tests/zz-scratch-repro.test.ts (1 test) 1253ms + ✓ scratch: repeated per-call preflight probes > runs 9 concurrent f.llm at capacity 1 1253ms + + Test Files 1 passed (1) + Tests 1 passed (1) + Start at 05:21:53 + Duration 2.40s (transform 547ms, setup 0ms, collect 976ms, tests 1.25s, environment 0ms, prepare 50ms) + diff --git a/packages/sdk/src/authored-preflight.ts b/packages/sdk/src/authored-preflight.ts new file mode 100644 index 000000000..93d3ef43f --- /dev/null +++ b/packages/sdk/src/authored-preflight.ts @@ -0,0 +1,51 @@ +import { dirname, resolve } from 'node:path'; +import { checkAuthoredFlow, readProjectConfig, type ProjectConfig } from './cli/check.js'; +import { probeCliAsync } from './cli/cli-probe.js'; +import { communicationInstruction } from './communication/spec.js'; +import { cliProbeKey, CliProbeError, resolvePreflight, type CliProbeOutcome } from './preflight.js'; +import type { FlowSpec } from './spec.js'; + +/** Auth/model facts belong to one execution, including concurrent cold callers. */ +export function authoredPreflight(path: string) { + const probeCache = new Map(); + const pending = new Map>(); + const directory = dirname(resolve(path)); + let communicationChecked = false; + return async (flow: FlowSpec) => { + let config: ProjectConfig; + try { config = readProjectConfig(directory); } + catch { return checkAuthoredFlow(flow, path); } + const resolved = resolvePreflight(flow, { + projectCli: config.cli, projectConfigPath: config.path, projectSearchStart: directory, + models: config.models, modelRegistryPath: config.modelRegistryPath, + }); + if (!resolved.ok) return checkAuthoredFlow(flow, path, { projectConfig: config }); + await Promise.all(resolved.resolutions.map(async resolution => { + const step = resolved.compiled!.steps.find(step => step.id === resolution.stepId)!; + const managed = step.type === 'agent' && communicationInstruction(step.instruction) !== undefined; + const key = cliProbeKey(resolution, managed); + if (probeCache.has(key)) return; + let probing = pending.get(key); + if (probing === undefined) { + probing = (async () => { + try { + probeCache.set(key, { result: await probeCliAsync(resolution.cli, + resolution.source === 'project' ? config.directory : directory, + resolution.model, managed ? 'managed' : undefined) }); + } catch (error) { + probeCache.set(key, { failure: error instanceof CliProbeError ? error.detail : null }); + } + })(); + pending.set(key, probing); + } + await probing; + pending.delete(key); + })); + const result = checkAuthoredFlow(flow, path, { projectConfig: config, probeCache, communicationChecked }); + if (result.report.ok && flow.steps.some(step => + step.type === 'agent' && communicationInstruction(step.instruction) !== undefined)) { + communicationChecked = true; + } + return result; + }; +} diff --git a/packages/sdk/src/authored-worker-step.ts b/packages/sdk/src/authored-worker-step.ts index a42244ec6..8008bce56 100644 --- a/packages/sdk/src/authored-worker-step.ts +++ b/packages/sdk/src/authored-worker-step.ts @@ -3,7 +3,7 @@ import type { AuthoredBudget } from './authored-budget.js'; import { parseBudget } from './budget.js'; import type { AgentOptions, AgentResult, LlmOptions, NamedGate } from '@relayflows/surface'; import { compileSpec, toKernelSpec } from './compile.js'; -import { checkAuthoredFlow } from './cli/check.js'; +import { authoredPreflight } from './authored-preflight.js'; import { classifyOutcome, type RunLifecycleOptions, type RunReport } from './cli/run.js'; import type { PreflightDiagnostic } from './preflight.js'; import { AuthoredFlowExecutionError } from './authored-flow-error.js'; @@ -30,6 +30,7 @@ export function authoredWorkerRunner( // slot instead of being admitted and parked for want of a free worker. const slots = workerCapacity === undefined ? undefined : { agent: new WorkerSlots(workerCapacity), llm: new WorkerSlots(workerCapacity) }; + const check = authoredPreflight(flowPath); const context: AuthoredStepContext = { ...(rootRunId === undefined ? {} : { rootRunId }), ...(waitOptions.dataDir === undefined ? {} : { dataDir: waitOptions.dataDir }), @@ -43,7 +44,7 @@ export function authoredWorkerRunner( // (cli/check.ts), searching for the nearest flows.json from `flowPath` // and real-probing auth/model readiness. An authored agent step gets // nothing for free just because it was declared in TS instead of YAML. - const { report, flow: resolved } = checkAuthoredFlow(authoring, flowPath); + const { report, flow: resolved } = await check(authoring); if (!report.ok || resolved === undefined) { const refusal = report.diagnostics.find( (diagnostic): diagnostic is PreflightDiagnostic & { severity: 'refusal' } => diff --git a/packages/sdk/src/cli/bundle-preflight.ts b/packages/sdk/src/cli/bundle-preflight.ts index bc0e98a36..f10112eb3 100644 --- a/packages/sdk/src/cli/bundle-preflight.ts +++ b/packages/sdk/src/cli/bundle-preflight.ts @@ -23,6 +23,6 @@ export async function checkRunnableBundle(directory: string, name: string): Prom // This deterministic-only slice declares no project models or executors. // Never discover flows.json in the deployment/cache tree or its ancestors. return checkAuthoredFlow(flow, join(directory, 'spec.canonical.json'), { - directory, models: [], executors: [], + projectConfig: { directory, models: [], executors: [] }, }); } diff --git a/packages/sdk/src/cli/check.ts b/packages/sdk/src/cli/check.ts index 6a91c4107..a4ff7f3a3 100644 --- a/packages/sdk/src/cli/check.ts +++ b/packages/sdk/src/cli/check.ts @@ -1,22 +1,12 @@ import { communicationInstruction } from '../communication/spec.js'; import { checkCommunicationEnvironment } from '../communication/preflight.js'; -import { agentEnvironment, brokerEnvironment } from '../communication/environment.js'; import { accessSync, constants, readFileSync } from 'node:fs'; import { dirname, isAbsolute, join, parse as parsePath, resolve } from 'node:path'; -import { spawnSync } from 'node:child_process'; import { parse as parseYaml } from 'yaml'; import { CompileError, compileSpec, kernelToAuthoring } from '../compile.js'; import { helperReady } from '../yaml-helper-effect.js'; import { flowRequirements, type FlowRequirements } from '../flow-requirements.js'; -import { - adapterIdentification, - authenticationProbe, - cliAdapterKind, - displayInvocation, - modelReadinessProbe, - type CliInvocation, -} from '../cli-adapter.js'; -import { MODEL_ENV } from '../worker-cli.js'; +import { probeCli, resolveExecutable } from './cli-probe.js'; import { modelNameError } from '../model-name.js'; import type { FlowSpec } from '../spec.js'; import type { McpServerConfig } from '../spec.js'; @@ -25,9 +15,7 @@ import type { StepGateInspection } from '../gate-contract.js'; import type { CheckFailureKind, CheckWarningKind } from '../failure-kinds.js'; import { preflight, - CliProbeError, type CliResolution, - type CliProbeResult, type PreflightDiagnostic, type PreflightProbes, } from '../preflight.js'; @@ -163,11 +151,17 @@ function safeRequirements(authoring: FlowSpec, projectCli: string | undefined): } } +export interface AuthoredCheckOptions { + projectConfig?: ProjectConfig; + probeCache?: Map; + communicationChecked?: boolean; +} + /** Preflight a validated authored flow through the same path as YAML/JSON. */ -export function checkAuthoredFlow(authoring: FlowSpec, path: string, projectConfig?: ProjectConfig): CheckExecution { +export function checkAuthoredFlow(authoring: FlowSpec, path: string, options: AuthoredCheckOptions = {}): CheckExecution { const absolutePath = resolve(path); try { - const config = projectConfig ?? readProjectConfig(dirname(absolutePath)); + const config = options.projectConfig ?? readProjectConfig(dirname(absolutePath)); const probes = systemProbes(dirname(absolutePath), config); const result = preflight(authoring, { projectCli: config.cli, @@ -176,6 +170,7 @@ export function checkAuthoredFlow(authoring: FlowSpec, path: string, projectConf models: config.models, ...(config.modelRegistryPath !== undefined ? { modelRegistryPath: config.modelRegistryPath } : {}), probes, + probeCache: options.probeCache, }); const flow = result.ok ? bindResolvedCliPaths( @@ -186,7 +181,7 @@ export function checkAuthoredFlow(authoring: FlowSpec, path: string, projectConf ) : undefined; if (flow?.steps.some(step => step.type === 'agent' && communicationInstruction(step.instruction))) { - try { checkCommunicationEnvironment(flow); } + try { if (!options.communicationChecked) checkCommunicationEnvironment(flow); } catch (error) { result.ok = false; result.diagnostics.push({ severity: 'refusal', kind: 'probe_failed', @@ -448,155 +443,6 @@ function canonicalCli(cli: string, directory: string): string { return resolve(directory, cli); } -function probeCli( - cli: string, - directory: string, - model?: string, - execution?: 'managed', -): CliProbeResult { - const executable = resolveExecutable(cli, directory); - if (executable === undefined) return { exists: false, authenticated: false }; - const kind = cliAdapterKind(executable); - // Relay owns interactive CLI launch/injection. Its generic PTY path is not - // the headless wrapper protocol; do not demand that protocol from Gemini, - // Cursor, OpenCode, or other interactive tools. Never invent an auth pass. - if (execution === 'managed' && kind === 'relayflows-wrapper-v1') { - return { exists: true, supported: true, authenticated: 'unverified' }; - } - const environment = execution === 'managed' - ? { ...brokerEnvironment(process.env), ...agentEnvironment(executable) } : process.env; - const probe = (invocation: CliInvocation) => runProbe(executable, directory, invocation, environment); - const identification = adapterIdentification(kind); - const identified = probe(identification.invocation); - if ( - identified.status !== 0 - || (identification.expectedStdout !== undefined - && identified.stdout.trim() !== identification.expectedStdout) - ) { - return { exists: true, supported: false, authenticated: false }; - } - const auth = authenticationProbe(kind); - const authCommand = displayInvocation(cli, auth); - if (model === undefined) { - return { - exists: true, - supported: true, - authenticated: probe(auth).status === 0, - authCommand, - }; - } - - const scoped = modelReadinessProbe(kind, model); - const modelCommand = displayInvocation(cli, scoped); - // A successful real provider round trip (or identified wrapper probe) - // proves both auth and exact-model access. On failure, run the adapter's - // actual auth command solely to classify auth vs model access truthfully. - if (probe(scoped).status === 0) { - return { - exists: true, - supported: true, - authenticated: true, - modelAvailable: true, - authCommand, - modelCommand, - }; - } - const authProbe = probe(auth); - const authenticated = authProbe.status === 0; - return { - exists: true, - supported: true, - authenticated, - modelAvailable: false, - authCommand, - modelCommand, - // Only on failure: on success there is nothing to explain, and the output - // is the most identity-bearing thing this function touches. - ...(authenticated - ? {} - : { - authExitCode: authProbe.status, - authFailureDetail: redactProbeOutput( - `${authProbe.stderr}${authProbe.stdout}`, - ).trim().slice(0, 500), - }), - }; -} - -/** - * Redact anything that looks like a credential or an account identifier. - * - * `auth status` output is diagnostic, but it is also the one place an account - * email, org id or token fragment can appear. The point of surfacing it is to - * say WHY a probe failed, which survives redaction; leaking an identity into a - * refusal message that gets pasted into issues does not. - */ -function redactProbeOutput(text: string): string { - return text - .replace(/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, '') - .replace(/\b(sk|pk|oat|rt)[-_][A-Za-z0-9._-]{8,}/gi, '') - .replace(/\b[A-Fa-f0-9]{32,}\b/g, ''); -} - -function runProbe( - executable: string, - directory: string, - invocation: CliInvocation, - environment: NodeJS.ProcessEnv = process.env, -): { status: number | null; stdout: string; stderr: string } { - const env = { ...environment }; - delete env[MODEL_ENV]; - if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; - const result = spawnSync(executable, invocation.args, { - cwd: directory, - encoding: 'utf8', - // stderr was 'ignore'. A failing `auth status` writes its reason there, so - // discarding it made every authentication refusal structurally - // undiagnosable: the refusal could say a probe exited non-zero and never - // what it said. Captured, then redacted at the point of use. - stdio: ['ignore', 'pipe', 'pipe'], - timeout: invocation.timeoutMs, - env, - }); - const failure = classifySpawnFailure(result.error, result.signal, invocation.timeoutMs); - if (failure !== undefined) throw failure; - return { - status: result.status, - stdout: result.stdout, - stderr: result.stderr ?? '', - }; -} - -function resolveExecutable(command: string, directory: string): string | undefined { - if (command.includes('/') || isAbsolute(command)) { - const path = isAbsolute(command) ? command : resolve(directory, command); - try { - accessSync(path, constants.X_OK); - return path; - } catch { - return undefined; - } - } - const result = spawnSync('which', [command], { encoding: 'utf8', timeout: 5_000 }); - const failure = classifySpawnFailure(result.error, result.signal, 5_000); - if (failure !== undefined) throw failure; - return result.status === 0 ? result.stdout.trim() : undefined; -} - -function classifySpawnFailure( - error: Error | undefined, - signal: NodeJS.Signals | null, - timeoutMs: number, -): CliProbeError | undefined { - if (error !== undefined) { - const detail = (error as NodeJS.ErrnoException).code === 'ETIMEDOUT' - ? `timeout:${timeoutMs}ms` as const - : 'spawn_failed' as const; - return new CliProbeError(detail); - } - return signal === null ? undefined : new CliProbeError(`signal:${signal}`); -} - function executableExists(command: string, directory: string): boolean { return resolveExecutable(command, directory) !== undefined; } diff --git a/packages/sdk/src/cli/cli-probe.ts b/packages/sdk/src/cli/cli-probe.ts new file mode 100644 index 000000000..2268f0ad7 --- /dev/null +++ b/packages/sdk/src/cli/cli-probe.ts @@ -0,0 +1,189 @@ +import { accessSync, constants } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; +import { execFile, spawnSync } from 'node:child_process'; +import { agentEnvironment, brokerEnvironment } from '../communication/environment.js'; +import { adapterIdentification, authenticationProbe, cliAdapterKind, displayInvocation, + modelReadinessProbe, type CliInvocation } from '../cli-adapter.js'; +import { MODEL_ENV } from '../worker-cli.js'; +import { CliProbeError, type CliProbeResult } from '../preflight.js'; + +interface ProbeRequest { + executable: string; + directory: string; + invocation: CliInvocation; + environment: NodeJS.ProcessEnv; +} +interface ProbeOutput { status: number | null; stdout: string; stderr: string } + +/** One decision sequence; checking uses synchronous I/O, live flows yield the loop. */ +export function probeCli(...args: Parameters): CliProbeResult { + return driveSync(probeSequence(...args)); +} + +export async function probeCliAsync(...args: Parameters): Promise { + const sequence = probeSequence(...args); + let next = sequence.next(); + while (!next.done) next = sequence.next(await runProbeAsync(next.value)); + return next.value; +} + +export function resolveExecutable(command: string, directory: string): string | undefined { + return driveSync(executableSequence(command, directory)); +} + +function driveSync(sequence: Generator): T { + let next = sequence.next(); + while (!next.done) { + const request = next.value; + const result = spawnSync(request.executable, request.invocation.args, probeOptions(request)); + const failure = classifySpawnFailure(result.error, result.signal, request.invocation.timeoutMs); + if (failure !== undefined) throw failure; + next = sequence.next({ status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' }); + } + return next.value; +} + +function probeOptions({ directory, invocation, environment }: ProbeRequest) { + const env = { ...environment }; + delete env[MODEL_ENV]; + if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; + return { cwd: directory, encoding: 'utf8' as const, timeout: invocation.timeoutMs, + maxBuffer: 1024 * 1024, env }; +} + +function runProbeAsync(request: ProbeRequest): Promise { + return new Promise((resolve, reject) => { + const child = execFile(request.executable, request.invocation.args, probeOptions(request), + (error, stdout, stderr) => { + // Numeric exit codes are probe results; launch, timeout, signal and buffer + // errors are failures to collect a fact, just as in the synchronous driver. + if (error?.killed && typeof error.code !== 'string') return reject(new CliProbeError(`timeout:${request.invocation.timeoutMs}ms`)); + const failure = classifySpawnFailure( + error !== null && typeof error.code !== 'number' && error.signal == null ? error : undefined, + error?.signal ?? null, request.invocation.timeoutMs); + if (failure !== undefined) return reject(failure); + resolve({ status: error === null ? 0 : typeof error.code === 'number' ? error.code : null, + stdout, stderr }); + }); + child.stdin?.end(); + }); +} + +function* probeSequence( + cli: string, + directory: string, + model?: string, + execution?: 'managed', +): Generator { + const executable = yield* executableSequence(cli, directory); + if (executable === undefined) return { exists: false, authenticated: false }; + const kind = cliAdapterKind(executable); + // Relay owns interactive CLI launch/injection. Its generic PTY path is not + // the headless wrapper protocol; do not demand that protocol from Gemini, + // Cursor, OpenCode, or other interactive tools. Never invent an auth pass. + if (execution === 'managed' && kind === 'relayflows-wrapper-v1') { + return { exists: true, supported: true, authenticated: 'unverified' }; + } + const environment = execution === 'managed' + ? { ...brokerEnvironment(process.env), ...agentEnvironment(executable) } : process.env; + const probe = (invocation: CliInvocation): ProbeRequest => ({ executable, directory, invocation, environment }); + const identification = adapterIdentification(kind); + const identified = yield probe(identification.invocation); + if ( + identified.status !== 0 + || (identification.expectedStdout !== undefined + && identified.stdout.trim() !== identification.expectedStdout) + ) { + return { exists: true, supported: false, authenticated: false }; + } + const auth = authenticationProbe(kind); + const authCommand = displayInvocation(cli, auth); + if (model === undefined) { + return { + exists: true, + supported: true, + authenticated: (yield probe(auth)).status === 0, + authCommand, + }; + } + + const scoped = modelReadinessProbe(kind, model); + const modelCommand = displayInvocation(cli, scoped); + // A successful real provider round trip (or identified wrapper probe) + // proves both auth and exact-model access. On failure, run the adapter's + // actual auth command solely to classify auth vs model access truthfully. + if ((yield probe(scoped)).status === 0) { + return { + exists: true, + supported: true, + authenticated: true, + modelAvailable: true, + authCommand, + modelCommand, + }; + } + const authProbe = yield probe(auth); + const authenticated = authProbe.status === 0; + return { + exists: true, + supported: true, + authenticated, + modelAvailable: false, + authCommand, + modelCommand, + // Only on failure: on success there is nothing to explain, and the output + // is the most identity-bearing thing this function touches. + ...(authenticated + ? {} + : { + authExitCode: authProbe.status, + authFailureDetail: redactProbeOutput( + `${authProbe.stderr}${authProbe.stdout}`, + ).trim().slice(0, 500), + }), + }; +} + +/** + * Redact anything that looks like a credential or an account identifier. + * + * `auth status` output is diagnostic, but it is also the one place an account + * email, org id or token fragment can appear. The point of surfacing it is to + * say WHY a probe failed, which survives redaction; leaking an identity into a + * refusal message that gets pasted into issues does not. + */ +function redactProbeOutput(text: string): string { + return text + .replace(/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, '') + .replace(/\b(sk|pk|oat|rt)[-_][A-Za-z0-9._-]{8,}/gi, '') + .replace(/\b[A-Fa-f0-9]{32,}\b/g, ''); +} + +function* executableSequence(command: string, directory: string): Generator { + if (command.includes('/') || isAbsolute(command)) { + const path = isAbsolute(command) ? command : resolve(directory, command); + try { + accessSync(path, constants.X_OK); + return path; + } catch { + return undefined; + } + } + const result = yield { executable: 'which', directory: process.cwd(), + invocation: { args: [command], timeoutMs: 5_000 }, environment: process.env }; + return result.status === 0 ? result.stdout.trim() : undefined; +} + +function classifySpawnFailure( + error: Error | undefined, + signal: NodeJS.Signals | null, + timeoutMs: number, +): CliProbeError | undefined { + if (error !== undefined) { + const detail = (error as NodeJS.ErrnoException).code === 'ETIMEDOUT' + ? `timeout:${timeoutMs}ms` as const + : 'spawn_failed' as const; + return new CliProbeError(detail); + } + return signal === null ? undefined : new CliProbeError(`signal:${signal}`); +} diff --git a/packages/sdk/src/preflight.ts b/packages/sdk/src/preflight.ts index 8cc27caf0..9cb717cf6 100644 --- a/packages/sdk/src/preflight.ts +++ b/packages/sdk/src/preflight.ts @@ -64,7 +64,7 @@ export class CliProbeError extends Error { } } -type CliProbeOutcome = +export type CliProbeOutcome = | { result: CliProbeResult } | { failure: CliProbeFailureDetail | null }; @@ -98,6 +98,7 @@ export interface PreflightOptions { /** Exact, project-owned model allowlist from the nearest flows.json. */ models?: readonly string[]; modelRegistryPath?: string; + probeCache?: Map; probes: PreflightProbes; } @@ -193,7 +194,12 @@ async function probeMcp(result: PreflightResult, options: PreflightOptions): Pro return { ...result, ok: !result.diagnostics.some(d => d.severity === 'refusal'), mcpTools: Object.freeze(inventory) }; } -function preflightSync(flow: unknown, options: PreflightOptions): PreflightResult { +type ResolutionOptions = Omit; + +/** Resolve static declarations before any CLI, model, command or trigger probe. */ +export function resolvePreflight(flow: unknown, options: ResolutionOptions): PreflightResult & { + compiled?: import('./compile.js').CompiledFlowSpec; +} { // Compile before touching any environment fact. `compileSpec` snapshots raw // input into inert data, validates it against the closed authoring schema, // and lowers `output` sugar into its json_schema gate — so the gate plan @@ -234,7 +240,6 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul const cliResolutionDiagnostics: PreflightDiagnostic[] = []; const resolutions: CliResolution[] = []; const resolutionByStep = new Map(); - const cliProbeResults = new Map(); // A pure fact about the compiled snapshot, collected before anything that // can return early. An unresolved CLI, an unknown model or a bad scope all @@ -282,6 +287,16 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul return { ok: false, gates: compiled.steps.map(inspectStepGate), resolutions, diagnostics }; } + return { ok: true, gates: compiled.steps.map(inspectStepGate), resolutions, diagnostics, compiled }; +} + +function preflightSync(flow: unknown, options: PreflightOptions): PreflightResult { + const { compiled, ...result } = resolvePreflight(flow, options); + if (!result.ok || compiled === undefined) return result; + const { diagnostics, resolutions } = result; + const resolutionByStep = new Map(resolutions.map(resolution => [resolution.stepId, resolution])); + const cliProbeResults = options.probeCache ?? new Map(); + for (const step of compiled.steps) { probeNamedGate(step, options.probes, diagnostics); warnOnVacuousGate(step, diagnostics); @@ -326,7 +341,7 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul * fact about the filesystem. Missing manifest means no known mounts — a grant * still parses but refuses as `mount_unknown` in that case. */ -function scopeDiagnostics(flow: FlowSpec, options: PreflightOptions): PreflightRefusal[] { +function scopeDiagnostics(flow: FlowSpec, options: ResolutionOptions): PreflightRefusal[] { const workspace = flow.workspace; const toolsFs = flow.tools?.fs; if (workspace === undefined && toolsFs === undefined) return []; @@ -350,7 +365,7 @@ function scopeDiagnostics(flow: FlowSpec, options: PreflightOptions): PreflightR /** Pure authoring validation: no executable, command, trigger, or daemon probe. */ function unknownModelDiagnostics( flow: FlowSpec, - options: PreflightOptions, + options: ResolutionOptions, resolutionByStep: ReadonlyMap = new Map(), ): PreflightRefusal[] { const diagnostics: PreflightRefusal[] = []; @@ -464,7 +479,7 @@ function unknownModelMessage( return `Step "${stepId}" declares model "${model}"${cliContext}, but it is not listed in ${source}; add the exact model only after verifying that project is allowed to use it.`; } -function unresolvedCliMessage(stepId: string, options: PreflightOptions): string { +function unresolvedCliMessage(stepId: string, options: ResolutionOptions): string { const context = options.projectConfigPath !== undefined ? ` Nearest project config "${options.projectConfigPath}" declares no cli; outer configs are shadowed.` : options.projectSearchStart !== undefined @@ -494,6 +509,10 @@ function resolveCli( return undefined; } +export function cliProbeKey(resolution: CliResolution, managed = false): string { + return JSON.stringify([resolution.cli, resolution.source, resolution.model ?? null, managed]); +} + function probeResolvedCli( resolution: CliResolution, probes: PreflightProbes, @@ -507,7 +526,7 @@ function probeResolvedCli( // Model is part of the key: the same CLI probed with two different models // is two different questions, and caching on the CLI alone would let a // model that the CLI cannot resolve inherit an earlier model's pass. - const cacheKey = JSON.stringify([resolution.cli, resolution.source, resolution.model ?? null, managed]); + const cacheKey = cliProbeKey(resolution, managed); let outcome = cache.get(cacheKey); if (outcome === undefined) { try { diff --git a/packages/sdk/tests/authored-parallel-llm.test.ts b/packages/sdk/tests/authored-parallel-llm.test.ts new file mode 100644 index 000000000..2e60623c0 --- /dev/null +++ b/packages/sdk/tests/authored-parallel-llm.test.ts @@ -0,0 +1,124 @@ +import { readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { flow } from '@relayflows/surface'; +import { executeAuthoredFlow } from '../src/authored-flow-executor.js'; +import { executeDurableAuthoredFlow } from '../src/authored-root.js'; +import { loadAuthoredFlow } from '../src/authored-flow-loader.js'; +import { LlmWorker } from '../src/llm-worker.js'; +import { DEFAULT_LOCAL_AGENT_CAPACITY } from '../src/worker-slots.js'; +import { chainFixture } from './flow-chain-fixture.js'; + +const closes: Array<() => Promise> = []; +afterEach(async () => { for (const close of closes.splice(0).reverse()) await close(); }); + +async function setup(capacity: number, probeDelayMs: number) { + const fixture = chainFixture(); + closes.push(() => fixture.close()); + const probes = join(fixture.root, 'probes.jsonl'); + const spans = join(fixture.root, 'spans.jsonl'); + writeFileSync(probes, ''); + writeFileSync(spans, ''); + writeFileSync(join(fixture.root, 'flows.json'), + JSON.stringify({ cli: fixture.wrapper, models: ['test-model', 'second-model'] })); + writeFileSync(fixture.wrapper, `#!/usr/bin/env node +import { receiveWrapperRequest } from ${JSON.stringify(resolve('../../testdata/preflight/wrapper-session.mjs'))}; +import { appendFileSync } from 'node:fs'; +if (process.argv[2] === 'auth') { + appendFileSync(${JSON.stringify(probes)}, JSON.stringify(process.env.RELAYFLOW_MODEL) + '\\n'); + await new Promise(done => setTimeout(done, ${probeDelayMs})); + process.exit(0); +} +const request = await receiveWrapperRequest(); +if (request) { + const start = Date.now(); + await new Promise(done => setTimeout(done, 150)); + appendFileSync(${JSON.stringify(spans)}, JSON.stringify({ start, end: Date.now() }) + '\\n'); + process.stdout.write('{"x":1}'); +} +`); + const client = await fixture.connect(); + const peer = client.createPeer(); + await peer.connect(); + await peer.hello('parallel-llm-regression'); + closes.push(async () => { peer.close(); }); + const worker = new LlmWorker(peer, 'parallel-llm', capacity); + await worker.attach(); + closes.push(() => worker.close()); + const lines = (path: string) => readFileSync(path, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); + async function assertJournals(expected: number) { + const ids = readdirSync(join(fixture.data, 'runs')).filter(name => name.endsWith('.sqlite3')); + expect(ids).toHaveLength(expected); + const expired: unknown[] = []; + for (const name of ids) { + const { entries } = await client.journalRead(name.slice(0, -8), 1, 500); + expired.push(...entries.filter(entry => entry.entry_type === 'step.completed' + && (entry.payload as { completionReason?: string })?.completionReason === 'lease_expired')); + } + expect(expired).toEqual([]); + } + return { fixture, client, assertJournals, assertProbes(count: number) { + expect(lines(probes)).toHaveLength(count); + }, assertCapacity() { + const intervals = lines(spans) as Array<{ start: number; end: number }>; + expect(intervals).toHaveLength(9); + const peak = Math.max(...intervals.map(({ start }) => + intervals.filter(other => other.start <= start && start < other.end).length)); + expect(peak).toBeLessThanOrEqual(capacity); + expect(peak).toBeGreaterThan(0); + } }; +} + +const output = { type: 'object', properties: { x: { type: 'number' } }, required: ['x'], additionalProperties: false }; +const many = flow('parallel-llm', async f => { + await Promise.all(Array.from({ length: 9 }, (_, i) => + f.llm(`Return {"x": ${i}+1} as JSON only.`, { output, model: 'test-model' }))); + f.done('success'); +}); + +describe.each([1, DEFAULT_LOCAL_AGENT_CAPACITY])('parallel llm capacity %i', capacity => { + it('deduplicates concurrent preflight probes', async () => { + const test = await setup(capacity, 0); + const result = await executeAuthoredFlow(many, test.client, undefined, { + flowPath: test.fixture.flowPath, workerCapacity: capacity, + }); + expect(result.completionReason).toBe('success'); + test.assertProbes(1); + test.assertCapacity(); + await test.assertJournals(10); + }, 120_000); + + it('completes nine calls without expired child leases during slow preflight', async () => { + // Before the fix, eight remaining probes block dispatch for 48 s (>30 s). + const test = await setup(capacity, 6_000); + const result = await executeAuthoredFlow(many, test.client, undefined, { + flowPath: test.fixture.flowPath, workerCapacity: capacity, + }); + await test.assertJournals(10); + expect(result.completionReason).toBe('success'); + test.assertProbes(1); + test.assertCapacity(); + }, 120_000); + + it('keeps the durable root lease alive across two cold models', async () => { + // A single cold probe exceeds the root's 30 s lease. Cache alone cannot + // rescue this; the process must keep handling heartbeats while probing. + const test = await setup(capacity, 45_000); + writeFileSync(test.fixture.flowPath, ` +import { flow } from '@relayflows/surface'; +export default flow('two-models', async f => { + await Promise.all(Array.from({ length: 9 }, (_, i) => + f.llm('Return {"x":1}', { output: ${JSON.stringify(output)}, model: i % 2 ? 'second-model' : 'test-model' }))); + f.done('success'); +}); +`); + const loaded = await loadAuthoredFlow(test.fixture.flowPath); + const result = await executeDurableAuthoredFlow(loaded, test.client, undefined, { + dataDir: test.fixture.data, workerCapacity: capacity, + }); + await test.assertJournals(11); + expect(result.completionReason).toBe('success'); + test.assertProbes(2); + test.assertCapacity(); + }, 120_000); +}); diff --git a/packages/sdk/tests/authored-preflight.test.ts b/packages/sdk/tests/authored-preflight.test.ts new file mode 100644 index 000000000..a577999f7 --- /dev/null +++ b/packages/sdk/tests/authored-preflight.test.ts @@ -0,0 +1,54 @@ +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, expect, it } from 'vitest'; +import { authoredPreflight } from '../src/authored-preflight.js'; +import { SPEC_SCHEMA_VERSION, type FlowSpec } from '../src/spec.js'; + +const directories: string[] = []; +afterEach(() => { for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); }); +function setup() { + const directory = mkdtempSync(join(tmpdir(), 'authored-preflight-')); + directories.push(directory); + const calls = join(directory, 'calls'); + const cli = join(directory, 'wrapper'); + writeFileSync(cli, `#!/usr/bin/env node +import { appendFileSync } from 'node:fs'; +appendFileSync(${JSON.stringify(calls)}, 'probe\\n'); +if (process.argv[2] === '--relayflows-adapter-v1') console.log('relayflows-agent-cli-v1'); +else process.exit(1); +`); + chmodSync(cli, 0o755); + writeFileSync(join(directory, 'flows.json'), JSON.stringify({ cli, models: ['allowed'] })); + return { calls, check: authoredPreflight(join(directory, 'test.flow.ts')) }; +} +function spec(id: string, model = 'allowed'): FlowSpec { + return { version: SPEC_SCHEMA_VERSION, name: 'test', steps: [{ id, type: 'llm', prompt: 'hello', model }] }; +} + +it('refuses unknown models before launching any provider probe', async () => { + const { check, calls } = setup(); + const result = await check(spec('one', 'forbidden')); + expect(result.report.diagnostics).toContainEqual(expect.objectContaining({ kind: 'model_unknown' })); + expect(existsSync(calls)).toBe(false); +}); + +it('refuses malformed specs before launching any provider probe', async () => { + const { check, calls } = setup(); + const invalid = spec('one'); + (invalid.steps[0] as { prompt: unknown }).prompt = 42; + expect((await check(invalid)).report.ok).toBe(false); + expect(existsSync(calls)).toBe(false); +}); + +it('shares failed facts across callers but retains each step identity', async () => { + const { check, calls } = setup(); + const results = await Promise.all(['one', 'two'].map(id => check(spec(id)))); + for (const [index, result] of results.entries()) { + expect(result.report.diagnostics).toContainEqual(expect.objectContaining({ + kind: 'cli_unauthenticated', stepId: index === 0 ? 'one' : 'two', + })); + } + // One identification, one exact-model probe, one auth classification. + expect(readFileSync(calls, 'utf8').trim().split('\n')).toHaveLength(3); +}); diff --git a/packages/sdk/tests/cli-probe.test.ts b/packages/sdk/tests/cli-probe.test.ts new file mode 100644 index 000000000..3d8934241 --- /dev/null +++ b/packages/sdk/tests/cli-probe.test.ts @@ -0,0 +1,62 @@ +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { probeCli, probeCliAsync } from '../src/cli/cli-probe.js'; +import * as adapters from '../src/cli-adapter.js'; + +const directories: string[] = []; +afterEach(() => { + vi.restoreAllMocks(); + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); +function wrapper(body: string) { + const directory = mkdtempSync(join(tmpdir(), 'probe-drivers-')); + directories.push(directory); + const path = join(directory, 'wrapper'); + writeFileSync(path, '#!/usr/bin/env node\n' + body); + chmodSync(path, 0o755); + return { path, directory }; +} +const identify = `if (process.argv[2] === '--relayflows-adapter-v1') { + console.log('relayflows-agent-cli-v1'); process.exit(0); +}`; + +it.each([ + ['success', 'process.exit(0)', { authenticated: true, modelAvailable: true }], + ['model denied', "process.exit(process.env.RELAYFLOW_MODEL ? 1 : 0)", { authenticated: true, modelAvailable: false }], + ['auth denied', "console.error('person@example.com sk-123456789abcdef'); process.exit(1)", + { authenticated: false, modelAvailable: false, authExitCode: 1, + authFailureDetail: ' ' }], +] as const)('keeps synchronous and asynchronous classification equal: %s', async (_name, body, expected) => { + const { path, directory } = wrapper(identify + body); + const sync = probeCli(path, directory, 'test-model'); + expect(sync).toMatchObject(expected); + expect(await probeCliAsync(path, directory, 'test-model')).toEqual(sync); +}); + +it('keeps missing executables and unsupported identification fail closed', async () => { + const { path, directory } = wrapper("console.log('not a wrapper')"); + expect(await probeCliAsync(path, directory)).toEqual(probeCli(path, directory)); + expect(await probeCliAsync('./missing', directory)).toEqual({ exists: false, authenticated: false }); + expect(await probeCliAsync('relayflows-no-such-executable', directory)).toEqual({ exists: false, authenticated: false }); +}); + +it('reports timeout in both drivers while the async driver leaves the loop free', async () => { + const { path, directory } = wrapper(identify + 'setTimeout(() => {}, 10_000);'); + const original = adapters.modelReadinessProbe; + vi.spyOn(adapters, 'modelReadinessProbe').mockImplementation((kind, model) => + ({ ...original(kind, model), timeoutMs: 100 })); + expect(() => probeCli(path, directory, 'test-model')).toThrow(expect.objectContaining({ detail: 'timeout:100ms' })); + let ticked = false; + const timer = setTimeout(() => { ticked = true; }, 20); + await expect(probeCliAsync(path, directory, 'test-model')).rejects.toMatchObject({ detail: 'timeout:100ms' }); + clearTimeout(timer); + expect(ticked).toBe(true); +}); + +it('reports signal termination in both drivers', async () => { + const { path, directory } = wrapper(identify + "process.kill(process.pid, 'SIGTERM');"); + expect(() => probeCli(path, directory, 'test-model')).toThrow(expect.objectContaining({ detail: 'signal:SIGTERM' })); + await expect(probeCliAsync(path, directory, 'test-model')).rejects.toMatchObject({ detail: 'signal:SIGTERM' }); +}); diff --git a/packages/sdk/tests/communication-preflight.test.ts b/packages/sdk/tests/communication-preflight.test.ts index ddb9cdb8f..7d8507b63 100644 --- a/packages/sdk/tests/communication-preflight.test.ts +++ b/packages/sdk/tests/communication-preflight.test.ts @@ -16,7 +16,7 @@ function fixture(cli: string) { const spec: FlowSpec = { version: '0.1.0', cli: executable, steps: [ { id: 'a', type: 'agent', instruction: 'send' }, { id: 'b', type: 'agent', instruction: 'receive' }, ], communication: { links: [{ from: 'a', to: 'b' }] } }; - return { spec, check: (flow = spec) => checkAuthoredFlow(flow, join(root, 'flow.yaml'), config) }; + return { spec, check: (flow = spec) => checkAuthoredFlow(flow, join(root, 'flow.yaml'), { projectConfig: config }) }; } describe('managed CLI preflight', () => { it('probes known CLIs with the same filtered credentials as execution', () => {